data-gen-bpu/export/redact.py

174 lines
7.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""脱敏。规则移植自 data_collect/redact_archive.py 的思路,不 import 那边任何东西。
agent 的 bash 输出会进轨迹:用户目录、主机名、IP、密钥、token、邮箱都要抹。
同一条轨迹内占位符稳定(同一个用户名每次映射到同一个 <USER_1>),保证轨迹可读。
"""
from __future__ import annotations
import json
import hashlib
import os
import re
import sys
from pathlib import Path
RULES: list[tuple[str, re.Pattern, str]] = [
("api_key", re.compile(r"\bsk-[A-Za-z0-9_\-]{10,}\b"), "<API_KEY>"),
("hf_token", re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"), "<HF_TOKEN>"),
("bearer", re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._\-]{16,}"), r"\1<TOKEN>"),
("kv_secret", re.compile(r"(?i)\b((?:api[_-]?key|token|secret|passw(?:or)?d|authorization)\s*[=:]\s*)['\"]?[^\s'\"]{6,}"), r"\1<REDACTED>"),
("ssh_target", re.compile(r"\b[a-z][a-z0-9._-]{0,31}@(?:\d{1,3}(?:\.\d{1,3}){3}|[a-z0-9.-]+\.[a-z]{2,})\b"), "<SSH_TARGET>"),
("email", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "<EMAIL>"),
("ipv4", re.compile(r"\b(?!(?:0|127|10|192\.168)\.)\d{1,3}(?:\.\d{1,3}){3}\b"), "<IP>"),
("hostname_dr", re.compile(r"\b[a-z0-9-]+\.d-robotics\.cc\b"), "<HOST>"),
("instance", re.compile(r"\binstance-[a-z0-9]{6,}\b"), "<INSTANCE>"),
]
USER_DIR = re.compile(r"(/mnt/data/|/home/)([A-Za-z0-9][A-Za-z0-9._-]{1,31})(?=/|\b)")
PRESERVE_VERSION = "source-seeded-redaction/1"
IDENTITY_FIELDS = frozenset({"id", "role", "type", "toolCallId", "tool_call_id", "call_id",
"message_id", "source_id", "observation_id", "stopReason", "state",
"status", "terminal", "execution_terminal", "run_id", "job_id"})
SECRET_FIELDS = frozenset({"password", "passwd", "api_key", "apikey", "authorization", "secret", "access_token"})
def redact_preserving(value):
"""Deterministic source-seeded redaction; identity fields are never rewritten.
Receipts contain hashes, not the original secrets or usernames. Validation
re-derives the seeds from the frozen source, never from this host's account.
Unknown fields survive; every changed scalar has an explicit receipt.
"""
def digest(obj):
return hashlib.sha256(json.dumps(obj, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
names = sorted({m.group(2) for m in USER_DIR.finditer(json.dumps(value, ensure_ascii=False))})
redactor = Redactor()
redactor.users = {name: f"<USER_{i + 1}>" for i, name in enumerate(names)}
bare = redactor.bare_user_regex()
changes, protected = [], []
def walk(obj, pointer="", key=None, forced=False):
if isinstance(obj, dict):
return {k: walk(v, pointer + "/" + k.replace("~", "~0").replace("/", "~1"), k,
forced or k.lower().replace("-", "_") in SECRET_FIELDS) for k, v in obj.items()}
if isinstance(obj, list):
out = []
for i, item in enumerate(obj):
previous = obj[i - 1] if i else None
secret_arg = isinstance(previous, str) and (
previous.lstrip("-").lower().replace("-", "_") in SECRET_FIELDS
or (previous == "-p" and "sshpass" in obj))
out.append(walk(item, pointer + f"/{i}", key, forced or secret_arg))
return out
if forced:
new = "<REDACTED>"
elif isinstance(obj, str):
new = redactor.text(obj)
if bare:
new = redactor.bare_pass(new, bare)
else:
return obj
if key in IDENTITY_FIELDS and new != obj:
protected.append(pointer)
return obj
if new != obj:
changes.append({"pointer": pointer, "rule": "secret-field" if forced else PRESERVE_VERSION,
"before_sha256": digest(obj), "after_sha256": digest(new)})
return new
output = walk(value)
return output, {"version": PRESERVE_VERSION, "source_seed_sha256": digest(names),
"transforms": changes, "protected_sensitive_fields": protected,
"counts": redactor.counts, "users": len(names)}
class Redactor:
def __init__(self):
self.users: dict[str, str] = {}
self.counts: dict[str, int] = {}
def _user(self, m: re.Match) -> str:
name = m.group(2)
if name not in self.users:
self.users[name] = f"<USER_{len(self.users) + 1}>"
self.counts["user_dir"] = self.counts.get("user_dir", 0) + 1
return m.group(1) + self.users[name]
def text(self, s: str) -> str:
if not isinstance(s, str) or not s:
return s
s = USER_DIR.sub(self._user, s)
for name, rx, rep in RULES:
s, n = rx.subn(rep, s)
if n:
self.counts[name] = self.counts.get(name, 0) + n
return s
def obj(self, o):
if isinstance(o, str):
return self.text(o)
if isinstance(o, list):
return [self.obj(x) for x in o]
if isinstance(o, dict):
return {k: self.obj(v) for k, v in o.items()}
return o
# ---- 第二遍:裸用户名
# 第一遍只抹路径里的用户名(/mnt/data/<u>、/home/<u>)。但 agent 的 bash 输出里到处是
# `ls -la` 的属主/属组列("drwxr-xr-x 4 ming01.wang ming01.wang")、`whoami`、进程表的 USER 列——
# 那些不是路径,第一遍抓不到。实测第一条轨迹:路径替换 64 处,裸用户名仍残留 62 处。
def bare_user_regex(self, extra: set[str] | None = None) -> re.Pattern | None:
names = set(self.users) | (extra or set())
names = {n for n in names if len(n) >= 3 and not n.isdigit()}
if not names:
return None
return re.compile(r"(?<![A-Za-z0-9._@-])(" + "|".join(re.escape(n) for n in sorted(names, key=len, reverse=True)) + r")(?![A-Za-z0-9-])")
def bare_pass(self, o, rx: re.Pattern):
def sub(s: str) -> str:
def rep(m: re.Match) -> str:
name = m.group(1)
if name not in self.users:
self.users[name] = f"<USER_{len(self.users) + 1}>"
self.counts["user_bare"] = self.counts.get("user_bare", 0) + 1
return self.users[name]
return rx.sub(rep, s)
if isinstance(o, str):
return sub(o)
if isinstance(o, list):
return [self.bare_pass(x, rx) for x in o]
if isinstance(o, dict):
return {k: self.bare_pass(v, rx) for k, v in o.items()}
return o
def local_usernames() -> set[str]:
"""本机可能出现的用户名:当前用户 + /home 下的目录名。用于裸用户名替换的种子。"""
out = set()
for k in ("USER", "LOGNAME"):
v = os.environ.get(k)
if v:
out.add(v)
try:
out |= {p.name for p in Path("/home").iterdir() if p.is_dir()}
except OSError:
pass
return {n for n in out if len(n) >= 3}
def redact_messages(messages: list[dict]) -> tuple[list[dict], dict]:
r = Redactor()
out = r.obj(messages)
rx = r.bare_user_regex(local_usernames())
if rx is not None:
out = r.bare_pass(out, rx)
return out, {"counts": r.counts, "users": len(r.users)}
if __name__ == "__main__":
for p in sys.argv[1:]:
d = json.loads(Path(p).read_text())
out, stats = redact_messages(d if isinstance(d, list) else [d])
print(json.dumps({"file": p, **stats}, ensure_ascii=False))