174 lines
7.5 KiB
Python
174 lines
7.5 KiB
Python
"""脱敏。规则移植自 data_collect/redact_archive.py 的思路,不 import 那边任何东西。
|
||
|
||
agent 的 bash 输出会进轨迹:用户目录、主机名、IP、密钥、token、邮箱都要抹。
|
||
同一条轨迹内占位符稳定(同一个用户名每次映射到同一个 <USER_1>),保证轨迹可读。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import hashlib
|
||
import os
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
RULES: list[tuple[str, re.Pattern, str]] = [
|
||
("api_key", re.compile(r"\bsk-[A-Za-z0-9_\-]{10,}\b"), "<API_KEY>"),
|
||
("hf_token", re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"), "<HF_TOKEN>"),
|
||
("bearer", re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._\-]{16,}"), r"\1<TOKEN>"),
|
||
("kv_secret", re.compile(r"(?i)\b((?:api[_-]?key|token|secret|passw(?:or)?d|authorization)\s*[=:]\s*)['\"]?[^\s'\"]{6,}"), r"\1<REDACTED>"),
|
||
("ssh_target", re.compile(r"\b[a-z][a-z0-9._-]{0,31}@(?:\d{1,3}(?:\.\d{1,3}){3}|[a-z0-9.-]+\.[a-z]{2,})\b"), "<SSH_TARGET>"),
|
||
("email", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "<EMAIL>"),
|
||
("ipv4", re.compile(r"\b(?!(?:0|127|10|192\.168)\.)\d{1,3}(?:\.\d{1,3}){3}\b"), "<IP>"),
|
||
("hostname_dr", re.compile(r"\b[a-z0-9-]+\.d-robotics\.cc\b"), "<HOST>"),
|
||
("instance", re.compile(r"\binstance-[a-z0-9]{6,}\b"), "<INSTANCE>"),
|
||
]
|
||
USER_DIR = re.compile(r"(/mnt/data/|/home/)([A-Za-z0-9][A-Za-z0-9._-]{1,31})(?=/|\b)")
|
||
|
||
PRESERVE_VERSION = "source-seeded-redaction/1"
|
||
IDENTITY_FIELDS = frozenset({"id", "role", "type", "toolCallId", "tool_call_id", "call_id",
|
||
"message_id", "source_id", "observation_id", "stopReason", "state",
|
||
"status", "terminal", "execution_terminal", "run_id", "job_id"})
|
||
SECRET_FIELDS = frozenset({"password", "passwd", "api_key", "apikey", "authorization", "secret", "access_token"})
|
||
|
||
|
||
def redact_preserving(value):
|
||
"""Deterministic source-seeded redaction; identity fields are never rewritten.
|
||
|
||
Receipts contain hashes, not the original secrets or usernames. Validation
|
||
re-derives the seeds from the frozen source, never from this host's account.
|
||
Unknown fields survive; every changed scalar has an explicit receipt.
|
||
"""
|
||
def digest(obj):
|
||
return hashlib.sha256(json.dumps(obj, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||
|
||
names = sorted({m.group(2) for m in USER_DIR.finditer(json.dumps(value, ensure_ascii=False))})
|
||
redactor = Redactor()
|
||
redactor.users = {name: f"<USER_{i + 1}>" for i, name in enumerate(names)}
|
||
bare = redactor.bare_user_regex()
|
||
changes, protected = [], []
|
||
|
||
def walk(obj, pointer="", key=None, forced=False):
|
||
if isinstance(obj, dict):
|
||
return {k: walk(v, pointer + "/" + k.replace("~", "~0").replace("/", "~1"), k,
|
||
forced or k.lower().replace("-", "_") in SECRET_FIELDS) for k, v in obj.items()}
|
||
if isinstance(obj, list):
|
||
out = []
|
||
for i, item in enumerate(obj):
|
||
previous = obj[i - 1] if i else None
|
||
secret_arg = isinstance(previous, str) and (
|
||
previous.lstrip("-").lower().replace("-", "_") in SECRET_FIELDS
|
||
or (previous == "-p" and "sshpass" in obj))
|
||
out.append(walk(item, pointer + f"/{i}", key, forced or secret_arg))
|
||
return out
|
||
if forced:
|
||
new = "<REDACTED>"
|
||
elif isinstance(obj, str):
|
||
new = redactor.text(obj)
|
||
if bare:
|
||
new = redactor.bare_pass(new, bare)
|
||
else:
|
||
return obj
|
||
if key in IDENTITY_FIELDS and new != obj:
|
||
protected.append(pointer)
|
||
return obj
|
||
if new != obj:
|
||
changes.append({"pointer": pointer, "rule": "secret-field" if forced else PRESERVE_VERSION,
|
||
"before_sha256": digest(obj), "after_sha256": digest(new)})
|
||
return new
|
||
|
||
output = walk(value)
|
||
return output, {"version": PRESERVE_VERSION, "source_seed_sha256": digest(names),
|
||
"transforms": changes, "protected_sensitive_fields": protected,
|
||
"counts": redactor.counts, "users": len(names)}
|
||
|
||
|
||
class Redactor:
|
||
def __init__(self):
|
||
self.users: dict[str, str] = {}
|
||
self.counts: dict[str, int] = {}
|
||
|
||
def _user(self, m: re.Match) -> str:
|
||
name = m.group(2)
|
||
if name not in self.users:
|
||
self.users[name] = f"<USER_{len(self.users) + 1}>"
|
||
self.counts["user_dir"] = self.counts.get("user_dir", 0) + 1
|
||
return m.group(1) + self.users[name]
|
||
|
||
def text(self, s: str) -> str:
|
||
if not isinstance(s, str) or not s:
|
||
return s
|
||
s = USER_DIR.sub(self._user, s)
|
||
for name, rx, rep in RULES:
|
||
s, n = rx.subn(rep, s)
|
||
if n:
|
||
self.counts[name] = self.counts.get(name, 0) + n
|
||
return s
|
||
|
||
def obj(self, o):
|
||
if isinstance(o, str):
|
||
return self.text(o)
|
||
if isinstance(o, list):
|
||
return [self.obj(x) for x in o]
|
||
if isinstance(o, dict):
|
||
return {k: self.obj(v) for k, v in o.items()}
|
||
return o
|
||
|
||
# ---- 第二遍:裸用户名
|
||
# 第一遍只抹路径里的用户名(/mnt/data/<u>、/home/<u>)。但 agent 的 bash 输出里到处是
|
||
# `ls -la` 的属主/属组列("drwxr-xr-x 4 ming01.wang ming01.wang")、`whoami`、进程表的 USER 列——
|
||
# 那些不是路径,第一遍抓不到。实测第一条轨迹:路径替换 64 处,裸用户名仍残留 62 处。
|
||
def bare_user_regex(self, extra: set[str] | None = None) -> re.Pattern | None:
|
||
names = set(self.users) | (extra or set())
|
||
names = {n for n in names if len(n) >= 3 and not n.isdigit()}
|
||
if not names:
|
||
return None
|
||
return re.compile(r"(?<![A-Za-z0-9._@-])(" + "|".join(re.escape(n) for n in sorted(names, key=len, reverse=True)) + r")(?![A-Za-z0-9-])")
|
||
|
||
def bare_pass(self, o, rx: re.Pattern):
|
||
def sub(s: str) -> str:
|
||
def rep(m: re.Match) -> str:
|
||
name = m.group(1)
|
||
if name not in self.users:
|
||
self.users[name] = f"<USER_{len(self.users) + 1}>"
|
||
self.counts["user_bare"] = self.counts.get("user_bare", 0) + 1
|
||
return self.users[name]
|
||
return rx.sub(rep, s)
|
||
if isinstance(o, str):
|
||
return sub(o)
|
||
if isinstance(o, list):
|
||
return [self.bare_pass(x, rx) for x in o]
|
||
if isinstance(o, dict):
|
||
return {k: self.bare_pass(v, rx) for k, v in o.items()}
|
||
return o
|
||
|
||
|
||
def local_usernames() -> set[str]:
|
||
"""本机可能出现的用户名:当前用户 + /home 下的目录名。用于裸用户名替换的种子。"""
|
||
out = set()
|
||
for k in ("USER", "LOGNAME"):
|
||
v = os.environ.get(k)
|
||
if v:
|
||
out.add(v)
|
||
try:
|
||
out |= {p.name for p in Path("/home").iterdir() if p.is_dir()}
|
||
except OSError:
|
||
pass
|
||
return {n for n in out if len(n) >= 3}
|
||
|
||
|
||
def redact_messages(messages: list[dict]) -> tuple[list[dict], dict]:
|
||
r = Redactor()
|
||
out = r.obj(messages)
|
||
rx = r.bare_user_regex(local_usernames())
|
||
if rx is not None:
|
||
out = r.bare_pass(out, rx)
|
||
return out, {"counts": r.counts, "users": len(r.users)}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
for p in sys.argv[1:]:
|
||
d = json.loads(Path(p).read_text())
|
||
out, stats = redact_messages(d if isinstance(d, list) else [d])
|
||
print(json.dumps({"file": p, **stats}, ensure_ascii=False))
|