116 lines
4.7 KiB
Python
116 lines
4.7 KiB
Python
"""脱敏。规则移植自 data_collect/redact_archive.py 的思路,不 import 那边任何东西。
|
||
|
||
agent 的 bash 输出会进轨迹:用户目录、主机名、IP、密钥、token、邮箱都要抹。
|
||
同一条轨迹内占位符稳定(同一个用户名每次映射到同一个 <USER_1>),保证轨迹可读。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
RULES: list[tuple[str, re.Pattern, str]] = [
|
||
("api_key", re.compile(r"\bsk-[A-Za-z0-9_\-]{10,}\b"), "<API_KEY>"),
|
||
("hf_token", re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"), "<HF_TOKEN>"),
|
||
("bearer", re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._\-]{16,}"), r"\1<TOKEN>"),
|
||
("kv_secret", re.compile(r"(?i)\b((?:api[_-]?key|token|secret|passw(?:or)?d|authorization)\s*[=:]\s*)['\"]?[^\s'\"]{6,}"), r"\1<REDACTED>"),
|
||
("ssh_target", re.compile(r"\b[a-z][a-z0-9._-]{0,31}@(?:\d{1,3}(?:\.\d{1,3}){3}|[a-z0-9.-]+\.[a-z]{2,})\b"), "<SSH_TARGET>"),
|
||
("email", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "<EMAIL>"),
|
||
("ipv4", re.compile(r"\b(?!(?:0|127|10|192\.168)\.)\d{1,3}(?:\.\d{1,3}){3}\b"), "<IP>"),
|
||
("hostname_dr", re.compile(r"\b[a-z0-9-]+\.d-robotics\.cc\b"), "<HOST>"),
|
||
("instance", re.compile(r"\binstance-[a-z0-9]{6,}\b"), "<INSTANCE>"),
|
||
]
|
||
USER_DIR = re.compile(r"(/mnt/data/|/home/)([A-Za-z0-9][A-Za-z0-9._-]{1,31})(?=/|\b)")
|
||
|
||
|
||
class Redactor:
|
||
def __init__(self):
|
||
self.users: dict[str, str] = {}
|
||
self.counts: dict[str, int] = {}
|
||
|
||
def _user(self, m: re.Match) -> str:
|
||
name = m.group(2)
|
||
if name not in self.users:
|
||
self.users[name] = f"<USER_{len(self.users) + 1}>"
|
||
self.counts["user_dir"] = self.counts.get("user_dir", 0) + 1
|
||
return m.group(1) + self.users[name]
|
||
|
||
def text(self, s: str) -> str:
|
||
if not isinstance(s, str) or not s:
|
||
return s
|
||
s = USER_DIR.sub(self._user, s)
|
||
for name, rx, rep in RULES:
|
||
s, n = rx.subn(rep, s)
|
||
if n:
|
||
self.counts[name] = self.counts.get(name, 0) + n
|
||
return s
|
||
|
||
def obj(self, o):
|
||
if isinstance(o, str):
|
||
return self.text(o)
|
||
if isinstance(o, list):
|
||
return [self.obj(x) for x in o]
|
||
if isinstance(o, dict):
|
||
return {k: self.obj(v) for k, v in o.items()}
|
||
return o
|
||
|
||
# ---- 第二遍:裸用户名
|
||
# 第一遍只抹路径里的用户名(/mnt/data/<u>、/home/<u>)。但 agent 的 bash 输出里到处是
|
||
# `ls -la` 的属主/属组列("drwxr-xr-x 4 ming01.wang ming01.wang")、`whoami`、进程表的 USER 列——
|
||
# 那些不是路径,第一遍抓不到。实测第一条轨迹:路径替换 64 处,裸用户名仍残留 62 处。
|
||
def bare_user_regex(self, extra: set[str] | None = None) -> re.Pattern | None:
|
||
names = set(self.users) | (extra or set())
|
||
names = {n for n in names if len(n) >= 3 and not n.isdigit()}
|
||
if not names:
|
||
return None
|
||
return re.compile(r"(?<![A-Za-z0-9._@-])(" + "|".join(re.escape(n) for n in sorted(names, key=len, reverse=True)) + r")(?![A-Za-z0-9-])")
|
||
|
||
def bare_pass(self, o, rx: re.Pattern):
|
||
def sub(s: str) -> str:
|
||
def rep(m: re.Match) -> str:
|
||
name = m.group(1)
|
||
if name not in self.users:
|
||
self.users[name] = f"<USER_{len(self.users) + 1}>"
|
||
self.counts["user_bare"] = self.counts.get("user_bare", 0) + 1
|
||
return self.users[name]
|
||
return rx.sub(rep, s)
|
||
if isinstance(o, str):
|
||
return sub(o)
|
||
if isinstance(o, list):
|
||
return [self.bare_pass(x, rx) for x in o]
|
||
if isinstance(o, dict):
|
||
return {k: self.bare_pass(v, rx) for k, v in o.items()}
|
||
return o
|
||
|
||
|
||
def local_usernames() -> set[str]:
|
||
"""本机可能出现的用户名:当前用户 + /home 下的目录名。用于裸用户名替换的种子。"""
|
||
out = set()
|
||
for k in ("USER", "LOGNAME"):
|
||
v = os.environ.get(k)
|
||
if v:
|
||
out.add(v)
|
||
try:
|
||
out |= {p.name for p in Path("/home").iterdir() if p.is_dir()}
|
||
except OSError:
|
||
pass
|
||
return {n for n in out if len(n) >= 3}
|
||
|
||
|
||
def redact_messages(messages: list[dict]) -> tuple[list[dict], dict]:
|
||
r = Redactor()
|
||
out = r.obj(messages)
|
||
rx = r.bare_user_regex(local_usernames())
|
||
if rx is not None:
|
||
out = r.bare_pass(out, rx)
|
||
return out, {"counts": r.counts, "users": len(r.users)}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
for p in sys.argv[1:]:
|
||
d = json.loads(Path(p).read_text())
|
||
out, stats = redact_messages(d if isinstance(d, list) else [d])
|
||
print(json.dumps({"file": p, **stats}, ensure_ascii=False))
|