From 50edd8c2a1ccef3dcb0e596d8fe4ceecfd4f14fb Mon Sep 17 00:00:00 2001 From: bench Date: Tue, 15 Sep 2026 12:21:38 +0800 Subject: [PATCH] sync 2026-09-15_12:21:38 d-robotics@d-roboticsdeMacBook-Pro --- REPORT-codex.md | 3 + agent/VERSION | 2 +- agent/extensions/budget-kill.ts | 6 +- agent/extensions/claim-card.ts | 2 +- agent/extensions/path-guard.ts | 2 +- bin/bench | 3 + bin/stageclient.py | 111 +++++++++++++++++++++++++++++ export/to_agentrl.py | 57 ++++++++++++--- harness/rollout.py | 23 ++++-- harness/stagerun.py | 22 +++--- harness/stageworker.py | 120 ++++++++++++++++++++++++++++++++ scripts/agent_jail.sh | 73 ++----------------- scripts/audit_rollouts.py | 56 +++++++++++++-- scripts/guard_smoke.sh | 9 +-- scripts/jail_probe.sh | 37 ++++++++++ scripts/jail_setup.py | 90 ++++++++++++++++++++++++ scripts/setup_agent.sh | 0 scripts/sync.sh | 5 +- 18 files changed, 511 insertions(+), 110 deletions(-) create mode 100644 REPORT-codex.md create mode 100644 bin/stageclient.py create mode 100644 harness/stageworker.py create mode 100755 scripts/jail_probe.sh create mode 100644 scripts/jail_setup.py mode change 100644 => 100755 scripts/setup_agent.sh diff --git a/REPORT-codex.md b/REPORT-codex.md new file mode 100644 index 0000000..b9e33ba --- /dev/null +++ b/REPORT-codex.md @@ -0,0 +1,3 @@ +# bench 隔离修复验证报告 + +实施及验证进行中;最终结果将在验证完成后补齐。 diff --git a/agent/VERSION b/agent/VERSION index bf0d87a..7ed6ff8 100644 --- a/agent/VERSION +++ b/agent/VERSION @@ -1 +1 @@ -4 \ No newline at end of file +5 diff --git a/agent/extensions/budget-kill.ts b/agent/extensions/budget-kill.ts index 72e63f8..2b4e950 100644 --- a/agent/extensions/budget-kill.ts +++ b/agent/extensions/budget-kill.ts @@ -28,7 +28,7 @@ let warned = false; function writeRollout(patch: Record) { if (!ROLLOUT) return; - const p = path.resolve(ROLLOUT, "rollout.json"); + const p = path.resolve(ROLLOUT, process.env.BENCH_JAILED === "1" ? "workspace/budget.json" : "rollout.json"); let rec: any = {}; try { rec = JSON.parse(fs.readFileSync(p, "utf8")); } catch { /* 新建 */ } Object.assign(rec, patch, { turns, tokens }); @@ -86,10 +86,10 @@ export default function (pi: ExtensionAPI) { } // 3. 兜底:耗尽后还赖着不走 - if (exhausted && turns >= MAX_TURNS + GRACE_TURNS) { + if (exhausted && turns >= MAX_TURNS) { writeRollout({ hard_killed_at_turn: turns }); try { ctx?.abort?.(); } catch { /* ignore */ } - setTimeout(() => process.exit(75), 2000).unref?.(); + process.exit(75); } }); diff --git a/agent/extensions/claim-card.ts b/agent/extensions/claim-card.ts index 0476c6d..74abe24 100644 --- a/agent/extensions/claim-card.ts +++ b/agent/extensions/claim-card.ts @@ -12,7 +12,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; const ROLLOUT = process.env.BENCH_ROLLOUT_DIR ?? ""; -const CLAIM = ROLLOUT ? path.resolve(ROLLOUT, "claim.json") : path.resolve(process.cwd(), "claim.json"); +const CLAIM = ROLLOUT ? path.resolve(ROLLOUT, process.env.BENCH_JAILED === "1" ? "workspace/claim.json" : "claim.json") : path.resolve(process.cwd(), "claim.json"); let claimed = false; diff --git a/agent/extensions/path-guard.ts b/agent/extensions/path-guard.ts index f19df6c..9f85b6b 100644 --- a/agent/extensions/path-guard.ts +++ b/agent/extensions/path-guard.ts @@ -28,7 +28,7 @@ const MODE = process.env.BENCH_MODE ?? "e2e"; const BENCH_ROOT = process.env.BENCH_ROOT ?? path.resolve(__dirname, "..", ".."); const WORKSPACE = ROLLOUT ? path.resolve(ROLLOUT, "workspace") : process.cwd(); const JOB_DIR = ROLLOUT ? path.resolve(ROLLOUT, "..", "..") : path.resolve(WORKSPACE, "..", "..", ".."); -const GUARD_LOG = ROLLOUT ? path.resolve(ROLLOUT, "agent", "guard.log") : ""; +const GUARD_LOG = ROLLOUT ? path.resolve(ROLLOUT, process.env.BENCH_JAILED === "1" ? "workspace" : "agent", "guard.log") : ""; // harness/rollout.py agent_env 传入:BENCH_GUARD_DENY = realpath(state):work_root(都是指向 /home 的链接/本地盘,绕开项目树白名单); // BENCH_GUARD_ALLOW = 本 rollout 在 work_root 下的编译工作目录。 diff --git a/bin/bench b/bin/bench index 75cb523..dd2b488 100755 --- a/bin/bench +++ b/bin/bench @@ -5,4 +5,7 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" export BENCH_ROOT="${BENCH_ROOT:-$(dirname "$HERE")}" export PYTHONPATH="$BENCH_ROOT${PYTHONPATH:+:$PYTHONPATH}" PY="${BENCH_PYTHON:-python3}" +if [ "${BENCH_JAILED:-}" = 1 ]; then + exec "$PY" "$HERE/stageclient.py" "$@" +fi exec "$PY" -m harness.cli "$@" diff --git a/bin/stageclient.py b/bin/stageclient.py new file mode 100644 index 0000000..ce58404 --- /dev/null +++ b/bin/stageclient.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Untrusted, stdlib-only stage ticket client. Authority lives in the outer worker.""" +import argparse +import json +import os +import re +import sys +import time +import uuid +from pathlib import Path + +NAMES = {n.split('-', 1)[1]: n for n in ('04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval')} + +def rollout_dir(): + return Path(os.environ['BENCH_ROLLOUT_DIR']) + +def submit(stage, config, poll_hint=60): + name = NAMES.get(stage, stage) + if name not in NAMES.values() and name != 'noop': + raise SystemExit('阶段不允许: ' + name) + rd = rollout_dir() + cfg = None + if config: + ws = rd / 'workspace' + cfg = Path(config) + if not cfg.is_absolute(): + if cfg.parts[0] == 'workspace': + cfg = Path(*cfg.parts[1:]) + cfg = ws / cfg + cfg = cfg.resolve() + if not cfg.is_relative_to(ws.resolve()) or not cfg.is_file(): + raise SystemExit('--config 必须是 workspace 内文件') + hid = name + '-' + uuid.uuid4().hex + hd = rd / 'handles' / hid + hd.mkdir() + (hd / 'status.json').write_text(json.dumps({'state': 'QUEUED', 'queued_at': time.time()})) + tmp = hd / 'request.tmp' + tmp.write_text(json.dumps({'stage': name, 'config': str(cfg) if cfg else None, 'submitted_at': time.time()})) + tmp.rename(hd / 'request.json') + print(hid) + print(f'# wait: bench stage wait {hid} --poll {poll_hint}') + return hid + +def wait(handle: str, poll: int = 60, quiet: bool = False, max_wait_s: int | None = None) -> int: + """阻塞到该阶段结束。每轮打一行心跳(排队中 / 运行中 + 已耗时),这样 agent 侧看得见进展, + pi 的 bash 工具也不会因为"长时间无输出"而误判。max_wait_s 到点就返回 2 并提示继续等(不是失败)。""" + hd = rollout_dir() / "handles" / handle + if not hd.exists(): + raise SystemExit(f"handle 不存在:{handle}") + t0 = time.time() + while True: + st = json.loads((hd / "status.json").read_text()) + if st.get("state") == "DONE": + break + if not quiet: + since = st.get("started_at") or st.get("queued_at") or t0 + print(f"[{time.strftime('%H:%M:%S')}] {handle}: {st.get('state')}" + f"{'(等 ' + st.get('pool', '?') + ' 池槽位)' if st.get('state') == 'QUEUED' else ''}" + f" 已 {int(time.time() - since)}s", flush=True) + if max_wait_s and time.time() - t0 > max_wait_s: + print(f"(已等 {int(time.time() - t0)}s 仍未结束;这不是失败,再调一次 bench stage wait {handle} 继续等)") + return 2 + time.sleep(poll) + _print_status(st) + return 0 if st["result"] in ("OK", "SKIPPED") else 1 + + +def status(handle: str) -> int: + hd = rollout_dir() / "handles" / handle + st = json.loads((hd / "status.json").read_text()) + _print_status(st) + return 0 + + +def _print_status(st: dict) -> None: + if st.get("state") != "DONE": + print(json.dumps(st)); return + print(f"result: {st['result']} exit_code: {st['exit_code']} error_class: {st['error_class']} {st['duration_s']}s") + print(f"stdout: {st['stdout']}\nstderr: {st['stderr']}\nmeta: {st['meta']}") + if st.get("measurements"): + print("measurements: " + json.dumps(st["measurements"], ensure_ascii=False)) + if st["result"] != "OK": + try: + tail = Path(st["stderr"]).read_text(errors="replace").splitlines()[-30:] + print("--- stderr tail ---"); print("\n".join(tail)) + except OSError: + pass + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('command', choices=['stage']) + p.add_argument('op', choices=['run', 'wait', 'status']) + p.add_argument('stage') + p.add_argument('--config') + p.add_argument('--poll', type=int, default=60) + a = p.parse_args() + if not re.fullmatch(r'[a-zA-Z0-9_-]+', a.stage): + p.error('invalid handle') + try: + if a.op == 'run': + submit(a.stage, a.config); return 0 + if a.op == 'wait': + return wait(a.stage, max(1, a.poll)) + return status(a.stage) + except (OSError, ValueError) as e: + print(f'handle 不存在或尚未就绪: {a.stage} ({e.strerror if isinstance(e, OSError) else e})', file=sys.stderr) + return 1 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/export/to_agentrl.py b/export/to_agentrl.py index c036d31..480c3bd 100644 --- a/export/to_agentrl.py +++ b/export/to_agentrl.py @@ -18,6 +18,9 @@ from __future__ import annotations import argparse import hashlib import json +import sqlite3 +import os +import re import sys import time from pathlib import Path @@ -223,12 +226,32 @@ def sft_eligibility(verdict: dict, reward: float) -> tuple[bool, str]: return False, f"{t}(未通过必选门,留给 RLVR 当负例)" +def readonly_db(): + state = Path(os.environ.get('BENCH_STATE_DIR', BENCH / 'state')) + return sqlite3.connect(f'file:{state / "jobs.sqlite"}?mode=ro', uri=True) + + def scan(run_ids: list[str]) -> list[Path]: - out = [] - for rid in run_ids: - for v in (BENCH / "runs" / rid).glob("*/rollouts/*/verdict.json"): - out.append(v.parent) - return sorted(out) + # DB index only: never enumerate the JuiceFS run tree. + with readonly_db() as con: + rows = [row for rid in run_ids for row in con.execute( + "SELECT run_id, job_id, kk FROM rollouts WHERE run_id=? AND state NOT IN ('RUNNING','QUEUED','VERIFYING')", (rid,))] + return sorted(rd for rid, jid, kk in rows + if (rd := BENCH / 'runs' / rid / jid / 'rollouts' / f'{kk:02d}').joinpath('verdict.json').exists()) + + +def annotation_reasons(rows, rid, jid, kk): + reasons = set() + for arun, ajob, note in rows: + if arun != rid or ajob not in (None, jid): + continue + match = re.search(r'rollout#(\d+)\b', note or '') + if match and int(match[1]) != kk: + continue + for needle, reason in [('contaminated:', 'contaminated'), ('stalled_killed', 'stalled_killed')]: + if needle in (note or ''): + reasons.add(reason) + return reasons def next_dataset_id() -> str: @@ -249,11 +272,18 @@ def main() -> int: ap.add_argument("--max-tokens", type=int, default=200000) ap.add_argument("--dataset-id") ap.add_argument("--agent-version", help="只收 rollout.json.agent_config_version == 该值的轨迹(旧口径的负样本不混进新数据集)") + ap.add_argument("--min-agent-version", type=int, default=int((BENCH / "agent/VERSION").read_text().strip())) a = ap.parse_args() + if a.agent_version: + print("deprecation: --agent-version 精确匹配;请用 --min-agent-version", file=sys.stderr) + with readonly_db() as con: + annotations = con.execute("SELECT run_id, job_id, note FROM annotations").fetchall() ds_id = a.dataset_id or next_dataset_id() out_dir = BENCH / "datasets" / ds_id - out_dir.mkdir(parents=True, exist_ok=True) + if (out_dir / "VOID").exists(): + print(f"skipped VOID dataset: {ds_id}"); return 0 + out_dir.mkdir(parents=True, exist_ok=False) files = {"teacher": open(out_dir / "teacher.jsonl", "w"), "student": open(out_dir / "student.jsonl", "w")} stats = {"scanned": 0, "written": 0, "dropped": {}, "review": 0, "by_terminal": {}} for rd in scan(a.runs): @@ -263,8 +293,15 @@ def main() -> int: policy = verdict.get("policy") or rj.get("policy") or "teacher" if a.policy and policy != a.policy: continue - if a.agent_version and str(rj.get("agent_config_version", "1")) != str(a.agent_version): - stats["dropped"]["agent_version"] = stats["dropped"].get("agent_version", 0) + 1; continue + reasons = annotation_reasons(annotations, rd.parents[2].name, rd.parents[1].name, int(rd.name)) + version = int(rj.get('agent_config_version') or 1) + if (str(version) != str(a.agent_version)) if a.agent_version else (version < a.min_agent_version): + reasons.add('old_version') + if reasons: + # Per-reason counts may overlap; scanned/written are unique rollouts. + for reason in reasons: + stats['dropped'][reason] = stats['dropped'].get(reason, 0) + 1 + continue if verdict["terminal"] == "TIMEOUT" and not a.include_timeout: stats["dropped"]["timeout"] = stats["dropped"].get("timeout", 0) + 1; continue raw = load_messages(rd) @@ -282,7 +319,7 @@ def main() -> int: if cst["drop"]: stats["dropped"]["r8"] = stats["dropped"].get("r8", 0) + 1; continue msgs, rst = redact_messages(msgs) - rw = compute_reward(verdict, a.max_turns, a.max_tokens) + rw = compute_reward(verdict, rj.get("max_turns", a.max_turns), rj.get("max_tokens", a.max_tokens)) sft_ok, sft_reason = sft_eligibility(verdict, rw["reward"]) if a.purpose == "sft" and not sft_ok: stats["dropped"][f"sft:{sft_reason}"] = stats["dropped"].get(f"sft:{sft_reason}", 0) + 1; continue @@ -319,7 +356,7 @@ def main() -> int: for f in files.values(): f.close() manifest = {"dataset_id": ds_id, "runs": a.runs, "reward_recipe": RECIPE_VERSION, "created_at": time.time(), - "filters": {"policy": a.policy, "include_timeout": a.include_timeout, "purpose": a.purpose, "min_reward": a.min_reward}, "stats": stats} + "filters": {"min_agent_version": a.min_agent_version if not a.agent_version else None, "agent_version": a.agent_version, "agent_version_mode": "exact" if a.agent_version else "minimum", "annotations_read": True, "dropped_counts_overlap": True, "policy": a.policy, "include_timeout": a.include_timeout, "purpose": a.purpose, "min_reward": a.min_reward}, "stats": stats} (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False)) print(json.dumps(manifest, ensure_ascii=False)) return 0 diff --git a/harness/rollout.py b/harness/rollout.py index 3a18ee5..87d6979 100644 --- a/harness/rollout.py +++ b/harness/rollout.py @@ -178,8 +178,7 @@ def jail_prefix(rd: Path, mode: str) -> list[str]: return [] js = paths.BENCH_ROOT / "scripts" / "agent_jail.sh" if not js.exists() or not shutil.which("unshare"): - print("[warn] agent_jail 不可用(缺 unshare 或脚本),本次只有 path-guard 一层", flush=True) - return [] + raise RuntimeError("agent_jail 不可用:拒绝降级隔离") return [str(js), str(rd), mode, "--"] @@ -189,7 +188,7 @@ def pi_command(rd: Path, prompt: str, mode: str = "e2e") -> list[str]: # 实测 pi-coding-agent 0.73.1(pi --help):--mode json 单独就是非交互 JSON 事件流;没有 -a 这个参数; # --append-system-prompt 接受文本或文件路径;--tools 是逗号分隔白名单;extensions/skills 由 workspace/.pi/settings.json 发现。 return jail_prefix(rd, mode) + [ - pi, "--mode", "json", "--session-dir", str(rd / "agent"), + pi, "--mode", "json", "--session-dir", str(rd / "workspace" / ".pi-sessions") if paths.config()["rollouts"].get("jail", True) else str(rd / "agent"), "--append-system-prompt", str(rd / "agent" / "system.md"), # 用 rollout 内的快照,与导出补回的 system 消息逐字一致 "--tools", ",".join(cfg["rollouts"]["tools"]), prompt] @@ -205,8 +204,22 @@ def run_rollout(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_r print(f"[dry-run] rollout 已准备:{rd}\n cwd={rd / 'workspace'}\n cmd={' '.join(cmd[:8])} …") return {"rollout_dir": str(rd), "dry_run": True} cfg = paths.config()["rollouts"] - r = watchdog.run(cmd, cwd=rd / "workspace", stdout=rd / "agent" / "events.jsonl", stderr=rd / "agent" / "pi.stderr.log", - env=agent_env(rd, mode), stall_s=cfg["hard_ceiling_s"], poll_s=30, hard_ceiling_s=cfg["hard_ceiling_s"]) + from contextlib import nullcontext + from .stageworker import Worker, config_bytes + work = paths.expand(paths.config()['paths'].get('work_root')) + if work: + (work / run_id / job_id / f'r{kk:02d}').mkdir(parents=True, exist_ok=True) + jailed = bool(jail_prefix(rd, mode)) + with Worker(rd) if jailed else nullcontext(): + r = watchdog.run(cmd, cwd=rd / "workspace", stdout=rd / "agent" / "events.jsonl", stderr=rd / "agent" / "pi.stderr.log", + env=agent_env(rd, mode), stall_s=cfg["hard_ceiling_s"], poll_s=30, hard_ceiling_s=cfg["hard_ceiling_s"]) + if jailed: + # These remain untrusted claims/logs, never verdict or authoritative budgets. + for src, dst in [('claim.json', 'claim.json'), ('guard.log', 'agent/guard.log')]: + try: + (rd / dst).write_bytes(config_bytes(rd / 'workspace', str(rd / 'workspace' / src))) + except (OSError, ValueError): + pass rj = json.loads((rd / "rollout.json").read_text()) if r.state == "STALLED" or r.exit_code == 75: rj["state"] = "TIMEOUT"; rj.setdefault("timeout_reason", r.stalled_reason or "budget") diff --git a/harness/stagerun.py b/harness/stagerun.py index 9b990aa..0df8d45 100644 --- a/harness/stagerun.py +++ b/harness/stagerun.py @@ -77,10 +77,16 @@ def submit(stage: str, config: str | None, poll_hint: int = 60) -> str: return hid -def _exec(hd: Path) -> int: - rd = hd.parent.parent +def _exec(hd: Path, *, rd=None, publish_status=None, protected=False) -> int: + rd = rd or hd.parent.parent + def put_status(st): + (hd / "status.json").write_text(json.dumps(st, ensure_ascii=False, default=str)) + if publish_status: + publish_status(st) req = json.loads((hd / "request.json").read_text()) ctx = _ctx_from_rollout(rd, Path(req["config"]) if req.get("config") else None) + if protected: + ctx.artifacts = rd / "artifacts" S = BY_NAME[req["stage"]] stage = S() # attempt = 该 rollout 内此阶段已提交次数(agent 自己的重试是同一条轨迹) @@ -101,16 +107,14 @@ def _exec(hd: Path) -> int: except Exception: pass # QUEUED = 已提交、在等池槽位;RUNNING = 真的在跑。两者分开,否则看板上分不出"排队 60 分钟"和"编译 60 分钟"。 - (hd / "status.json").write_text(json.dumps({"state": "QUEUED", "pool": stage.pool, "attempt": ctx.attempt, - "queued_at": time.time()})) + put_status({"state": "QUEUED", "pool": stage.pool, "attempt": ctx.attempt, "queued_at": time.time()}) with db.connect() as con: db.stage_start(con, ctx.run_id, ctx.job["job_id"], stage.name, ctx.attempt, ctx.rollout, lane="queued") with scheduler.acquire(stage.pool, lane="agent") as slot: # agent 优先拿保留槽,不被 oracle 饿死 - (hd / "status.json").write_text(json.dumps({"state": "RUNNING", "pool": stage.pool, "slot": slot, - "attempt": ctx.attempt, "started_at": time.time()})) + put_status({"state": "RUNNING", "pool": stage.pool, "slot": slot, "attempt": ctx.attempt, "started_at": time.time()}) with db.connect() as con: # 拿到槽位才算真的开始跑:看板据此区分"排队"与"在算" - con.execute("UPDATE stages SET lane='agent', started_at=? WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?", - (time.time(), ctx.run_id, ctx.job["job_id"], ctx.rollout, stage.name, ctx.attempt)) + con.execute("UPDATE stages SET lane=?, started_at=? WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?", + ("agent-long" if ctx.lane == "long" else "agent", time.time(), ctx.run_id, ctx.job["job_id"], ctx.rollout, stage.name, ctx.attempt)) ctx.extra["slot"] = slot out = stage.run(ctx) with db.connect() as con: @@ -120,7 +124,7 @@ def _exec(hd: Path) -> int: "duration_s": round(out.duration_s, 1), "attempt": ctx.attempt, "stdout": str(sd / "stdout.log"), "stderr": str(sd / "stderr.log"), "meta": str(sd / "meta.json"), "measurements": out.measurements, "finished_at": time.time()} - (hd / "status.json").write_text(json.dumps(status, indent=2, ensure_ascii=False, default=str)) + put_status(status) return 0 diff --git a/harness/stageworker.py b/harness/stageworker.py new file mode 100644 index 0000000..d968a2d --- /dev/null +++ b/harness/stageworker.py @@ -0,0 +1,120 @@ +"""Per-rollout outer worker. Only enumerates this rollout's handles directory. + +Untrusted tickets are copied into a protected queue. Never execute or write +through a client-supplied path; status publication uses a pinned directory fd. +""" +import json +import os +import re +import stat +import threading +import time +from pathlib import Path +from . import stagerun +from .stages import AGENT_RUNNABLE + + +def read_regular(parent_fd, name): + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent_fd) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode) or os.fstat(fd).st_size > 1024 * 1024: + raise ValueError('ticket/config must be a regular file <= 1 MiB') + return os.read(fd, 1024 * 1024 + 1) + finally: + os.close(fd) + + +def config_bytes(ws, value): + p = Path(value) + rel = p.relative_to(ws) + if not rel.parts or '..' in rel.parts: + raise ValueError('config outside workspace') + fd = os.open(ws, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + for part in rel.parts[:-1]: + nxt = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd); fd = nxt + return read_regular(fd, rel.name) + finally: + os.close(fd) + + +def publish(fd, status): + # Exclusive temporary creation + rename replaces a malicious status symlink. + import uuid + name = '.status-' + uuid.uuid4().hex + f = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=fd) + with os.fdopen(f, 'w') as out: + json.dump(status, out, ensure_ascii=False) + os.rename(name, 'status.json', src_dir_fd=fd, dst_dir_fd=fd) + + +class Worker: + def __init__(self, rd, *, test_noop=False): + self.rd = Path(rd).resolve() + self.test_noop = test_noop and 'guard-smoke' in self.rd.parts + self.stop_event = threading.Event() + self.thread = threading.Thread(target=self.run, name='stage-ticket-worker') + self.seen = set() + + def __enter__(self): + self.thread.start(); return self + + def __exit__(self, *exc): + self.stop_event.set() + self.thread.join() # Never kill an in-flight stage; drain before verification. + + def run(self): + while True: + self.scan() + if self.stop_event.wait(0.5): + self.scan() + return + + def scan(self): + for hd in sorted((self.rd / 'handles').iterdir()): + if hd.name in self.seen or not re.fullmatch(r'[a-zA-Z0-9_-]+', hd.name): + continue + try: + fd = os.open(hd, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except OSError: + continue + try: + try: + req = json.loads(read_regular(fd, 'request.json')) + except FileNotFoundError: + continue + self.seen.add(hd.name) + self.execute(hd.name, fd, req) + except Exception as e: + self.seen.add(hd.name) + publish(fd, {'state': 'DONE', 'result': 'FAIL', 'exit_code': 2, + 'error_class': 'invalid_request', 'duration_s': 0, + 'stdout': '', 'stderr': '', 'meta': '', 'detail': str(e)}) + finally: + os.close(fd) + + def execute(self, hid, fd, req): + name = req.get('stage') + if name not in AGENT_RUNNABLE and not (name == 'noop' and self.test_noop): + raise ValueError('stage not allowed') + queue = self.rd / 'stages' / '_queue' / hid + queue.mkdir(parents=True, exist_ok=False) + config = None + if req.get('config'): + config = queue / 'config.yaml' + config.write_bytes(config_bytes(self.rd / 'workspace', req['config'])) + trusted = {'stage': name, 'config': str(config) if config else None} + (queue / 'request.json').write_text(json.dumps(trusted)) + # _exec publishes QUEUED/RUNNING/DONE synchronously through this callback. + with (queue / 'runner.log').open('a') as log: + log.write(f'outer worker pid={os.getpid()} stage={name}\n'); log.flush() + if name == 'noop': + publish(fd, {'state': 'RUNNING', 'started_at': time.time()}) + result = {'state': 'DONE', 'result': 'OK', 'exit_code': 0, 'error_class': None, + 'duration_s': 0, 'stdout': str(queue / 'runner.log'), 'stderr': '', 'meta': '', + 'worker_pid': os.getpid()} + publish(fd, result) + (queue / 'status.json').write_text(json.dumps(result)) + else: + stagerun._exec(queue, rd=self.rd, publish_status=lambda st: publish(fd, st), protected=True) diff --git a/scripts/agent_jail.sh b/scripts/agent_jail.sh index 847d1fd..ace2bdb 100755 --- a/scripts/agent_jail.sh +++ b/scripts/agent_jail.sh @@ -1,72 +1,7 @@ #!/usr/bin/env bash -# agent_jail.sh —— 用 mount namespace 把 agent 够不着的东西从文件系统里摘掉,再 exec 真正的命令。 -# -# 为什么要有这个:path-guard 是解析 shell 字符串的白名单,v1→v4 每一轮都还能找到新绕法 -# (换行分隔的 cd、$IFS、$(…)、$VAR/ 拼接、workspace 里造软链…)。解析永远追不平所有变体。 -# 这里换一层:进程根本看不到 oracle/、别的 rollout,绕法再多也没用。guard 退化成"给句人话的理由"。 -# -# 摘掉什么(e2e): -# /oracle → 空目录 -# /rollouts/*(除自己) → 整个 rollouts/ 换成只含 的空壳,再把自己 bind 回去 -# (逐个遮兄弟不行:并行采集时兄弟目录是启动之后才出现的,遮不到) -# work_root///*(除 r) → 同上(oracle 与兄弟的编译中间产物、compile.yaml 都在那) -# bench/.env.local → /dev/null -# 不摘 ~/.pi:pi 进程自己要读 models.json 里的 LLM key(PI_CODING_AGENT_DIR 换地方也一样要可读), -# 同一个进程里跑的 bash 工具就一定读得到——这是结构性限制,只能靠 path-guard 拒 token + 定期轮换 key, -# 别假装 jail 能藏住它。 -# mode=repair 时保留 oracle/(repair 任务的输入就是 oracle 的失败现场)。 -# -# 非 root 可用(user + mount namespace,unshare -rm)。进程在 ns 内 uid 显示为 0,落盘文件仍归真实用户; -# 实测 ns 内 ssh -i 指定密钥可连板卡(07-board 走 harness 子进程,密钥路径来自 boards.yaml,不靠 getpwuid)。 -# -# 用法(rollout.py 里把 pi 的启动命令包一层): -# scripts/agent_jail.sh -- [args...] -# 自检:scripts/agent_jail.sh e2e -- ls /oracle /rollouts → 应分别为空 / 只有 +# Fail closed. PID namespace also removes /proc//root escape routes. set -euo pipefail - RD=$(readlink -f "$1"); MODE="${2:-e2e}"; shift 2 -[ "${1:-}" = "--" ] && shift - -JOB=$(readlink -f "$RD/../..") -KK=$(basename "$RD") -BENCH=$(readlink -f "$(dirname "$0")/..") -RUN=$(basename "$(dirname "$JOB")") -JOBID=$(basename "$JOB") - -# 每个 rollout 自己的空壳目录。必须在 job 树之外(本地盘 /tmp):遮掉 rollouts/ 之后 /… 路径就指向空壳了, -# 放在 rollout 里的话第 3 步 bind 回自己时找不到源。 -SHELL_DIR="${TMPDIR:-/tmp}/bench-jail/$RUN--$JOBID--$KK" -rm -rf "$SHELL_DIR" -mkdir -p "$SHELL_DIR/empty" "$SHELL_DIR/rollouts/$KK" "$SHELL_DIR/work/r$KK" "$SHELL_DIR/self" "$SHELL_DIR/selfwork" - -WORK_ROOT=$(python3 - "$BENCH/config.yaml" <<'EOF' -import sys, yaml, os -p = (yaml.safe_load(open(sys.argv[1])).get("paths") or {}).get("work_root") or "" -print(os.path.expanduser(p)) -EOF -) -WORK_JOB="" -[ -n "$WORK_ROOT" ] && WORK_JOB="$WORK_ROOT/$RUN/$JOBID" -[ -n "$WORK_JOB" ] && mkdir -p "$WORK_JOB/r$KK" # 自己的编译工作目录先建好,才能 bind 回去 - -ORACLE="" -[ "$MODE" != "repair" ] && [ -d "$JOB/oracle" ] && ORACLE="$JOB/oracle" -ENVLOCAL="" -[ -f "$BENCH/.env.local" ] && ENVLOCAL="$BENCH/.env.local" - -export RD KK JOB ORACLE ENVLOCAL SHELL_DIR WORK_JOB -exec unshare -rm bash -c ' - set -e - # 1) 先把"自己"从真实路径 bind 到别处备用(后面遮 rollouts/ 之后原路径就指向空壳了) - mount --bind "$RD" "$SHELL_DIR/self" - if [ -n "$WORK_JOB" ]; then mount --bind "$WORK_JOB/r$KK" "$SHELL_DIR/selfwork"; fi - # 2) 遮:oracle → 空;rollouts/ → 只含 的空壳;work_root// → 只含 r 的空壳;.env.local → /dev/null - [ -n "$ORACLE" ] && mount --bind "$SHELL_DIR/empty" "$ORACLE" - mount --bind "$SHELL_DIR/rollouts" "$JOB/rollouts" - if [ -n "$WORK_JOB" ]; then mount --bind "$SHELL_DIR/work" "$WORK_JOB"; fi - [ -n "$ENVLOCAL" ] && mount --bind /dev/null "$ENVLOCAL" - # 3) 把自己 bind 回原位 - mount --bind "$SHELL_DIR/self" "$JOB/rollouts/$KK" - if [ -n "$WORK_JOB" ]; then mount --bind "$SHELL_DIR/selfwork" "$WORK_JOB/r$KK"; fi - exec "$@" -' _ "$@" +[ "${1:-}" = -- ] && shift +BENCH=$(cd "$(dirname "$0")/.." && pwd) +exec unshare -Urmpf --mount-proc python3 "$BENCH/scripts/jail_setup.py" "$BENCH" "$RD" "$MODE" -- "$@" diff --git a/scripts/audit_rollouts.py b/scripts/audit_rollouts.py index 3f078aa..980ea36 100644 --- a/scripts/audit_rollouts.py +++ b/scripts/audit_rollouts.py @@ -44,14 +44,48 @@ def _text(x) -> str: return "" +DECIMAL = re.compile(r'(? dict: ev = rd / "agent" / "events.jsonl" out = {"turns": 0, "tool_calls": 0, "oracle_read": 0, "sibling_read": 0, "harness_read": 0, "toolchain_read": 0, - "blocked_attempts": 0, "claim_calls": 0, "has_agent_end": False} + "oracle_content_match": 0, "state_access": 0, "other_job_read": 0, "blocked_attempts": 0, "claim_calls": 0, "has_agent_end": False} if not ev.exists(): out["no_events"] = True return out sib_rx = re.compile(r"/rollouts/(?!%02d(/|$))\d\d(/|$)" % kk) + needles = oracle_needles(rd) + run_name = rd.parents[2].name + job_name = rd.parents[1].name + other_rx = re.compile(r'runs/' + re.escape(run_name) + r'/([^/\s\x22\x27]+)') pending: dict[str, str] = {} # toolCallId → 目标字符串 for line in ev.open(errors="replace"): try: @@ -77,6 +111,14 @@ def scan_events(rd: Path, kk: int) -> dict: if blocked: out["blocked_attempts"] += 1 continue + # Only actual tool results, excluding guard denials / failed filesystem probes. + full_result = json.dumps(e.get('result'), ensure_ascii=False) + if any(n in full_result for n in needles): + out['oracle_content_match'] += 1 + if STATE_RX.search(target): + out['state_access'] += 1 + if any(m[1] != job_name for m in other_rx.finditer(target)): + out['other_job_read'] += 1 if ORACLE_RX.search(target): out["oracle_read"] += 1 if sib_rx.search(target): @@ -104,6 +146,9 @@ def audit_one(rd: Path, kk: int, min_version: int, mode: str) -> dict: reasons = [] if mode != "repair" and s["oracle_read"]: reasons.append(f"oracle_read×{s['oracle_read']}") + for reason in ('oracle_content_match', 'state_access', 'other_job_read'): + if s[reason] and (reason != 'oracle_content_match' or mode != 'repair'): + reasons.append(f'{reason}×{s[reason]}') if s["sibling_read"]: reasons.append(f"sibling_read×{s['sibling_read']}") if s["harness_read"] or s["toolchain_read"]: @@ -120,7 +165,7 @@ def audit_one(rd: Path, kk: int, min_version: int, mode: str) -> dict: reasons.append("claim_absent") elif cc is False: reasons.append("claim_mismatch") - contaminated = bool((mode != "repair" and s["oracle_read"]) or s["sibling_read"]) + contaminated = bool((mode != "repair" and s["oracle_read"]) or s["sibling_read"] or s["state_access"] or s["other_job_read"] or (mode != "repair" and s["oracle_content_match"])) running = rj.get("state") in ("RUNNING", "VERIFYING") or not s["has_agent_end"] and rj.get("state") == "RUNNING" rl_ok = (not contaminated) and (not stalled) and ver >= min_version and not running sft_ok = rl_ok and terminal == "PASS" and cc is True @@ -141,7 +186,8 @@ def main() -> int: vp = paths.AGENT / "VERSION" min_version = a.min_version or (int(vp.read_text().strip()) if vp.exists() else 1) - with db.connect() as con: + import sqlite3 + with sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True) as con: rows = con.execute("SELECT job_id, kk, mode, state FROM rollouts WHERE run_id=? AND state!='QUEUED' ORDER BY job_id, kk", (a.run,)).fetchall() existing = {r[0] for r in con.execute("SELECT note FROM annotations WHERE run_id=?", (a.run,)).fetchall()} @@ -168,7 +214,7 @@ def main() -> int: n_rl = sum(r["rl_ok"] for r in results); n_sft = sum(r["sft_ok"] for r in results) n_cont = sum(r["contaminated"] for r in results); n_stall = sum(r["stalled_killed"] for r in results) print(f"\n可进 RL:{n_rl} 可进 SFT:{n_sft} 污染(读了 oracle/兄弟):{n_cont} 被看门狗误杀:{n_stall}") - print("导出时默认只收 agent_config_version ≥ 当前 VERSION 的:export/to_agentrl.py --agent-version", min_version) + print("导出时默认只收 agent_config_version ≥ 当前 VERSION 的:export/to_agentrl.py --min-agent-version", min_version) if a.annotate: n = 0 @@ -176,7 +222,7 @@ def main() -> int: for r in results: tags = [] if r["contaminated"]: - tags.append("contaminated:" + "+".join(x.split("×")[0] for x in r["reasons"] if x.startswith(("oracle_read", "sibling_read")))) + tags.append("contaminated:" + "+".join(x.split("×")[0] for x in r["reasons"] if x.startswith(("oracle_read", "sibling_read", "oracle_content_match", "state_access", "other_job_read")))) if r["stalled_killed"]: tags.append("stalled_killed") for tag in tags: diff --git a/scripts/guard_smoke.sh b/scripts/guard_smoke.sh index f563ad0..ab57e56 100755 --- a/scripts/guard_smoke.sh +++ b/scripts/guard_smoke.sh @@ -14,7 +14,7 @@ MODEL="$(python3 -c "import yaml;print(yaml.safe_load(open('config.yaml'))['endp TS="$(date +%Y%m%d-%H%M%S)" JOB="$BENCH_ROOT/state/guard-smoke/$TS/runs/rS/cls--smoke--nash-e--ptq--int8" RO="$JOB/rollouts/01"; WS="$RO/workspace" -mkdir -p "$WS/.pi" "$RO/agent" "$RO/stages" "$JOB/oracle/stages/04-calibrate" "$JOB/rollouts/00/workspace" "$JOB/data" +mkdir -p "$WS/.pi" "$RO/agent" "$RO/stages" "$RO/handles" "$JOB/oracle/stages/04-calibrate" "$JOB/rollouts/00/workspace" "$JOB/data" echo '{"job_id":"cls--smoke","task":"cls","march":"nash-e"}' > "$JOB/job.json" echo '{"verdict":"PASS","secret":"ORACLE-SHOULD-NOT-LEAK"}' > "$JOB/oracle/result.json" echo 'calibration_type: kl # SIBLING-SHOULD-NOT-LEAK' > "$JOB/rollouts/00/workspace/compile.yaml" @@ -57,11 +57,11 @@ export HOME PATH="$BENCH_ROOT/bin:$(dirname "$NODE_BIN"):$PATH" LANG=C.UTF-8 TER export BENCH_ROOT BENCH_ROLLOUT_DIR="$RO" BENCH_MODE=e2e BENCH_MAX_TURNS=40 BENCH_MAX_TOKENS=400000 PYTHONPATH="$BENCH_ROOT" unset BENCH_GUARD_DENY BENCH_GUARD_ALLOW # 故意不传:验证 path-guard 自己从 config.yaml / state 推出 deny/allow(guard.log 第一行 init) echo "smoke dir: $RO" -( cd "$WS" && "$PI_BIN" --mode json --session-dir "$RO/agent" --append-system-prompt "$RO/agent/system.md" \ +( cd "$WS" && "$BENCH_ROOT/scripts/agent_jail.sh" "$RO" e2e -- "$PI_BIN" --mode json --session-dir "$WS/.pi-sessions" --append-system-prompt "$RO/agent/system.md" \ --tools read,edit,write,bash,grep,claim "$PROMPT" > "$RO/agent/events.jsonl" 2> "$RO/agent/pi.stderr.log" ) || echo "pi exit=$?" echo "---- pi.stderr (扩展加载失败会在这里)"; head -20 "$RO/agent/pi.stderr.log" -echo "---- guard.log(每次拒绝一行)"; cat "$RO/agent/guard.log" 2>/dev/null || echo "(无 guard.log —— 扩展没生效!)" +echo "---- guard.log(每次拒绝一行)"; cat "$WS/guard.log" 2>/dev/null || echo "(无 guard.log —— 扩展没生效!)" echo "---- 泄漏检查(下面两个词不应出现在任何 tool result 里)" python3 - "$RO/agent/events.jsonl" <<'EOF' import json, sys @@ -81,4 +81,5 @@ print(f"tool calls={calls} blocked by path-guard={blocked} leaks={leak}") print("RESULT:", "OK" if blocked >= 10 and not any(leak.values()) else "CHECK MANUALLY") EOF echo "---- 文件侧:escape.txt 不应存在,note.txt/done.txt 应存在" -ls "$JOB/rollouts/escape.txt" 2>&1 | head -1; ls "$WS" +test ! -e "$JOB/rollouts/escape.txt" +ls "$WS" diff --git a/scripts/jail_probe.sh b/scripts/jail_probe.sh new file mode 100755 index 0000000..745f0ae --- /dev/null +++ b/scripts/jail_probe.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +exec "$ROOT/scripts/agent_jail.sh" "$1" "${2:-e2e}" -- python3 - <<'PY' +import os, subprocess +from pathlib import Path +rd=Path(os.environ['BENCH_ROLLOUT_DIR']); bench=Path(os.environ['BENCH_ROOT']) +jd=rd.parent.parent +fails=[] +def check(label, ok): + print(('PASS ' if ok else 'FAIL ')+label, flush=True) + if not ok: fails.append(label) +def unreadable(p): + try: p.read_bytes(); return False + except OSError: return True +def unwritable(p): + # Never truncate a real existing file, even on a broken jail. + try: + fd=os.open(p, os.O_WRONLY | os.O_APPEND); os.close(fd); return False + except OSError: return True +other=jd.parent/'cls--resnet18--nash-m--ptq--int8'/'oracle'/'result.json' +check('other job oracle hidden', unreadable(other)) +check('state sqlite hidden', unreadable(bench/'state/jobs.sqlite')) +check('real state sqlite hidden', unreadable(Path.home()/'bench-state/jobs.sqlite')) +for p in [bench/'registry/gates.yaml', bench/'harness/gates.py', other, bench/'state/x', Path.home()/'bench-state/x',rd/'rollout.json']: + check('write denied '+str(p), unwritable(p)) +# Only create a NEW workspace artifact. +p=rd/'workspace'/('probe-'+str(os.getpid())) +try: p.write_text('probe'); check('workspace writable', True) +except OSError: check('workspace writable', False) +r=subprocess.run([str(bench/'bin/bench'),'stage','status','nonexistent'],capture_output=True,text=True) +check('stdlib stage status executes',r.returncode==1 and 'handle 不存在' in r.stderr and 'harness' not in r.stderr) +r=subprocess.run(['mount','-o','remount,bind,rw',str(rd)],capture_output=True) +check('cannot undo read-only mounts',r.returncode!=0) +check('host processes hidden',len(list(Path('/proc').glob('[0-9]*')))<15) +raise SystemExit(bool(fails)) +PY diff --git a/scripts/jail_setup.py b/scripts/jail_setup.py new file mode 100644 index 0000000..75f8461 --- /dev/null +++ b/scripts/jail_setup.py @@ -0,0 +1,90 @@ +"""Mount construction runs before untrusted code, then drops every capability.""" +import os +import sys +import subprocess +import tempfile +from pathlib import Path +import yaml + + +def mount(*args): + subprocess.run(['mount', *map(str, args)], check=True) + + +def main(): + bench, rd = map(Path, sys.argv[1:3]) + mode = sys.argv[3] + command = sys.argv[5:] + cfg = yaml.safe_load((bench / 'config.yaml').read_text()) + jd = rd.parent.parent + run = jd.parent + work = Path(os.path.expanduser(cfg['paths']['work_root'])) if cfg['paths'].get('work_root') else None + stash = Path(tempfile.mkdtemp(prefix='bench-jail-')) + saved = [] + + def save(src, dest=None, writable=False): + src = Path(src).resolve() + if not src.exists(): + return + target = stash / str(len(saved)) + if src.is_dir(): + target.mkdir() + else: + target.touch() + mount('--bind', src, target) + saved.append((target, Path(dest or src), writable, src.is_dir())) + + # Preserve only named inputs; never enumerate runs or work_root. + for name in ('bin', 'agent', 'data', 'cache'): + save(bench / name, bench / name) + real = (bench / name).resolve() + if real != bench / name: + save(real) + save(bench.parent / '.horizon') + save(jd / 'job.json') + save(jd / 'data') + if mode == 'repair': + save(jd / 'oracle') + save(rd) + save(rd / 'workspace', writable=True) + save(rd / 'handles', writable=True) + if work: + own = work / run.name / jd.name / ('r' + rd.name) + # Caller prepares production work directories; missing probe work stays empty. + save(own) + + hidden = [bench, (bench / 'state').resolve()] + if work: + hidden.append(work.resolve()) + # Save paths can live under state in the isolated smoke fixture. + for p in dict.fromkeys(hidden): + if p.is_dir(): + mount('-t', 'tmpfs', '-o', 'mode=755', 'tmpfs', p) + for src, dst, writable, directory in saved: + dst.parent.mkdir(parents=True, exist_ok=True) + if directory: + dst.mkdir(exist_ok=True) + elif not dst.exists(): + dst.touch() + mount('--bind', src, dst) + # Read-only ALL visible mounts (including aliases and mount construction stash). + mounts = [] + for line in Path('/proc/self/mountinfo').read_text().splitlines(): + p = line.split()[4].replace('\\040', ' ') + if p not in ('/proc', '/dev') and not p.startswith(('/proc/', '/dev/')): + mounts.append(p) + for p in sorted(set(mounts), key=len, reverse=True): + mount('-o', 'remount,bind,ro', p) + for src, dst, writable, directory in saved: + if writable: + mount('-o', 'remount,bind,rw', dst) + # Hide temporary bind aliases and host /tmp; the agent gets its own scratch. + mount('-t', 'tmpfs', '-o', 'mode=1777', 'tmpfs', '/tmp') + os.environ.update(BENCH_JAILED='1', BENCH_ROLLOUT_DIR=str(rd), BENCH_ROOT=str(bench)) + os.chdir(rd / 'workspace') + os.execvp('setpriv', ['setpriv', '--bounding-set=-all', '--inh-caps=-all', + '--ambient-caps=-all', '--no-new-privs', '--', *command]) + + +if __name__ == '__main__': + main() diff --git a/scripts/setup_agent.sh b/scripts/setup_agent.sh old mode 100644 new mode 100755 diff --git a/scripts/sync.sh b/scripts/sync.sh index 137171d..8f61160 100755 --- a/scripts/sync.sh +++ b/scripts/sync.sh @@ -45,8 +45,9 @@ if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then fi find harness agent export scripts -name __pycache__ -type d -prune -exec rm -rf {} + 2>/dev/null || true # 注意:rsync 多个源到一个目录会把 registry/x.yaml 摊平到 $RDIR/x.yaml(踩过),所以整目录同步 + 排除 models.csv -rsync -az --delete-excluded --exclude '__pycache__' --exclude '.sync' --exclude '.git' --exclude 'registry/models.csv' --exclude 'registry/models_源清单.xlsx' \ - harness agent export scripts bin registry config.yaml CONTRACT.md README.md requirements.txt "$REMOTE:$RDIR/" 2>&1 | tail -1 +# 踩过两次:--delete-excluded 会把"被排除的文件"从远端删掉 → registry/models.csv 两次消失。必须用 --delete(排除项在远端受保护)。 +rsync -az --delete --exclude '__pycache__' --exclude '.sync' --exclude '.git' --exclude 'registry/models.csv' --exclude 'registry/models_源清单.xlsx' \ + harness agent export scripts bin registry config.yaml CONTRACT.md HANDBOOK.md REPORT-codex.md README.md requirements.txt "$REMOTE:$RDIR/" 2>&1 | tail -1 # registry/models.csv 是机器上生成的(excel_to_models),只在 --pull 方向走 # 只在代码目录里清 __pycache__:find 整棵树会扫进 runs/(JuiceFS 上 190 个 job 目录,几分钟) ssh -o BatchMode=yes "$REMOTE" "cd $RDIR && chmod +x bin/bench scripts/*.sh agent/*.sh 2>/dev/null; find harness agent export scripts -name __pycache__ -type d -prune -exec rm -rf {} + 2>/dev/null; true"