The agent could read and copy harness/adapters (model loading, preprocessing, export, decoding), and every bench dev container carried the whole harness, registry, scripts and export trees, evaluator and gates included. That did the model-specific half of the deployment for it. - guard and private-home jail no longer expose harness/adapters; bench dev packs only container_site - the evaluator still decodes outputs with the platform's own code, so agent/io-contracts states what the graph's inputs and outputs mean (9 families, 71 models so far); the prompt appends the task's contract - SYSTEM.md: says where official sources, weights and environments are; the compile.yaml template is replaced by the facts that differ from the docs - e2e prompt: whole-class int16/float fallback and >1.5x board latency do not pass - agent/VERSION 24 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
158 lines
11 KiB
JavaScript
158 lines
11 KiB
JavaScript
// node agent/extensions/guard-core.test.js —— path-guard 判定核心的单测(含另一个会话报的 5 个绕过 + 兄弟 rollout 泄漏)
|
||
"use strict";
|
||
const { makeGuard } = require("./guard-core.js");
|
||
|
||
const P = "/mnt/data/u/proj";
|
||
const B = P + "/bench";
|
||
const JOB = B + "/runs/r1/cls--m--nash-e--ptq--int8";
|
||
const RO = JOB + "/rollouts/01";
|
||
const WS = RO + "/workspace";
|
||
const base = { WORKSPACE: WS, ROLLOUT: RO, JOB_DIR: JOB, BENCH_ROOT: B, PROJECT_ROOT: P, HOME: "/home/u",
|
||
DENY_EXTRA: ["/home/u/bench-state", "/home/u/bench-work"],
|
||
ALLOW_EXTRA: ["/home/u/bench-work/r1/cls--m--nash-e--ptq--int8/r01"] };
|
||
|
||
const e2e = makeGuard({ ...base, MODE: "e2e" });
|
||
const repair = makeGuard({ ...base, MODE: "repair" });
|
||
|
||
let fails = 0;
|
||
function expect(name, got, want) {
|
||
const ok = (got === null) === (want === "allow");
|
||
if (!ok) { fails++; console.log(`FAIL ${name}: want ${want}, got ${got ? "block(" + got.reason + ")" : "allow"}`); }
|
||
else console.log(`ok ${name} → ${want}`);
|
||
}
|
||
|
||
// ---- read tool paths
|
||
expect("read job.json", e2e.judgePath("../../../job.json"), "allow");
|
||
expect("read own agent/prompt", e2e.judgePath("../agent/prompt.md"), "allow");
|
||
expect("read own stages log", e2e.judgePath(RO + "/stages/05-compile/a00/stderr.log"), "allow");
|
||
expect("read data", e2e.judgePath("../../../data/calib/cls/1x3x224x224/manifest.json"), "allow");
|
||
expect("read cache onnx", e2e.judgePath(B + "/cache/onnx/abc/m.onnx"), "allow");
|
||
expect("read horizon skill", e2e.judgePath(P + "/.horizon/skills/hmct/SKILL.md"), "allow");
|
||
expect("read oracle (e2e)", e2e.judgePath("../../../oracle/result.json"), "block");
|
||
expect("read oracle (repair)", repair.judgePath("../../../oracle/result.json"), "allow");
|
||
expect("read sibling claim", e2e.judgePath("../../00/claim.json"), "block");
|
||
expect("read sibling via rollouts/", e2e.judgePath("../../00/workspace/compile.yaml"), "block");
|
||
expect("read platform adapter", e2e.judgePath(B + "/harness/adapters/segnext_tiny.py"), "block");
|
||
expect("copy platform adapter", e2e.judgeCommand("cp " + B + "/harness/adapters/segnext_tiny.py ./model.py"), "block");
|
||
expect("write public adapter", e2e.judgePath(B + "/harness/adapters/segnext_tiny.py", { write: true }), "block");
|
||
expect("read harness", e2e.judgePath(B + "/harness/gates.py"), "block");
|
||
expect("read state sqlite", e2e.judgePath(B + "/state/jobs.sqlite"), "block");
|
||
expect("read ~/.pi", e2e.judgePath("~/.pi/agent/models.json"), "block");
|
||
expect("read toolchain src", e2e.judgePath("/home/u/toolchains/oe-3.7.0/venv/lib/python3.10/site-packages/hmct/x.py"), "block");
|
||
expect("read /usr/lib", e2e.judgePath("/usr/lib/python3.10/json/__init__.py"), "allow");
|
||
expect("read job dir itself", e2e.judgePath("../../.."), "block");
|
||
expect("read env.local", e2e.judgePath(B + "/.env.local"), "block");
|
||
expect("read state realpath", e2e.judgePath("/home/u/bench-state/jobs.sqlite"), "block");
|
||
expect("read own work dir", e2e.judgePath("/home/u/bench-work/r1/cls--m--nash-e--ptq--int8/r01/05-compile-work/x.bc"), "allow");
|
||
expect("read sibling work dir", e2e.judgePath("/home/u/bench-work/r1/cls--m--nash-e--ptq--int8/r00/05-compile-work/x.bc"), "block");
|
||
expect("read work_root", e2e.judgePath("/home/u/bench-work/r1/job/oracle/04-calibrate/x.yaml"), "block");
|
||
expect("read data realpath($HOME 默认拒:合成路径没有真实软链,落到 HOME 兜底 → block;真机上 bench/data 的 realpath 已进白名单)", e2e.judgePath("/home/u/bench-data/calib/x.npy"), "block");
|
||
|
||
// ---- write tool
|
||
expect("write ws", e2e.judgePath("compile.yaml", { write: true }), "allow");
|
||
expect("write ws abs", e2e.judgePath(WS + "/x/y.yaml", { write: true }), "allow");
|
||
expect("write agent dir", e2e.judgePath("../agent/x", { write: true }), "block");
|
||
expect("write job.json", e2e.judgePath("../../../job.json", { write: true }), "block");
|
||
|
||
// ---- bash(另一个会话实测的 5 个绕过)
|
||
expect("cat ../../oracle", e2e.judgeCommand("cat ../../../oracle/result.json"), "block");
|
||
expect("python open oracle", e2e.judgeCommand("python3 -c 'print(open(\"../../../oracle/result.json\").read())'"), "block");
|
||
expect("cd && cat oracle", e2e.judgeCommand("cd ../../.. && cat oracle/result.json"), "block");
|
||
expect("cd;cd;head oracle", e2e.judgeCommand("cd .. ; cd .. ; cd .. ; head oracle/stages/04-calibrate/compile.yaml"), "block");
|
||
expect("ls ../../../ora*", e2e.judgeCommand("ls ../../../ora*"), "block");
|
||
expect("find .. -exec cat (own rollout only)", e2e.judgeCommand("find .. -name result.json -exec cat {} +"), "allow");
|
||
expect("find ../.. -exec cat", e2e.judgeCommand("find ../../.. -name result.json -exec cat {} +"), "block");
|
||
expect("cat sibling claim", e2e.judgeCommand("cat ../../02/claim.json"), "block");
|
||
expect("ls sibling ws", e2e.judgeCommand("ls ../../00/workspace"), "block");
|
||
expect("cd only", e2e.judgeCommand("cd .. && ls"), "block");
|
||
expect("pushd", e2e.judgeCommand("pushd ../.. && ls"), "block");
|
||
expect("cd in subshell", e2e.judgeCommand("(cd ../..; ls)"), "block");
|
||
expect("cd to own workspace (abs)", e2e.judgeCommand("cd " + WS + " && bench stage run compile --config compile.yaml 2>&1 | tail -3"), "allow");
|
||
expect("cd to own workspace (trailing slash)", e2e.judgeCommand("cd " + WS + "/ && ls"), "allow");
|
||
expect("cd .", e2e.judgeCommand("cd . && ls"), "allow");
|
||
expect("cd to ws subdir", e2e.judgeCommand("cd artifacts && cat ../../../../oracle/result.json"), "block");
|
||
expect("cd ws then cd ..", e2e.judgeCommand("cd " + WS + " && cd .. && ls"), "block");
|
||
expect("cd ws ; cat oracle", e2e.judgeCommand("cd " + WS + "; cat ../../../oracle/result.json"), "block");
|
||
expect("word containing cd", e2e.judgeCommand("echo abcd/x.txt > note.txt"), "allow");
|
||
|
||
// ---- bash 合法用法
|
||
expect("bench stage run", e2e.judgeCommand("bench stage run compile --config workspace/compile.yaml"), "allow");
|
||
expect("bench stage run abs", e2e.judgeCommand("bench stage run compile --config " + WS + "/compile.yaml"), "allow");
|
||
expect("cat job.json", e2e.judgeCommand("cat ../../../job.json | head -50"), "allow");
|
||
expect("tail own stage log", e2e.judgeCommand("tail -50 ../stages/05-compile/a00/stderr.log 2>/dev/null"), "allow");
|
||
expect("ls skills", e2e.judgeCommand("ls " + P + "/.horizon/skills/"), "allow");
|
||
expect("python one-liner no path", e2e.judgeCommand("python3 -c 'import yaml; print(yaml.safe_load(open(\"compile.yaml\")))'"), "allow");
|
||
expect("redirect to /dev/null", e2e.judgeCommand("bench stage wait h --poll 60 2>/dev/null"), "allow");
|
||
expect("version-like token", e2e.judgeCommand("echo 3.5.3/hbdk 4.7.5"), "allow");
|
||
expect("url", e2e.judgeCommand("echo https://x.y/z"), "allow");
|
||
expect("read own rollout.json", e2e.judgeCommand("cat ../rollout.json"), "allow");
|
||
expect("ls ..(own rollout)", e2e.judgeCommand("ls .."), "allow");
|
||
expect("ls ../.. (rollouts dir)", e2e.judgeCommand("ls ../.."), "block");
|
||
expect("read speed reference", e2e.judgeCommand("cat " + B + "/agent/speed_ref/cls--m--nash-e--ptq--int8.json"), "allow");
|
||
expect("other agent files stay closed", e2e.judgeCommand("cat " + B + "/agent/SYSTEM.md"), "block");
|
||
expect("repair reads oracle yaml", repair.judgeCommand("cp ../../../oracle/stages/04-calibrate/compile.yaml ./compile.yaml"), "allow");
|
||
|
||
async function extensionCases() {
|
||
// Use the installed Pi TypeScript loader when provided; no test transpiler.
|
||
if (!process.env.BENCH_TEST_JITI) return;
|
||
const fs = require("node:fs"), os = require("node:os"), path = require("node:path");
|
||
const { createJiti } = require(process.env.BENCH_TEST_JITI);
|
||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "bench-guard-test-"));
|
||
const previous = { ...process.env };
|
||
try {
|
||
const bench = path.join(root, "bench"), rollout = path.join(bench, "runs/r/j/rollouts/01");
|
||
fs.mkdirSync(path.join(rollout, "workspace"), { recursive: true });
|
||
Object.assign(process.env, { BENCH_ROOT: bench, BENCH_ROLLOUT_DIR: rollout,
|
||
HOME: path.join(root, "host-home"), BENCH_GUARD_DENY: "", BENCH_GUARD_ALLOW: "" });
|
||
for (const policy of ["legacy-pi", "private-home"]) {
|
||
Object.assign(process.env, { BENCH_JAILED: "1", BENCH_JAIL_POLICY: policy });
|
||
const jiti = createJiti(__filename, { moduleCache: false, tryNative: false });
|
||
const extension = await jiti.import(path.join(__dirname, "path-guard.ts"), { default: true });
|
||
let handler;
|
||
extension({ on: (name, callback) => { if (name === "tool_call") handler = callback; } });
|
||
for (const command of ['mkdir -p src && cd src && echo ok', 'VENV=.venv; "$VENV/bin/python" --version',
|
||
'pip install --target .venv/lib package.whl']) {
|
||
expect(policy + " " + command, (await handler({ toolName: "bash", input: { command } })) || null,
|
||
policy === "private-home" ? "allow" : "block");
|
||
}
|
||
for (const command of ["grep -E 'error| hb_compile' ../stages/stderr.log", "rg 'ssh; hb_compile' ."]) {
|
||
expect(policy + " quoted search " + command, (await handler({toolName: "bash", input: {command}})) || null, "allow");
|
||
}
|
||
for (const command of ["find / -maxdepth 6 -name harness", "find '/' -name harness", "find -L / -name harness", "true && /usr/bin/find / -name harness"]) {
|
||
expect(policy + " root scan " + command, (await handler({toolName: "bash", input: {command}})) || null, "block");
|
||
}
|
||
for (const command of ["rg 'find /' .", "find . -name harness", "find . -name '/'", "find ./src -name '*.py'"]) {
|
||
expect(policy + " scoped search " + command, (await handler({toolName: "bash", input: {command}})) || null, "allow");
|
||
}
|
||
expect(policy + " shared compile resource", await handler({ toolName: "bash", input: { command: "hb_compile -c x.yaml" } }), "block");
|
||
expect(policy + " public adapter write", await handler({ toolName: "write", input: { path: bench + "/harness/adapters/example.py" } }), "block");
|
||
}
|
||
} finally {
|
||
for (const key of Object.keys(process.env)) if (!(key in previous)) delete process.env[key];
|
||
Object.assign(process.env, previous);
|
||
fs.rmSync(root, { recursive: true, force: true });
|
||
}
|
||
}
|
||
extensionCases().then(() => {
|
||
console.log(fails ? `\n${fails} FAILED` : "\nall passed");
|
||
process.exitCode = fails ? 1 : 0;
|
||
}).catch(error => { console.error(error); process.exitCode = 1; });
|
||
|
||
const {forbiddenCommand, shellSegments} = require('./guard-core');
|
||
const forbidden = ['hb_compile', 'ssh'];
|
||
for (const command of ["grep -E 'error| hb_compile' ../stages/log", 'rg "ssh; hb_compile" .', "printf '%s' 'a| ssh'"]) {
|
||
if (forbiddenCommand(command, forbidden) !== null) throw new Error('search/data falsely blocked: ' + command);
|
||
}
|
||
for (const command of ['true | hb_compile -c x', 'echo ok\nssh host', '/opt/bin/hb_compile x']) {
|
||
if (!forbiddenCommand(command, forbidden)?.command) throw new Error('command position missed: ' + command);
|
||
}
|
||
if (shellSegments('echo "$(ssh host)"') !== null) throw new Error('dynamic shell must stay uncertain');
|
||
|
||
const {rootFind} = require('./guard-core');
|
||
for (const command of ["find / -maxdepth 6 -name harness", "find '/' -name harness", "find -L / -name harness", "true && /usr/bin/find / -name harness"]) {
|
||
if (!rootFind(command)) throw new Error('root scan missed: ' + command);
|
||
}
|
||
for (const command of ["rg 'find /' .", "printf '%s' 'find /'", "find . -name harness", "find . -name '/'", "find /tmp -name harness"]) {
|
||
if (rootFind(command)) throw new Error('data or scoped search falsely blocked: ' + command);
|
||
}
|