Adding gates to running tasks' job.json (10-24: G1 require_bpu_compute, G4 budgets) changes the job.json hash in the review binding, so every reviewed PASS became "unreviewed" and dropped out of SFT (05/06: 141/181 -> 3/10). check() now falls back to the latest earlier receipt whose binding differs only by job.json and by gate judgments it never made (absent, null, "na"); transcript, measurements, stage files and every earlier judgment must match. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
680 lines
39 KiB
Python
680 lines
39 KiB
Python
"""Training materialization regressions: evidence retention, pairing and refresh."""
|
|
import hashlib
|
|
import json
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from export import to_agentrl as ex
|
|
from export.reward import compute
|
|
from harness.db import SCHEMA
|
|
from harness.verify import VERDICT_SCHEMA, VERIFICATION_RULE, eval_set
|
|
|
|
|
|
def dialogue():
|
|
return [{"role": "user", "content": "Compile and evaluate."},
|
|
{"role": "assistant", "content": [{"type": "toolCall", "id": "t1", "name": "bash", "arguments": {"command": "bench stage status h"}}]},
|
|
{"role": "toolResult", "toolCallId": "t1", "content": [{"type": "text", "text": "DONE"}]}]
|
|
|
|
|
|
class MaterializationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
(self.root / "agent").mkdir()
|
|
(self.root / "agent/VERSION").write_text("6")
|
|
(self.root / "state").mkdir()
|
|
self.con = sqlite3.connect(self.root / "state/jobs.sqlite")
|
|
self.con.executescript(SCHEMA)
|
|
self.patch = patch.object(ex, "BENCH", self.root)
|
|
self.patch.start()
|
|
self.env = patch.dict("os.environ", {"BENCH_STATE_DIR": str(self.root / "state")})
|
|
self.env.start()
|
|
|
|
def tearDown(self):
|
|
self.env.stop(); self.patch.stop(); self.con.close(); self.tmp.cleanup()
|
|
|
|
def add(self, kk=0, terminal="PASS", execution="DONE", review=False, ended=True, version=6, mode="e2e"):
|
|
jd = self.root / "runs/r/j"
|
|
rd = jd / "rollouts" / f"{kk:02d}"
|
|
(rd / "agent").mkdir(parents=True)
|
|
job = {"run_id": "r", "job_id": "j", "task": "cls", "model": "resnet18", "march": "nash-m",
|
|
"target": {"board_type": "s100p", "march": "nash-m"}, "backend": "ptq",
|
|
"gates": {"required": ["G1", "G3"], "G1": {}, "G2": {},
|
|
"G3": {"metric": "top1", "kind": "rel_pct", "tol": 5, "require_source": "board"}, "G4": {}}}
|
|
job.update(data={"dataset": "fixture", "t1": {"n": 100, "content_sha": "a" * 64,
|
|
"manifest_sha": "b" * 64, "eval_set_id": "fixture-100"}}, evaluator_version="fixture-v1")
|
|
(jd / "job.json").write_text(json.dumps(job))
|
|
(rd / "rollout.json").write_text(json.dumps({"state": execution, "model": "glm-5.3", "policy": "teacher", "mode": mode,
|
|
"runner_version": "pi-sdk-v1", "execution_id": "exec-fixture",
|
|
"agent_config_version": version, "max_turns": 60, "max_tokens": 100}))
|
|
verdict = {"schema": VERDICT_SCHEMA, "verification_rule": VERIFICATION_RULE, "mode": mode, "run_id": "r", "job_id": "j", "rollout": kk, "terminal": terminal,
|
|
"execution_terminal": execution, "timeout_reason": "budget" if execution == "TIMEOUT" else None,
|
|
"claim_consistent": True, "claim": {"terminal": terminal}, "review": review,
|
|
"gates": {"G1": {"verdict": "pass"}, "G2": {"verdict": "pass"}, "G3": {"verdict": "pass-screen"}},
|
|
"cost": {"turns": 30, "tokens": 50}}
|
|
(rd / "verdict.json").write_text(json.dumps(verdict))
|
|
events = [self.context_header(), {"type": "agent_start"}]
|
|
complete_dialogue = dialogue() + [{"role": "assistant", "content": "The measured subset passed within the task tolerance."}]
|
|
events += [{"type": "message_end", "message": message} for message in complete_dialogue]
|
|
if ended:
|
|
events.append({"type": "agent_end", "messages": complete_dialogue})
|
|
(rd / "agent/events.jsonl").write_text("\n".join(json.dumps(e) for e in events))
|
|
(rd / "agent/system.md").write_text("Use tools and report measured results.")
|
|
(rd / "agent/tools.json").write_text('[{"name":"bash","parameters":{"type":"object"}}]')
|
|
# A recorded PASS label alone is not evidence. Every successful fixture
|
|
# supplies the same protected measurement/HBM binding as the verifier.
|
|
(rd / "claim.json").write_text(json.dumps({"terminal": terminal}))
|
|
hbm = rd / "artifacts/j.hbm"
|
|
hbm.parent.mkdir()
|
|
hbm.write_bytes(b"test artifact; never deployed")
|
|
digest = hashlib.sha256(hbm.read_bytes()).hexdigest()
|
|
for stage, measurements in {
|
|
"06-hbm": {"G1": {"hbm_exists": True, "march_ok": True, "io_ok": True,
|
|
"verifier_ok": True, "log_has_error": False, "hbm_sha256": digest}},
|
|
"08-eval": {"G3": {"fp32": 80, "quant": 79.2 if terminal == "PASS" else 50,
|
|
"n": 100, "source": "board", "tier": "t1", "hbm_sha256": digest,
|
|
"eval_set": eval_set(job)}},
|
|
}.items():
|
|
stage_dir = rd / "stages" / stage
|
|
stage_dir.mkdir(parents=True)
|
|
(stage_dir / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements}))
|
|
self.con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','j',?,?)", (kk, terminal))
|
|
self.con.commit()
|
|
self.approve_fixture(rd)
|
|
return rd
|
|
|
|
@staticmethod
|
|
def approve_fixture(rd):
|
|
"""Synthetic reviewer receipt for isolated export tests, never real evidence."""
|
|
from export import content_review as cr, trajectory
|
|
from harness.verify import verify_rollout
|
|
bound = cr.binding(rd, verify_rollout(rd, write=False), trajectory.freeze(rd), ex._gate_sources(rd))
|
|
path = cr.receipt_path(rd, bound)
|
|
path.parent.mkdir(exist_ok=True)
|
|
path.write_text(json.dumps({"schema": "bench.content-review/1", "binding": bound,
|
|
"state": "COMPLETE", "decision": "PASS", "reason": "unit test fixture only",
|
|
"issues": [], "claims": [{"quote": "fixture", "evidence": "fixture", "assessment": "SUPPORTED", "reason": "fixture"}], "model": "synthetic-test-reviewer"}))
|
|
|
|
@staticmethod
|
|
def context_header():
|
|
return {"type": "session", "version": 3, "id": "session-fixture", "bench": {
|
|
"runner_version": "pi-sdk-v1", "execution_id": "exec-fixture",
|
|
"system_prompt": "Use tools and report measured results.",
|
|
"tools": [{"name": "bash", "parameters": {"type": "object"}}]}}
|
|
|
|
def test_legacy_append_context_is_retained_not_sft(self):
|
|
rd = self.add()
|
|
metadata = json.loads((rd / "rollout.json").read_text())
|
|
metadata["runner_version"] = "legacy-pi"
|
|
(rd / "rollout.json").write_text(json.dumps(metadata))
|
|
path = rd / "agent/events.jsonl"
|
|
events = [json.loads(line) for line in path.read_text().splitlines()]
|
|
events[0].pop("bench")
|
|
path.write_text("\n".join(map(json.dumps, events)))
|
|
stats = ex.materialize(["r"], "legacy-context")["stats"]
|
|
self.assertEqual(stats["sft_eligible"], 0)
|
|
record = json.loads((self.root / "datasets/legacy-context/retained.jsonl").read_text())
|
|
self.assertEqual(record["system_source"], "legacy-append-only")
|
|
self.assertEqual(len(record["messages"]), 5)
|
|
self.assertFalse(record["trajectory"]["validation"]["checks"]["REQUIRED_CONTEXT"]["pass"])
|
|
|
|
def test_active_db_state_cannot_use_stale_done_file(self):
|
|
self.add()
|
|
for index, state in enumerate(('PAUSED_INFRA', 'CANCEL_REQUESTED', 'CLEANING', 'PENDING')):
|
|
with self.subTest(state=state):
|
|
self.con.execute('UPDATE rollouts SET state=?', (state,)); self.con.commit()
|
|
stats = ex.materialize(['r'], f'active-{index}')['stats']
|
|
self.assertEqual((stats['scanned'], stats['sft_eligible']), (0, 0))
|
|
|
|
def test_invalid_stage_json_retains_failure_without_blocking_other_rollouts(self):
|
|
bad = self.add(kk=0)
|
|
self.add(kk=1)
|
|
malformed = bad / 'stages/06-hbm/a01/meta.json'
|
|
malformed.parent.mkdir()
|
|
malformed.write_bytes(b'')
|
|
result = ex.materialize(['r'], 'corrupt-stage')
|
|
self.assertEqual((result['stats']['scanned'], result['stats']['sft_eligible']), (2, 1))
|
|
output = self.root / 'datasets/corrupt-stage'
|
|
records = [json.loads(line) for line in (output / 'disposition.jsonl').read_text().splitlines()]
|
|
self.assertEqual(records[0]['reasons'], ['reverification_failed:JSONDecodeError'])
|
|
self.assertEqual(records[1]['disposition'], 'accepted')
|
|
evidence = [json.loads(line) for line in (output / 'source-manifest.jsonl').read_text().splitlines()]
|
|
self.assertEqual(evidence[0]['errors'], ['HARD_GATES_UNAVAILABLE:JSONDecodeError'])
|
|
self.assertEqual(malformed.read_bytes(), b'')
|
|
self.assertEqual(json.loads((output / 'sft.jsonl').read_text())['rollout'], 1)
|
|
|
|
def test_db_reactivation_during_build_fails_closed(self):
|
|
self.add()
|
|
reconstruct = ex.preserved.reconstruct
|
|
def activate(source):
|
|
result = reconstruct(source)
|
|
self.con.execute("UPDATE rollouts SET state='PAUSED_INFRA'"); self.con.commit()
|
|
return result
|
|
with patch.object(ex.preserved, 'reconstruct', side_effect=activate):
|
|
self.assertEqual(ex.materialize(['r'], 'changed-state')['stats']['sft_eligible'], 0)
|
|
|
|
def test_matching_invalid_eval_identities_are_not_evidence(self):
|
|
rd = self.add()
|
|
job_path, meta_path = rd.parent.parent / 'job.json', rd / 'stages/08-eval/meta.json'
|
|
original_job, original_meta = job_path.read_text(), meta_path.read_text()
|
|
invalid = [('dataset', ''), ('evaluator_version', ''), ('content_sha', ''),
|
|
('manifest_sha', 'not-a-hash'), ('n', 0), ('n', True)]
|
|
for index, (field, value) in enumerate(invalid):
|
|
with self.subTest(field=field, value=value):
|
|
job, meta = json.loads(original_job), json.loads(original_meta)
|
|
if field == 'dataset': job['data'][field] = value
|
|
elif field == 'evaluator_version': job[field] = value
|
|
else: job['data']['t1'][field] = value
|
|
meta['measurements']['G3']['eval_set'][field] = value
|
|
if field == 'n': meta['measurements']['G3']['n'] = value
|
|
job_path.write_text(json.dumps(job)); meta_path.write_text(json.dumps(meta))
|
|
self.assertEqual(ex.materialize(['r'], f'invalid-id-{index}')['stats']['sft_eligible'], 0)
|
|
|
|
def test_completed_tool_tail_is_preserved(self):
|
|
msgs, stats = ex.clean(dialogue())
|
|
self.assertEqual(msgs[-1]["role"], "tool")
|
|
self.assertEqual(len(msgs), 3)
|
|
self.assertTrue(stats["tool_pairing"]["valid"])
|
|
|
|
def test_retry_export_retains_completed_prefix(self):
|
|
rd = self.add()
|
|
first = dialogue()
|
|
last = {"role": "assistant", "content": "finished after provider retry"}
|
|
events = [{"type": "agent_start"}]
|
|
events += [{"type": "message_end", "message": m} for m in first]
|
|
events += [{"type": "agent_end", "messages": first}, {"type": "agent_start"},
|
|
{"type": "message_end", "message": last}, {"type": "agent_end", "messages": [last]}]
|
|
(rd / "agent/events.jsonl").write_text("\n".join(map(json.dumps, events)))
|
|
raw, _ = ex.load_trajectory(rd)
|
|
self.assertEqual(raw, first + [last])
|
|
|
|
def test_dangling_and_orphan_tool_evidence_is_labeled(self):
|
|
msgs, stats = ex.clean(dialogue()[:-1])
|
|
self.assertEqual(len(msgs), 2)
|
|
self.assertEqual(stats["tool_pairing"]["missing_results"], 1)
|
|
_, stats = ex.clean([dialogue()[0], dialogue()[2]])
|
|
self.assertEqual(stats["tool_pairing"]["orphan_results"], 1)
|
|
|
|
def test_timeout_keeps_verified_stage_rewards(self):
|
|
rd = self.add(execution="TIMEOUT")
|
|
reward = compute(json.loads((rd / "verdict.json").read_text()), 60, 100)
|
|
self.assertEqual(reward["recipe"], "v0.8")
|
|
self.assertEqual(reward["terms"]["G3"], 2.0)
|
|
self.assertEqual(reward["terms"]["timeout"], -1.0)
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
|
|
self.assertEqual(manifest["stats"]["by_march"], {"nash-m": 1})
|
|
self.assertEqual(manifest["stats"]["sft_eligible_by_march"], {"nash-m": 1})
|
|
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertEqual(rec["board_type"], "s100p")
|
|
self.assertEqual(rec["verdict"]["terminal"], "PASS")
|
|
self.assertEqual(rec["execution"]["terminal"], "TIMEOUT")
|
|
|
|
def test_all_retains_timeout_failure_and_partial_without_inventing_messages(self):
|
|
self.add(terminal="TIMEOUT", execution="TIMEOUT", ended=False)
|
|
self.add(kk=1, terminal="FAIL")
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["written"], 2)
|
|
self.assertEqual(manifest["stats"]["partial"], 1)
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
|
|
recs = [json.loads(line) for line in (self.root / "datasets/data/teacher.jsonl").read_text().splitlines()]
|
|
self.assertEqual(recs[0]["trajectory"]["source"], "message_end")
|
|
self.assertEqual(sum(m["role"] == "assistant" for m in recs[0]["messages"]), 2)
|
|
|
|
def legacy_evidence(self, rd, *, mode="e2e", claim="PASS"):
|
|
original = json.loads((rd / "verdict.json").read_text())
|
|
original.update(schema="bench.verdict/2", review=True,
|
|
oracle_ref={"terminal": "FAIL", "G3_delta": -99})
|
|
(rd / "verdict.json").write_text(json.dumps(original))
|
|
(rd / "claim.json").write_text(json.dumps({"terminal": claim}))
|
|
hbm = rd / "artifacts/j.hbm"
|
|
hbm.parent.mkdir(exist_ok=True)
|
|
hbm.write_bytes(b"fixture HBM identity; never loaded on a board")
|
|
digest = hashlib.sha256(hbm.read_bytes()).hexdigest()
|
|
for stage, measurements in {
|
|
"06-hbm": {"G1": {"hbm_exists": True, "march_ok": True, "io_ok": True,
|
|
"verifier_ok": True, "log_has_error": False, "hbm_sha256": digest}},
|
|
"08-eval": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1",
|
|
"hbm_sha256": digest,
|
|
"eval_set": eval_set(json.loads((rd.parent.parent / "job.json").read_text()))}},
|
|
}.items():
|
|
sd = rd / "stages" / stage
|
|
sd.mkdir(parents=True, exist_ok=True)
|
|
(sd / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements}))
|
|
self.approve_fixture(rd)
|
|
return original
|
|
|
|
def test_legacy_missing_hbm_identity_does_not_become_sft_evidence(self):
|
|
rd = self.add(review=True)
|
|
self.legacy_evidence(rd)
|
|
path = rd / "stages/08-eval/meta.json"
|
|
metadata = json.loads(path.read_text())
|
|
del metadata["measurements"]["G3"]["hbm_sha256"]
|
|
path.write_text(json.dumps(metadata))
|
|
before = {p: p.read_bytes() for p in rd.rglob("*") if p.is_file()}
|
|
stats = ex.materialize(["r"], "sft", purpose="sft")["stats"]
|
|
self.assertEqual(stats["written"], 0)
|
|
self.assertEqual(before, {p: p.read_bytes() for p in before})
|
|
|
|
def test_legacy_oracle_review_is_reverified_without_modifying_sources(self):
|
|
rd = self.add(review=True)
|
|
self.legacy_evidence(rd)
|
|
before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in rd.rglob("*") if p.is_file()}
|
|
before_db = list(self.con.execute("SELECT * FROM rollouts"))
|
|
manifest = ex.materialize(["r"], "data", purpose="sft")
|
|
self.assertEqual(manifest["stats"]["reverified"], 1)
|
|
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertTrue(rec["sft_eligible"])
|
|
self.assertTrue(rec["training_uses"]["reward_learning_candidate"])
|
|
self.assertEqual(rec["verification"]["method"], "readonly-reverification")
|
|
self.assertEqual(rec["verdict"]["schema"], VERDICT_SCHEMA)
|
|
self.assertFalse(rec["verdict"]["review"])
|
|
self.assertNotIn("oracle_ref", rec)
|
|
self.assertNotIn("history", rec)
|
|
self.assertTrue(rec["provenance"]["original_verdict"]["review"])
|
|
self.assertEqual(rec["provenance"]["original_verdict"]["sha256"], before[rd / "verdict.json"])
|
|
self.assertEqual(before, {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in before})
|
|
self.assertEqual(before_db, list(self.con.execute("SELECT * FROM rollouts")))
|
|
|
|
def test_failed_reverification_is_excluded_without_legacy_fallback(self):
|
|
rd = self.add(review=True)
|
|
self.legacy_evidence(rd)
|
|
(rd / "stages/08-eval/meta.json").write_text("broken evidence")
|
|
stats = ex.materialize(["r"], "data")["stats"]
|
|
self.assertEqual(stats["written"], 0)
|
|
self.assertEqual(stats["dropped"], {"reverification_failed:JSONDecodeError": 1})
|
|
|
|
def test_historical_repair_remains_analysis_only(self):
|
|
rd = self.add(mode="repair")
|
|
self.legacy_evidence(rd)
|
|
ex.materialize(["r"], "all")
|
|
rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text())
|
|
self.assertEqual(rec["mode"], "repair")
|
|
self.assertFalse(rec["sft_eligible"])
|
|
self.assertEqual(rec["sft_reason"], "historical-repair-context")
|
|
self.assertFalse(rec["training_uses"]["reward_learning_candidate"])
|
|
self.assertEqual(ex.materialize(["r"], "sft", purpose="sft")["stats"]["written"], 0)
|
|
self.assertEqual(ex.materialize(["r"], "rl", purpose="rl")["stats"]["written"], 0)
|
|
|
|
def test_blocked_self_report_cannot_override_measured_failure(self):
|
|
rd = self.add(terminal="BLOCKED")
|
|
v = json.loads((rd / "verdict.json").read_text())
|
|
v.update(blocked_correct=True, oracle_ref={"terminal": "FAIL"})
|
|
(rd / "verdict.json").write_text(json.dumps(v))
|
|
ex.materialize(["r"], "all")
|
|
rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text())
|
|
self.assertFalse(rec["sft_eligible"])
|
|
self.assertEqual(rec["verdict"]["terminal"], "FAIL")
|
|
self.assertTrue(rec["training_uses"]["reward_learning_candidate"])
|
|
self.assertNotIn("blocked", rec["reward_terms"])
|
|
|
|
def test_oracle_independence_does_not_bypass_transcript_or_annotation_checks(self):
|
|
first = self.add(ended=False)
|
|
self.legacy_evidence(first)
|
|
second = self.add(kk=1)
|
|
self.legacy_evidence(second)
|
|
(second / "claim.json").unlink()
|
|
third = self.add(kk=2)
|
|
self.legacy_evidence(third)
|
|
self.con.execute("INSERT INTO annotations(run_id,job_id,note) VALUES('r','j','rollout#2 contaminated: copied solution')")
|
|
self.con.commit()
|
|
stats = ex.materialize(["r"], "data", purpose="sft")["stats"]
|
|
self.assertEqual(stats["written"], 0)
|
|
self.assertEqual(stats["dropped"], {"sft:partial-transcript": 1, "sft:content-review-unreviewed": 1,
|
|
"contaminated": 1})
|
|
|
|
def test_old_version_and_missing_verdict_have_explicit_exclusions(self):
|
|
self.add(version=5)
|
|
rd = self.add(kk=1)
|
|
(rd / "verdict.json").unlink()
|
|
stats = ex.materialize(["r"], "data")["stats"]
|
|
self.assertEqual((stats["scanned"], stats["excluded"], stats["written"]), (2, 2, 0))
|
|
exclusions = (self.root / "datasets/data/exclusions.jsonl").read_text()
|
|
self.assertIn("old_version", exclusions)
|
|
self.assertIn("missing_verdict", exclusions)
|
|
|
|
def test_sft_and_rl_exports_do_not_read_historical_pipeline(self):
|
|
self.add()
|
|
with patch.object(ex, "pipeline_records", side_effect=OSError("unreadable old pipeline")) as pipeline:
|
|
for purpose in ("sft", "rl"):
|
|
with self.subTest(purpose=purpose):
|
|
result = ex.materialize(["r"], purpose, purpose=purpose)
|
|
self.assertEqual(result["stats"]["agent_written"], 1)
|
|
self.assertEqual(result["stats"]["pipeline_written"], 0)
|
|
self.assertEqual((self.root / "datasets" / purpose / "pipeline.jsonl").read_text(), "")
|
|
pipeline.assert_not_called()
|
|
|
|
def test_pipeline_is_real_separate_and_failed_jobs_are_retained(self):
|
|
self.add()
|
|
jd = self.root / "runs/r/j"
|
|
sd = jd / "oracle/stages/05-compile/a00"
|
|
sd.mkdir(parents=True)
|
|
(sd / "meta.json").write_text(json.dumps({"cmd": ["hb_compile", "--config", "compile.yaml"], "state": "FAIL", "error_class": "unsupported"}))
|
|
(jd / "oracle/result.json").write_text('{"terminal":"FAIL","error_class":"unsupported"}')
|
|
self.con.execute("INSERT INTO jobs(run_id,job_id,mode,state,finished_at) VALUES('r','j','oracle','FAIL',123)")
|
|
self.con.execute("INSERT INTO stages(run_id,job_id,rollout,stage,state) VALUES('r','j',-1,'05-compile','FAIL')")
|
|
self.con.commit()
|
|
stats = ex.materialize(["r"], "data")["stats"]
|
|
self.assertEqual((stats["agent_written"], stats["pipeline_written"]), (1, 1))
|
|
rec = json.loads((self.root / "datasets/data/pipeline.jsonl").read_text())
|
|
self.assertEqual(rec["source"], "deterministic_pipeline")
|
|
self.assertNotIn("messages", rec)
|
|
self.assertEqual(rec["stages"][0]["command"][0], "hb_compile")
|
|
|
|
def test_refresh_is_explicit_stable_and_hash_checked(self):
|
|
self.add()
|
|
ex.materialize(["r"], "data")
|
|
out = self.root / "datasets/data"
|
|
first = (out / "teacher.jsonl").read_bytes()
|
|
with self.assertRaises(FileExistsError):
|
|
ex.materialize(["r"], "data")
|
|
manifest = ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
|
|
self.assertEqual(manifest["files"]["teacher.jsonl"]["sha256"], hashlib.sha256(first).hexdigest())
|
|
self.assertFalse(list(out.glob(".refresh-*")))
|
|
with patch.object(ex, "_write_dataset", side_effect=RuntimeError("interrupted")):
|
|
with self.assertRaises(RuntimeError):
|
|
ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
|
|
self.assertFalse(list(out.glob(".refresh-*")))
|
|
|
|
def test_refresh_preserves_filters_and_pi_threshold_after_version_bump(self):
|
|
self.add()
|
|
first = ex.materialize(["r"], "data", purpose="sft", policy="teacher", min_reward=1,
|
|
max_turns=70, max_tokens=500)
|
|
(self.root / "agent/VERSION").write_text("100")
|
|
second = ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first["filters"], second["filters"])
|
|
self.assertEqual(second["stats"]["written"], 1)
|
|
self.assertEqual(second["filter_history"], [])
|
|
changed = ex.materialize(["r"], "data", refresh=True, purpose="all", policy=None)
|
|
self.assertEqual(changed["filter_history"][-1]["explicit_fields"], ["policy", "purpose"])
|
|
self.assertEqual(changed["filter_history"][-1]["reason"], "explicit_filter_change")
|
|
self.assertEqual(changed["filters"]["min_agent_version"], 6)
|
|
|
|
def test_exact_version_refresh_and_explicit_minimum_are_audited(self):
|
|
self.add(version=5)
|
|
first = ex.materialize(["r"], "data", agent_version="5")
|
|
(self.root / "agent/VERSION").write_text("100")
|
|
second = ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first["filters"], second["filters"])
|
|
self.assertEqual(second["stats"]["written"], 1)
|
|
third = ex.materialize(["r"], "data", refresh=True, agent_version=None, min_agent_version=10)
|
|
self.assertEqual(third["stats"]["written"], 0)
|
|
self.assertEqual(third["filter_history"][-1]["explicit_fields"], ["agent_version", "min_agent_version"])
|
|
|
|
def test_default_pi_version_does_not_filter_other_runner(self):
|
|
rd = self.add(version=1)
|
|
rollout = json.loads((rd / "rollout.json").read_text())
|
|
rollout["runner"] = "opencode"
|
|
(rd / "rollout.json").write_text(json.dumps(rollout))
|
|
with patch.object(ex, "load_trajectory", return_value=(dialogue(), {"partial": False, "source": "fixture"})):
|
|
first = ex.materialize(["r"], "data")
|
|
self.assertEqual(first["stats"]["written"], 1)
|
|
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertEqual(rec["runner"], "opencode")
|
|
explicit = ex.materialize(["r"], "explicit", min_agent_version=6)
|
|
self.assertEqual(explicit["stats"]["written"], 0)
|
|
|
|
def opencode(self, rd):
|
|
meta = json.loads((rd / "rollout.json").read_text())
|
|
meta.update(runner="opencode", runner_version="1.2.0", agent_config_version="1")
|
|
(rd / "rollout.json").write_text(json.dumps(meta))
|
|
(rd / "agent/session-id.json").write_text('{"session_id":"ses_fixture"}')
|
|
raw = {"info": {"id": "ses_fixture"}, "messages": [
|
|
{"info": {"role": "user"}, "parts": [{"type": "text", "text": "Compile and evaluate."}]},
|
|
{"info": {"role": "assistant", "time": {"completed": 123}}, "parts": [
|
|
{"type": "text", "text": "Measured result."},
|
|
{"type": "tool", "callID": "t1", "tool": "bash",
|
|
"state": {"status": "completed", "input": {"command": "bench stage status h"}, "output": "DONE"}}]}]}
|
|
(rd / "agent/raw-session.json").write_text(json.dumps(raw))
|
|
(rd / "agent/raw-system.json").write_text('["Actual runner system", "Actual harness instruction"]')
|
|
|
|
def test_opencode_exports_actual_context_and_retains_runner_identity(self):
|
|
rd = self.add()
|
|
self.opencode(rd)
|
|
result = ex.materialize(["r"], "data")
|
|
self.assertEqual(result["stats"]["sft_eligible"], 0)
|
|
record = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertEqual(record["runner_version"], "1.2.0")
|
|
self.assertEqual(record["system_source"], "runner-raw-system")
|
|
self.assertEqual(record["messages"][0]["content"], "Actual runner system\n\nActual harness instruction")
|
|
self.assertEqual(record["trajectory"]["system_segments"], ["Actual runner system", "Actual harness instruction"])
|
|
self.assertEqual(record["trajectory"]["source"], "opencode-raw-session")
|
|
self.assertEqual(record["sft_reason"], "unsupported-strict-sft-source")
|
|
|
|
def test_opencode_missing_system_or_tools_is_not_sft_and_never_pi_redumps(self):
|
|
rd = self.add()
|
|
self.opencode(rd)
|
|
(rd / "agent/raw-system.json").unlink()
|
|
self.assertEqual(ex.system_prompt(rd), ("", "missing"))
|
|
ex.materialize(["r"], "no-system")
|
|
record = json.loads((self.root / "datasets/no-system/teacher.jsonl").read_text())
|
|
self.assertFalse(record["sft_eligible"])
|
|
self.assertEqual(record["sft_reason"], "unsupported-strict-sft-source")
|
|
(rd / "agent/tools.json").unlink()
|
|
(rd / "agent/cmd.json").write_text('{"cmd":["pi","--tools","bash"]}')
|
|
with patch("subprocess.run", side_effect=AssertionError("must not redump Pi")):
|
|
self.assertEqual(ex.tool_defs(rd), ([], "missing"))
|
|
|
|
def test_current_verdict_cannot_bypass_hard_gate_reverification(self):
|
|
for i, mutation in enumerate(("hash", "source", "threshold", "missing")):
|
|
with self.subTest(mutation=mutation):
|
|
rd = self.add(kk=i)
|
|
path = rd / "stages/08-eval/meta.json"
|
|
meta = json.loads(path.read_text())
|
|
if mutation == "hash":
|
|
meta["measurements"]["G3"]["hbm_sha256"] = "0" * 64
|
|
elif mutation == "source":
|
|
meta["measurements"]["G3"]["source"] = "simulator"
|
|
elif mutation == "threshold":
|
|
meta["measurements"]["G3"]["quant"] = 10
|
|
else:
|
|
meta["measurements"] = {}
|
|
path.write_text(json.dumps(meta))
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
|
|
self.assertEqual(manifest["stats"]["written"], 4)
|
|
|
|
def test_review_nested_secret_and_private_identity_never_publish(self):
|
|
from export.test_trajectory import segment, message, call, result
|
|
nested = "REVIEW_TOKEN_DO_NOT_PUBLISH"
|
|
secret_id = "sk-REVIEWFAKE01234567890123"
|
|
rd = self.add()
|
|
msgs = dialogue() + [{"role": "assistant", "content": "Measured subset complete."}]
|
|
msgs[1]["content"][0]["arguments"]["authorization"] = {"Bearer": [nested, {"nested": nested}]}
|
|
(rd / "agent/events.jsonl").write_text("\n".join(map(json.dumps, [self.context_header()] + segment(msgs))))
|
|
self.approve_fixture(rd)
|
|
private = self.add(kk=1)
|
|
events = segment([message("user", "task"), call(secret_id), result(secret_id)]) + [{"type": secret_id}]
|
|
(private / "agent/events.jsonl").write_text("\n".join(map(json.dumps, events)))
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
|
|
for path in (self.root / "datasets/data").glob("*.jsonl"):
|
|
self.assertNotIn(nested, path.read_text(), path.name)
|
|
self.assertNotIn(secret_id, path.read_text(), path.name)
|
|
|
|
def test_review_survives_job_json_edit_that_only_adds_gates(self):
|
|
rd = self.add()
|
|
job_path = rd.parent.parent / "job.json"
|
|
job = json.loads(job_path.read_text())
|
|
job["gates"]["G1"]["require_bpu_compute"] = True
|
|
job_path.write_text(json.dumps(job))
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
|
|
record = json.loads((self.root / "datasets/data/sft.jsonl").read_text())
|
|
self.assertIn("carried_from_binding_sha256", record["content_review"])
|
|
meta_path = rd / "stages/08-eval/meta.json"
|
|
meta = json.loads(meta_path.read_text())
|
|
meta["measurements"]["G3"]["quant"] = 79.0 # still within tolerance, but not what was reviewed
|
|
meta_path.write_text(json.dumps(meta))
|
|
self.assertEqual(ex.materialize(["r"], "data2")["stats"]["sft_eligible"], 0)
|
|
|
|
def test_gate_only_change_allows_new_judgments_only(self):
|
|
from copy import deepcopy
|
|
from export.content_review import gate_only_change
|
|
old = {"rule": "r", "gate_manifest": {"job.json": {"sha256": "a"}, "rollout.json": {"sha256": "c"}},
|
|
"authoritative_result": {"terminal": "PASS", "required": ["G1", "G3"],
|
|
"gates": {"G3": {"verdict": "pass", "quant": 79.2},
|
|
"G4": {"verdict": "na", "budget_ms": None}}}}
|
|
new = deepcopy(old)
|
|
new["gate_manifest"]["job.json"] = {"sha256": "b"}
|
|
new["authoritative_result"]["required"].append("G4")
|
|
new["authoritative_result"]["gates"]["G4"] = {"verdict": "pass", "budget_ms": 3.7}
|
|
self.assertTrue(gate_only_change(old, new))
|
|
for path, value in ((("gates", "G3", "verdict"), "pass-screen"), (("terminal",), "FAIL"),
|
|
(("required",), ["G1", "G4"])):
|
|
changed = deepcopy(new)
|
|
target = changed["authoritative_result"]
|
|
for key in path[:-1]:
|
|
target = target[key]
|
|
target[path[-1]] = value
|
|
self.assertFalse(gate_only_change(old, changed), path)
|
|
changed = deepcopy(new)
|
|
changed["gate_manifest"]["rollout.json"] = {"sha256": "d"}
|
|
self.assertFalse(gate_only_change(old, changed))
|
|
self.assertFalse(gate_only_change(None, new))
|
|
|
|
def test_review_unknown_execution_state_is_not_stable(self):
|
|
for i, state in enumerate(("PENDING", "future-terminal", None)):
|
|
rd = self.add(kk=i)
|
|
path = rd / "rollout.json"
|
|
metadata = json.loads(path.read_text())
|
|
metadata["state"] = state
|
|
path.write_text(json.dumps(metadata))
|
|
self.assertEqual(ex.materialize(["r"], "data")["stats"]["sft_eligible"], 0)
|
|
|
|
def test_review_source_change_after_validation_is_not_accepted(self):
|
|
from export import trajectory as tr
|
|
rd = self.add()
|
|
validate = tr.validate
|
|
def mutate(source, output):
|
|
checked = validate(source, output)
|
|
with (rd / "agent/events.jsonl").open("a") as stream:
|
|
stream.write('\n{"type":"message_end","message":{"role":"assistant","content":"later"}}')
|
|
return checked
|
|
with patch.object(tr, "validate", side_effect=mutate):
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
|
|
record = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertIn("SOURCE_CHANGED", record["trajectory"]["validation"]["checks"]["SOURCE_INTEGRITY"]["errors"])
|
|
|
|
def test_review_g3_identity_and_unwidened_tolerance(self):
|
|
for i, mutation in enumerate(("dataset", "content_sha", "manifest_sha", "eval_set_id", "evaluator_version", "missing", "se")):
|
|
rd = self.add(kk=i)
|
|
path = rd / "stages/08-eval/meta.json"
|
|
meta = json.loads(path.read_text())
|
|
values = meta["measurements"]["G3"]
|
|
if mutation == "missing":
|
|
values.pop("eval_set", None)
|
|
elif mutation == "se":
|
|
values.update(quant=75.2, se=1)
|
|
else:
|
|
values.setdefault("eval_set", {})[mutation] = "wrong"
|
|
path.write_text(json.dumps(meta))
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
|
|
|
|
def test_review_data010_runs_in_materialization_without_truncation(self):
|
|
from export.test_trajectory import segment, message, call, result
|
|
rd = self.add()
|
|
full = json.dumps({"padding": "p" * 6100, "critical_metric": {"score": 0.9123}, "tail": "t" * 2500})
|
|
msgs = [message("user", "task"), call("one"), result("one", full),
|
|
message("assistant", "Use `/critical_metric/score` from the result.")]
|
|
(rd / "agent/events.jsonl").write_text("\n".join(map(json.dumps, [self.context_header()] + segment(msgs))))
|
|
self.approve_fixture(rd)
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
|
|
out = self.root / "datasets/data"
|
|
annotations = [json.loads(line) for line in (out / "annotations.jsonl").read_text().splitlines()]
|
|
critical = [a for a in annotations if a.get("pointer") == "/critical_metric/score"]
|
|
self.assertEqual(critical[0]["tag"], "REFERENCED_CONTENT_REMOVED")
|
|
self.assertEqual(critical[0]["comparison_source"], "legacy-clean-replay")
|
|
self.assertEqual(json.loads((out / "sft.jsonl").read_text())["messages"][3]["content"][0]["text"], full)
|
|
|
|
def test_new_eval_receipt_binds_actual_package_and_evaluator(self):
|
|
import io
|
|
import tarfile
|
|
from harness.stages.s08_eval import Eval
|
|
rd = self.add()
|
|
job = json.loads((rd.parent.parent / "job.json").read_text())
|
|
package = self.root / "test-eval.tar"
|
|
manifest = {"dataset": "fixture", "sha256": "a" * 64,
|
|
"items": [{"id": str(i), "file": str(i)} for i in range(100)]}
|
|
blob = json.dumps(manifest).encode()
|
|
with tarfile.open(package, "w") as archive:
|
|
info = tarfile.TarInfo("manifest.json"); info.size = len(blob)
|
|
archive.addfile(info, io.BytesIO(blob))
|
|
job["data"]["t1"].update(path=str(package), manifest_sha=hashlib.sha256(package.read_bytes()).hexdigest())
|
|
stage_dir = rd / "stages/08-eval"
|
|
measured = {"G3": {"n": 100}, "evaluator_version": "fixture-v1", "t1_manifest_sha": "a" * 64,
|
|
"n": 100, "task": "cls", "model": "resnet18"}
|
|
path = stage_dir / "eval.json"
|
|
path.write_text(json.dumps(measured))
|
|
ctx, outcome = SimpleNamespace(job=job), SimpleNamespace(meta={})
|
|
self.assertEqual(Eval().measure(ctx, stage_dir, outcome)["G3"]["eval_set"], eval_set(job))
|
|
measured["evaluator_version"] = "unrelated-evaluator"
|
|
path.write_text(json.dumps(measured))
|
|
with self.assertRaisesRegex(ValueError, "evaluated set identity"):
|
|
Eval().measure(ctx, stage_dir, outcome)
|
|
|
|
def test_published_sidecars_and_source_validator_cover_each_attempt(self):
|
|
self.add()
|
|
self.add(kk=1, terminal="FAIL")
|
|
self.add(kk=2, version=1)
|
|
manifest = ex.materialize(["r"], "data")
|
|
out = self.root / "datasets/data"
|
|
rows = lambda name: [json.loads(line) for line in (out / (name + ".jsonl")).read_text().splitlines()]
|
|
self.assertEqual(len(rows("disposition")), 3)
|
|
self.assertEqual(len(rows("source-manifest")), 3)
|
|
self.assertEqual(len(rows("sft")), 1)
|
|
for name, info in manifest["files"].items():
|
|
self.assertEqual(hashlib.sha256((out / name).read_bytes()).hexdigest(), info["sha256"])
|
|
record = rows("sft")[0]
|
|
lineage = [row for row in rows("lineage") if row["rollout"] == 0]
|
|
excluded = [row for row in rows("excluded_observations") if row["rollout"] == 0]
|
|
self.assertTrue(ex.validate_exported_record(self.root, record, lineage, excluded)["pass"])
|
|
record["messages"].pop()
|
|
self.assertFalse(ex.validate_exported_record(self.root, record, lineage, excluded)["pass"])
|
|
|
|
def test_bad_runner_session_is_excluded_without_aborting_dataset(self):
|
|
rd = self.add()
|
|
self.opencode(rd)
|
|
(rd / "agent/session-id.json").write_text('{"session_id":"wrong"}')
|
|
result = ex.materialize(["r"], "data")
|
|
self.assertEqual(result["stats"]["dropped"], {"invalid_trajectory:ValueError": 1})
|
|
|
|
def test_explicit_and_frozen_minimum_do_not_read_current_version_file(self):
|
|
self.add()
|
|
(self.root / "agent/VERSION").unlink()
|
|
first = ex.materialize(["r"], "data", min_agent_version=1)
|
|
(self.root / "agent/VERSION").write_text("broken")
|
|
refreshed = ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first["filters"], refreshed["filters"])
|
|
self.assertEqual(refreshed["stats"]["written"], 1)
|
|
|
|
def test_claim_is_not_an_outcome_gate(self):
|
|
self.assertTrue(ex.sft_eligibility({"terminal": "PASS"}, 4.0)[0])
|
|
self.assertTrue(ex.sft_eligibility({"terminal": "PASS", "claim_consistent": True}, 4.0)[0])
|
|
|
|
def test_redump_uses_configured_node_and_path(self):
|
|
rd = self.add()
|
|
(rd / "agent/tools.json").unlink()
|
|
(rd / "agent/cmd.json").write_text('{"cmd":["pi","--tools","bash"]}')
|
|
(self.root / "agent/dump_tools.js").write_text("// test")
|
|
with patch("harness.paths.config", return_value={"paths": {"node_bin": "/opt/bench-node/bin/node"}}), \
|
|
patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout='[{"name":"bash"}]')) as run:
|
|
definitions, source = ex.tool_defs(rd)
|
|
self.assertEqual(source, "redumped")
|
|
self.assertEqual(definitions[0]["name"], "bash")
|
|
self.assertEqual(run.call_args.args[0][0], "/opt/bench-node/bin/node")
|
|
self.assertTrue(run.call_args.kwargs["env"]["PATH"].startswith("/opt/bench-node/bin:"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|