data-gen-bpu/export/test_materialization.py

698 lines
39 KiB
Python

"""Training materialization regressions: evidence retention, pairing and refresh."""
import hashlib
import json
import sqlite3
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from export import to_agentrl as ex
from export.reward import compute
from harness.db import SCHEMA
from harness.verify import VERDICT_SCHEMA, VERIFICATION_RULE, eval_set
def dialogue():
return [{"role": "user", "content": "Compile and evaluate."},
{"role": "assistant", "content": [{"type": "toolCall", "id": "t1", "name": "bash", "arguments": {"command": "bench stage status h"}}]},
{"role": "toolResult", "toolCallId": "t1", "content": [{"type": "text", "text": "DONE"}]}]
class MaterializationTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
(self.root / "agent").mkdir()
(self.root / "agent/VERSION").write_text("6")
(self.root / "state").mkdir()
self.con = sqlite3.connect(self.root / "state/jobs.sqlite")
self.con.executescript(SCHEMA)
self.patch = patch.object(ex, "BENCH", self.root)
self.patch.start()
self.env = patch.dict("os.environ", {"BENCH_STATE_DIR": str(self.root / "state")})
self.env.start()
def tearDown(self):
self.env.stop(); self.patch.stop(); self.con.close(); self.tmp.cleanup()
def add(self, kk=0, terminal="PASS", execution="DONE", review=False, ended=True, version=6, mode="e2e"):
jd = self.root / "runs/r/j"
rd = jd / "rollouts" / f"{kk:02d}"
(rd / "agent").mkdir(parents=True)
job = {"run_id": "r", "job_id": "j", "task": "cls", "model": "resnet18", "march": "nash-m",
"target": {"board_type": "s100p", "march": "nash-m"}, "backend": "ptq",
"gates": {"required": ["G1", "G3"], "G1": {}, "G2": {},
"G3": {"metric": "top1", "kind": "rel_pct", "tol": 5, "require_source": "board"}, "G4": {}}}
job.update(data={"dataset": "fixture", "t1": {"n": 100, "content_sha": "a" * 64,
"manifest_sha": "b" * 64, "eval_set_id": "fixture-100"}}, evaluator_version="fixture-v1")
(jd / "job.json").write_text(json.dumps(job))
(rd / "rollout.json").write_text(json.dumps({"state": execution, "model": "glm-5.3", "policy": "teacher", "mode": mode,
"runner_version": "pi-sdk-v1", "execution_id": "exec-fixture",
"agent_config_version": version, "max_turns": 60, "max_tokens": 100}))
verdict = {"schema": VERDICT_SCHEMA, "verification_rule": VERIFICATION_RULE, "mode": mode, "run_id": "r", "job_id": "j", "rollout": kk, "terminal": terminal,
"execution_terminal": execution, "timeout_reason": "budget" if execution == "TIMEOUT" else None,
"claim_consistent": True, "claim": {"terminal": terminal}, "review": review,
"gates": {"G1": {"verdict": "pass"}, "G2": {"verdict": "pass"}, "G3": {"verdict": "pass-screen"}},
"cost": {"turns": 30, "tokens": 50}}
(rd / "verdict.json").write_text(json.dumps(verdict))
events = [self.context_header(), {"type": "agent_start"}]
complete_dialogue = dialogue() + [{"role": "assistant", "content": "The measured subset passed within the task tolerance."}]
events += [{"type": "message_end", "message": message} for message in complete_dialogue]
if ended:
events.append({"type": "agent_end", "messages": complete_dialogue})
(rd / "agent/events.jsonl").write_text("\n".join(json.dumps(e) for e in events))
(rd / "agent/system.md").write_text("Use tools and report measured results.")
(rd / "agent/tools.json").write_text('[{"name":"bash","parameters":{"type":"object"}}]')
# A recorded PASS label alone is not evidence. Every successful fixture
# supplies the same protected measurement/HBM binding as the verifier.
(rd / "claim.json").write_text(json.dumps({"terminal": terminal}))
hbm = rd / "artifacts/j.hbm"
hbm.parent.mkdir()
hbm.write_bytes(b"test artifact; never deployed")
digest = hashlib.sha256(hbm.read_bytes()).hexdigest()
for stage, measurements in {
"06-hbm": {"G1": {"hbm_exists": True, "march_ok": True, "io_ok": True,
"verifier_ok": True, "log_has_error": False, "hbm_sha256": digest}},
"08-eval": {"G3": {"fp32": 80, "quant": 79.2 if terminal == "PASS" else 50,
"n": 100, "source": "board", "tier": "t1", "hbm_sha256": digest,
"eval_set": eval_set(job)}},
}.items():
stage_dir = rd / "stages" / stage
stage_dir.mkdir(parents=True)
(stage_dir / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements}))
self.con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','j',?,?)", (kk, terminal))
self.con.commit()
self.approve_fixture(rd)
return rd
@staticmethod
def approve_fixture(rd):
"""Synthetic reviewer receipt for isolated export tests, never real evidence."""
from export import content_review as cr, trajectory
from harness.verify import verify_rollout
bound = cr.binding(rd, verify_rollout(rd, write=False), trajectory.freeze(rd), ex._gate_sources(rd))
path = cr.receipt_path(rd, bound)
path.parent.mkdir(exist_ok=True)
path.write_text(json.dumps({"schema": "bench.content-review/1", "binding": bound,
"state": "COMPLETE", "decision": "PASS", "reason": "unit test fixture only",
"issues": [], "claims": [{"quote": "fixture", "evidence": "fixture", "assessment": "SUPPORTED", "reason": "fixture"}], "model": "synthetic-test-reviewer"}))
@staticmethod
def context_header():
return {"type": "session", "version": 3, "id": "session-fixture", "bench": {
"runner_version": "pi-sdk-v1", "execution_id": "exec-fixture",
"system_prompt": "Use tools and report measured results.",
"tools": [{"name": "bash", "parameters": {"type": "object"}}]}}
def test_legacy_append_context_is_retained_not_sft(self):
rd = self.add()
metadata = json.loads((rd / "rollout.json").read_text())
metadata["runner_version"] = "legacy-pi"
(rd / "rollout.json").write_text(json.dumps(metadata))
path = rd / "agent/events.jsonl"
events = [json.loads(line) for line in path.read_text().splitlines()]
events[0].pop("bench")
path.write_text("\n".join(map(json.dumps, events)))
stats = ex.materialize(["r"], "legacy-context")["stats"]
self.assertEqual(stats["sft_eligible"], 0)
record = json.loads((self.root / "datasets/legacy-context/retained.jsonl").read_text())
self.assertEqual(record["system_source"], "legacy-append-only")
self.assertEqual(len(record["messages"]), 5)
self.assertFalse(record["trajectory"]["validation"]["checks"]["REQUIRED_CONTEXT"]["pass"])
def test_active_db_state_cannot_use_stale_done_file(self):
self.add()
for index, state in enumerate(('PAUSED_INFRA', 'CANCEL_REQUESTED', 'CLEANING', 'PENDING')):
with self.subTest(state=state):
self.con.execute('UPDATE rollouts SET state=?', (state,)); self.con.commit()
stats = ex.materialize(['r'], f'active-{index}')['stats']
self.assertEqual((stats['scanned'], stats['sft_eligible']), (0, 0))
def test_invalid_stage_json_retains_failure_without_blocking_other_rollouts(self):
bad = self.add(kk=0)
self.add(kk=1)
malformed = bad / 'stages/06-hbm/a01/meta.json'
malformed.parent.mkdir()
malformed.write_bytes(b'')
result = ex.materialize(['r'], 'corrupt-stage')
self.assertEqual((result['stats']['scanned'], result['stats']['sft_eligible']), (2, 1))
output = self.root / 'datasets/corrupt-stage'
records = [json.loads(line) for line in (output / 'disposition.jsonl').read_text().splitlines()]
self.assertEqual(records[0]['reasons'], ['reverification_failed:JSONDecodeError'])
self.assertEqual(records[1]['disposition'], 'accepted')
evidence = [json.loads(line) for line in (output / 'source-manifest.jsonl').read_text().splitlines()]
self.assertEqual(evidence[0]['errors'], ['HARD_GATES_UNAVAILABLE:JSONDecodeError'])
self.assertEqual(malformed.read_bytes(), b'')
self.assertEqual(json.loads((output / 'sft.jsonl').read_text())['rollout'], 1)
def test_db_reactivation_during_build_fails_closed(self):
self.add()
reconstruct = ex.preserved.reconstruct
def activate(source):
result = reconstruct(source)
self.con.execute("UPDATE rollouts SET state='PAUSED_INFRA'"); self.con.commit()
return result
with patch.object(ex.preserved, 'reconstruct', side_effect=activate):
self.assertEqual(ex.materialize(['r'], 'changed-state')['stats']['sft_eligible'], 0)
def test_matching_invalid_eval_identities_are_not_evidence(self):
rd = self.add()
job_path, meta_path = rd.parent.parent / 'job.json', rd / 'stages/08-eval/meta.json'
original_job, original_meta = job_path.read_text(), meta_path.read_text()
invalid = [('dataset', ''), ('evaluator_version', ''), ('content_sha', ''),
('manifest_sha', 'not-a-hash'), ('n', 0), ('n', True)]
for index, (field, value) in enumerate(invalid):
with self.subTest(field=field, value=value):
job, meta = json.loads(original_job), json.loads(original_meta)
if field == 'dataset': job['data'][field] = value
elif field == 'evaluator_version': job[field] = value
else: job['data']['t1'][field] = value
meta['measurements']['G3']['eval_set'][field] = value
if field == 'n': meta['measurements']['G3']['n'] = value
job_path.write_text(json.dumps(job)); meta_path.write_text(json.dumps(meta))
self.assertEqual(ex.materialize(['r'], f'invalid-id-{index}')['stats']['sft_eligible'], 0)
def test_completed_tool_tail_is_preserved(self):
msgs, stats = ex.clean(dialogue())
self.assertEqual(msgs[-1]["role"], "tool")
self.assertEqual(len(msgs), 3)
self.assertTrue(stats["tool_pairing"]["valid"])
def test_retry_export_retains_completed_prefix(self):
rd = self.add()
first = dialogue()
last = {"role": "assistant", "content": "finished after provider retry"}
events = [{"type": "agent_start"}]
events += [{"type": "message_end", "message": m} for m in first]
events += [{"type": "agent_end", "messages": first}, {"type": "agent_start"},
{"type": "message_end", "message": last}, {"type": "agent_end", "messages": [last]}]
(rd / "agent/events.jsonl").write_text("\n".join(map(json.dumps, events)))
raw, _ = ex.load_trajectory(rd)
self.assertEqual(raw, first + [last])
def test_dangling_and_orphan_tool_evidence_is_labeled(self):
msgs, stats = ex.clean(dialogue()[:-1])
self.assertEqual(len(msgs), 2)
self.assertEqual(stats["tool_pairing"]["missing_results"], 1)
_, stats = ex.clean([dialogue()[0], dialogue()[2]])
self.assertEqual(stats["tool_pairing"]["orphan_results"], 1)
def test_timeout_keeps_verified_stage_rewards(self):
rd = self.add(execution="TIMEOUT")
reward = compute(json.loads((rd / "verdict.json").read_text()), 60, 100)
self.assertEqual(reward["recipe"], "v0.8")
self.assertEqual(reward["terms"]["G3"], 2.0)
self.assertEqual(reward["terms"]["timeout"], -1.0)
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
self.assertEqual(manifest["stats"]["by_march"], {"nash-m": 1})
self.assertEqual(manifest["stats"]["sft_eligible_by_march"], {"nash-m": 1})
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertEqual(rec["board_type"], "s100p")
self.assertEqual(rec["verdict"]["terminal"], "PASS")
self.assertEqual(rec["execution"]["terminal"], "TIMEOUT")
def test_all_retains_timeout_failure_and_partial_without_inventing_messages(self):
self.add(terminal="TIMEOUT", execution="TIMEOUT", ended=False)
self.add(kk=1, terminal="FAIL")
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["written"], 2)
self.assertEqual(manifest["stats"]["partial"], 1)
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
recs = [json.loads(line) for line in (self.root / "datasets/data/teacher.jsonl").read_text().splitlines()]
self.assertEqual(recs[0]["trajectory"]["source"], "message_end")
self.assertEqual(sum(m["role"] == "assistant" for m in recs[0]["messages"]), 2)
def legacy_evidence(self, rd, *, mode="e2e", claim="PASS"):
original = json.loads((rd / "verdict.json").read_text())
original.update(schema="bench.verdict/2", review=True,
oracle_ref={"terminal": "FAIL", "G3_delta": -99})
(rd / "verdict.json").write_text(json.dumps(original))
(rd / "claim.json").write_text(json.dumps({"terminal": claim}))
hbm = rd / "artifacts/j.hbm"
hbm.parent.mkdir(exist_ok=True)
hbm.write_bytes(b"fixture HBM identity; never loaded on a board")
digest = hashlib.sha256(hbm.read_bytes()).hexdigest()
for stage, measurements in {
"06-hbm": {"G1": {"hbm_exists": True, "march_ok": True, "io_ok": True,
"verifier_ok": True, "log_has_error": False, "hbm_sha256": digest}},
"08-eval": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1",
"hbm_sha256": digest,
"eval_set": eval_set(json.loads((rd.parent.parent / "job.json").read_text()))}},
}.items():
sd = rd / "stages" / stage
sd.mkdir(parents=True, exist_ok=True)
(sd / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements}))
self.approve_fixture(rd)
return original
def test_legacy_missing_hbm_identity_does_not_become_sft_evidence(self):
rd = self.add(review=True)
self.legacy_evidence(rd)
path = rd / "stages/08-eval/meta.json"
metadata = json.loads(path.read_text())
del metadata["measurements"]["G3"]["hbm_sha256"]
path.write_text(json.dumps(metadata))
before = {p: p.read_bytes() for p in rd.rglob("*") if p.is_file()}
stats = ex.materialize(["r"], "sft", purpose="sft")["stats"]
self.assertEqual(stats["written"], 0)
self.assertEqual(before, {p: p.read_bytes() for p in before})
def test_legacy_oracle_review_is_reverified_without_modifying_sources(self):
rd = self.add(review=True)
self.legacy_evidence(rd)
before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in rd.rglob("*") if p.is_file()}
before_db = list(self.con.execute("SELECT * FROM rollouts"))
manifest = ex.materialize(["r"], "data", purpose="sft")
self.assertEqual(manifest["stats"]["reverified"], 1)
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertTrue(rec["sft_eligible"])
self.assertTrue(rec["training_uses"]["reward_learning_candidate"])
self.assertEqual(rec["verification"]["method"], "readonly-reverification")
self.assertEqual(rec["verdict"]["schema"], VERDICT_SCHEMA)
self.assertFalse(rec["verdict"]["review"])
self.assertNotIn("oracle_ref", rec)
self.assertNotIn("history", rec)
self.assertTrue(rec["provenance"]["original_verdict"]["review"])
self.assertEqual(rec["provenance"]["original_verdict"]["sha256"], before[rd / "verdict.json"])
self.assertEqual(before, {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in before})
self.assertEqual(before_db, list(self.con.execute("SELECT * FROM rollouts")))
def test_failed_reverification_is_excluded_without_legacy_fallback(self):
rd = self.add(review=True)
self.legacy_evidence(rd)
(rd / "stages/08-eval/meta.json").write_text("broken evidence")
stats = ex.materialize(["r"], "data")["stats"]
self.assertEqual(stats["written"], 0)
self.assertEqual(stats["dropped"], {"reverification_failed:JSONDecodeError": 1})
def test_historical_repair_remains_analysis_only(self):
rd = self.add(mode="repair")
self.legacy_evidence(rd)
ex.materialize(["r"], "all")
rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text())
self.assertEqual(rec["mode"], "repair")
self.assertFalse(rec["sft_eligible"])
self.assertEqual(rec["sft_reason"], "historical-repair-context")
self.assertFalse(rec["training_uses"]["reward_learning_candidate"])
self.assertEqual(ex.materialize(["r"], "sft", purpose="sft")["stats"]["written"], 0)
self.assertEqual(ex.materialize(["r"], "rl", purpose="rl")["stats"]["written"], 0)
def test_blocked_self_report_cannot_override_measured_failure(self):
rd = self.add(terminal="BLOCKED")
v = json.loads((rd / "verdict.json").read_text())
v.update(blocked_correct=True, oracle_ref={"terminal": "FAIL"})
(rd / "verdict.json").write_text(json.dumps(v))
ex.materialize(["r"], "all")
rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text())
self.assertFalse(rec["sft_eligible"])
self.assertEqual(rec["verdict"]["terminal"], "FAIL")
self.assertTrue(rec["training_uses"]["reward_learning_candidate"])
self.assertNotIn("blocked", rec["reward_terms"])
def test_oracle_independence_does_not_bypass_transcript_or_annotation_checks(self):
first = self.add(ended=False)
self.legacy_evidence(first)
second = self.add(kk=1)
self.legacy_evidence(second)
(second / "claim.json").unlink()
third = self.add(kk=2)
self.legacy_evidence(third)
self.con.execute("INSERT INTO annotations(run_id,job_id,note) VALUES('r','j','rollout#2 contaminated: copied solution')")
self.con.commit()
stats = ex.materialize(["r"], "data", purpose="sft")["stats"]
self.assertEqual(stats["written"], 0)
self.assertEqual(stats["dropped"], {"sft:partial-transcript": 1, "sft:content-review-unreviewed": 1,
"contaminated": 1})
def test_old_version_and_missing_verdict_have_explicit_exclusions(self):
self.add(version=5)
rd = self.add(kk=1)
(rd / "verdict.json").unlink()
stats = ex.materialize(["r"], "data")["stats"]
self.assertEqual((stats["scanned"], stats["excluded"], stats["written"]), (2, 2, 0))
exclusions = (self.root / "datasets/data/exclusions.jsonl").read_text()
self.assertIn("old_version", exclusions)
self.assertIn("missing_verdict", exclusions)
def test_sft_and_rl_exports_do_not_read_historical_pipeline(self):
self.add()
with patch.object(ex, "pipeline_records", side_effect=OSError("unreadable old pipeline")) as pipeline:
for purpose in ("sft", "rl"):
with self.subTest(purpose=purpose):
result = ex.materialize(["r"], purpose, purpose=purpose)
self.assertEqual(result["stats"]["agent_written"], 1)
self.assertEqual(result["stats"]["pipeline_written"], 0)
self.assertEqual((self.root / "datasets" / purpose / "pipeline.jsonl").read_text(), "")
pipeline.assert_not_called()
def test_pipeline_is_real_separate_and_failed_jobs_are_retained(self):
self.add()
jd = self.root / "runs/r/j"
sd = jd / "oracle/stages/05-compile/a00"
sd.mkdir(parents=True)
(sd / "meta.json").write_text(json.dumps({"cmd": ["hb_compile", "--config", "compile.yaml"], "state": "FAIL", "error_class": "unsupported"}))
(jd / "oracle/result.json").write_text('{"terminal":"FAIL","error_class":"unsupported"}')
self.con.execute("INSERT INTO jobs(run_id,job_id,mode,state,finished_at) VALUES('r','j','oracle','FAIL',123)")
self.con.execute("INSERT INTO stages(run_id,job_id,rollout,stage,state) VALUES('r','j',-1,'05-compile','FAIL')")
self.con.commit()
stats = ex.materialize(["r"], "data")["stats"]
self.assertEqual((stats["agent_written"], stats["pipeline_written"]), (1, 1))
rec = json.loads((self.root / "datasets/data/pipeline.jsonl").read_text())
self.assertEqual(rec["source"], "deterministic_pipeline")
self.assertNotIn("messages", rec)
self.assertEqual(rec["stages"][0]["command"][0], "hb_compile")
def test_refresh_is_explicit_stable_and_hash_checked(self):
self.add()
ex.materialize(["r"], "data")
out = self.root / "datasets/data"
first = (out / "teacher.jsonl").read_bytes()
with self.assertRaises(FileExistsError):
ex.materialize(["r"], "data")
manifest = ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
self.assertEqual(manifest["files"]["teacher.jsonl"]["sha256"], hashlib.sha256(first).hexdigest())
self.assertFalse(list(out.glob(".refresh-*")))
with patch.object(ex, "_write_dataset", side_effect=RuntimeError("interrupted")):
with self.assertRaises(RuntimeError):
ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
self.assertFalse(list(out.glob(".refresh-*")))
def test_refresh_preserves_filters_and_pi_threshold_after_version_bump(self):
self.add()
first = ex.materialize(["r"], "data", purpose="sft", policy="teacher", min_reward=1,
max_turns=70, max_tokens=500)
(self.root / "agent/VERSION").write_text("100")
second = ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first["filters"], second["filters"])
self.assertEqual(second["stats"]["written"], 1)
self.assertEqual(second["filter_history"], [])
changed = ex.materialize(["r"], "data", refresh=True, purpose="all", policy=None)
self.assertEqual(changed["filter_history"][-1]["explicit_fields"], ["policy", "purpose"])
self.assertEqual(changed["filter_history"][-1]["reason"], "explicit_filter_change")
self.assertEqual(changed["filters"]["min_agent_version"], 6)
def test_exact_version_refresh_and_explicit_minimum_are_audited(self):
self.add(version=5)
first = ex.materialize(["r"], "data", agent_version="5")
(self.root / "agent/VERSION").write_text("100")
second = ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first["filters"], second["filters"])
self.assertEqual(second["stats"]["written"], 1)
third = ex.materialize(["r"], "data", refresh=True, agent_version=None, min_agent_version=10)
self.assertEqual(third["stats"]["written"], 0)
self.assertEqual(third["filter_history"][-1]["explicit_fields"], ["agent_version", "min_agent_version"])
def test_default_pi_version_does_not_filter_other_runner(self):
rd = self.add(version=1)
self.opencode(rd)
first = ex.materialize(["r"], "data")
self.assertEqual(first["stats"]["written"], 1)
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertEqual(rec["runner"], "opencode")
explicit = ex.materialize(["r"], "explicit", min_agent_version=6)
self.assertEqual(explicit["stats"]["written"], 0)
def opencode(self, rd):
from export.test_opencode_source import fixture
fixture(rd)
self.approve_fixture(rd)
def test_opencode_exports_actual_context_and_retains_runner_identity(self):
rd = self.add()
self.opencode(rd)
result = ex.materialize(["r"], "data")
self.assertEqual(result["stats"]["sft_eligible"], 1)
record = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertEqual(record["runner_version"], "1.18.31")
self.assertEqual(record["system_source"], "runner-raw-system")
self.assertEqual(record["messages"][0]["content"], "Actual runner system\n\nActual harness instruction")
self.assertEqual(record["trajectory"]["system_segments"], ["Actual runner system", "Actual harness instruction"])
self.assertEqual(record["trajectory"]["source"], "opencode-raw-session")
self.assertTrue(record["trajectory"]["validation"]["pass"])
self.assertTrue(ex.validate_exported_record(self.root, record)["pass"])
from export.publication import build
rules = Path(__file__).resolve().parents[1] / "docs/plans/sft-collection-v1/rules/sft-preserve-v1.json"
sealed = build(self.root / "datasets/data", self.root / "published", rules, self.root)
self.assertTrue(sealed["dataset_version"])
def test_opencode_missing_system_or_tools_is_not_sft_and_never_pi_redumps(self):
rd = self.add()
self.opencode(rd)
(rd / "agent/raw-system.json").unlink()
self.assertEqual(ex.system_prompt(rd), ("", "missing"))
ex.materialize(["r"], "no-system")
record = json.loads((self.root / "datasets/no-system/teacher.jsonl").read_text())
self.assertFalse(record["sft_eligible"])
self.assertFalse(record["trajectory"]["validation"]["checks"]["REQUIRED_CONTEXT"]["pass"])
(rd / "agent/tools.json").unlink()
(rd / "agent/cmd.json").write_text('{"cmd":["pi","--tools","bash"]}')
with patch("subprocess.run", side_effect=AssertionError("must not redump Pi")):
self.assertEqual(ex.tool_defs(rd), ([], "missing"))
def test_opencode_still_requires_content_review_and_hard_gates(self):
rd = self.add()
self.opencode(rd)
for p in (rd / "content-reviews").glob("*.json"):
p.unlink()
first = ex.materialize(["r"], "no-review")
self.assertEqual(first["stats"]["sft_eligible"], 0)
record = json.loads((self.root / "datasets/no-review/teacher.jsonl").read_text())
self.assertFalse(record["content_review"]["pass"])
self.approve_fixture(rd)
(rd / "stages/08-eval/meta.json").unlink()
second = ex.materialize(["r"], "no-measurements")
self.assertEqual(second["stats"]["sft_eligible"], 0)
def test_current_verdict_cannot_bypass_hard_gate_reverification(self):
for i, mutation in enumerate(("hash", "source", "threshold", "missing")):
with self.subTest(mutation=mutation):
rd = self.add(kk=i)
path = rd / "stages/08-eval/meta.json"
meta = json.loads(path.read_text())
if mutation == "hash":
meta["measurements"]["G3"]["hbm_sha256"] = "0" * 64
elif mutation == "source":
meta["measurements"]["G3"]["source"] = "simulator"
elif mutation == "threshold":
meta["measurements"]["G3"]["quant"] = 10
else:
meta["measurements"] = {}
path.write_text(json.dumps(meta))
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
self.assertEqual(manifest["stats"]["written"], 4)
def test_review_nested_secret_and_private_identity_never_publish(self):
from export.test_trajectory import segment, message, call, result
nested = "REVIEW_TOKEN_DO_NOT_PUBLISH"
secret_id = "sk-REVIEWFAKE01234567890123"
rd = self.add()
msgs = dialogue() + [{"role": "assistant", "content": "Measured subset complete."}]
msgs[1]["content"][0]["arguments"]["authorization"] = {"Bearer": [nested, {"nested": nested}]}
(rd / "agent/events.jsonl").write_text("\n".join(map(json.dumps, [self.context_header()] + segment(msgs))))
self.approve_fixture(rd)
private = self.add(kk=1)
events = segment([message("user", "task"), call(secret_id), result(secret_id)]) + [{"type": secret_id}]
(private / "agent/events.jsonl").write_text("\n".join(map(json.dumps, events)))
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
for path in (self.root / "datasets/data").glob("*.jsonl"):
self.assertNotIn(nested, path.read_text(), path.name)
self.assertNotIn(secret_id, path.read_text(), path.name)
def test_review_survives_job_json_edit_that_only_adds_gates(self):
rd = self.add()
job_path = rd.parent.parent / "job.json"
job = json.loads(job_path.read_text())
job["gates"]["G1"]["require_bpu_compute"] = True
job_path.write_text(json.dumps(job))
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
record = json.loads((self.root / "datasets/data/sft.jsonl").read_text())
self.assertIn("carried_from_binding_sha256", record["content_review"])
meta_path = rd / "stages/08-eval/meta.json"
meta = json.loads(meta_path.read_text())
meta["measurements"]["G3"]["quant"] = 79.0 # still within tolerance, but not what was reviewed
meta_path.write_text(json.dumps(meta))
self.assertEqual(ex.materialize(["r"], "data2")["stats"]["sft_eligible"], 0)
def test_gate_only_change_allows_new_judgments_only(self):
from copy import deepcopy
from export.content_review import gate_only_change
old = {"rule": "r", "gate_manifest": {"job.json": {"sha256": "a"}, "rollout.json": {"sha256": "c"}},
"authoritative_result": {"terminal": "PASS", "required": ["G1", "G3"],
"gates": {"G3": {"verdict": "pass", "quant": 79.2},
"G4": {"verdict": "na", "budget_ms": None}}}}
new = deepcopy(old)
new["gate_manifest"]["job.json"] = {"sha256": "b"}
new["authoritative_result"]["required"].append("G4")
new["authoritative_result"]["gates"]["G4"] = {"verdict": "pass", "budget_ms": 3.7}
self.assertTrue(gate_only_change(old, new))
for path, value in ((("gates", "G3", "verdict"), "pass-screen"), (("terminal",), "FAIL"),
(("required",), ["G1", "G4"])):
changed = deepcopy(new)
target = changed["authoritative_result"]
for key in path[:-1]:
target = target[key]
target[path[-1]] = value
self.assertFalse(gate_only_change(old, changed), path)
changed = deepcopy(new)
changed["gate_manifest"]["rollout.json"] = {"sha256": "d"}
self.assertFalse(gate_only_change(old, changed))
self.assertFalse(gate_only_change(None, new))
def test_review_unknown_execution_state_is_not_stable(self):
for i, state in enumerate(("PENDING", "future-terminal", None)):
rd = self.add(kk=i)
path = rd / "rollout.json"
metadata = json.loads(path.read_text())
metadata["state"] = state
path.write_text(json.dumps(metadata))
self.assertEqual(ex.materialize(["r"], "data")["stats"]["sft_eligible"], 0)
def test_review_source_change_after_validation_is_not_accepted(self):
from export import trajectory as tr
rd = self.add()
validate = tr.validate
def mutate(source, output):
checked = validate(source, output)
with (rd / "agent/events.jsonl").open("a") as stream:
stream.write('\n{"type":"message_end","message":{"role":"assistant","content":"later"}}')
return checked
with patch.object(tr, "validate", side_effect=mutate):
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
record = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertIn("SOURCE_CHANGED", record["trajectory"]["validation"]["checks"]["SOURCE_INTEGRITY"]["errors"])
def test_review_g3_identity_and_unwidened_tolerance(self):
for i, mutation in enumerate(("dataset", "content_sha", "manifest_sha", "eval_set_id", "evaluator_version", "missing", "se")):
rd = self.add(kk=i)
path = rd / "stages/08-eval/meta.json"
meta = json.loads(path.read_text())
values = meta["measurements"]["G3"]
if mutation == "missing":
values.pop("eval_set", None)
elif mutation == "se":
values.update(quant=75.2, se=1)
else:
values.setdefault("eval_set", {})[mutation] = "wrong"
path.write_text(json.dumps(meta))
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
def test_review_data010_runs_in_materialization_without_truncation(self):
from export.test_trajectory import segment, message, call, result
rd = self.add()
full = json.dumps({"padding": "p" * 6100, "critical_metric": {"score": 0.9123}, "tail": "t" * 2500})
msgs = [message("user", "task"), call("one"), result("one", full),
message("assistant", "Use `/critical_metric/score` from the result.")]
(rd / "agent/events.jsonl").write_text("\n".join(map(json.dumps, [self.context_header()] + segment(msgs))))
self.approve_fixture(rd)
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
out = self.root / "datasets/data"
annotations = [json.loads(line) for line in (out / "annotations.jsonl").read_text().splitlines()]
critical = [a for a in annotations if a.get("pointer") == "/critical_metric/score"]
self.assertEqual(critical[0]["tag"], "REFERENCED_CONTENT_REMOVED")
self.assertEqual(critical[0]["comparison_source"], "legacy-clean-replay")
self.assertEqual(json.loads((out / "sft.jsonl").read_text())["messages"][3]["content"][0]["text"], full)
def test_new_eval_receipt_binds_actual_package_and_evaluator(self):
import io
import tarfile
from harness.stages.s08_eval import Eval
rd = self.add()
job = json.loads((rd.parent.parent / "job.json").read_text())
package = self.root / "test-eval.tar"
manifest = {"dataset": "fixture", "sha256": "a" * 64,
"items": [{"id": str(i), "file": str(i)} for i in range(100)]}
blob = json.dumps(manifest).encode()
with tarfile.open(package, "w") as archive:
info = tarfile.TarInfo("manifest.json"); info.size = len(blob)
archive.addfile(info, io.BytesIO(blob))
job["data"]["t1"].update(path=str(package), manifest_sha=hashlib.sha256(package.read_bytes()).hexdigest())
stage_dir = rd / "stages/08-eval"
measured = {"G3": {"n": 100}, "evaluator_version": "fixture-v1", "t1_manifest_sha": "a" * 64,
"n": 100, "task": "cls", "model": "resnet18"}
path = stage_dir / "eval.json"
path.write_text(json.dumps(measured))
ctx, outcome = SimpleNamespace(job=job), SimpleNamespace(meta={})
self.assertEqual(Eval().measure(ctx, stage_dir, outcome)["G3"]["eval_set"], eval_set(job))
measured["evaluator_version"] = "unrelated-evaluator"
path.write_text(json.dumps(measured))
with self.assertRaisesRegex(ValueError, "evaluated set identity"):
Eval().measure(ctx, stage_dir, outcome)
def test_published_sidecars_and_source_validator_cover_each_attempt(self):
self.add()
self.add(kk=1, terminal="FAIL")
self.add(kk=2, version=1)
manifest = ex.materialize(["r"], "data")
out = self.root / "datasets/data"
rows = lambda name: [json.loads(line) for line in (out / (name + ".jsonl")).read_text().splitlines()]
self.assertEqual(len(rows("disposition")), 3)
self.assertEqual(len(rows("source-manifest")), 3)
self.assertEqual(len(rows("sft")), 1)
for name, info in manifest["files"].items():
self.assertEqual(hashlib.sha256((out / name).read_bytes()).hexdigest(), info["sha256"])
record = rows("sft")[0]
lineage = [row for row in rows("lineage") if row["rollout"] == 0]
excluded = [row for row in rows("excluded_observations") if row["rollout"] == 0]
self.assertTrue(ex.validate_exported_record(self.root, record, lineage, excluded)["pass"])
record["messages"].pop()
self.assertFalse(ex.validate_exported_record(self.root, record, lineage, excluded)["pass"])
def test_bad_runner_session_is_retained_but_not_sft(self):
rd = self.add()
self.opencode(rd)
(rd / "agent/session-id.json").write_text('{"session_id":"wrong"}')
result = ex.materialize(["r"], "data")
self.assertEqual(result["stats"]["sft_eligible"], 0)
record = json.loads((self.root / "datasets/data/retained.jsonl").read_text())
self.assertIn("IDENTITY_PAYLOAD_CONFLICT", record["trajectory"]["validation"]["checks"]["SOURCE_INTEGRITY"]["errors"])
def test_malformed_opencode_session_does_not_abort_dataset(self):
rd = self.add()
self.opencode(rd)
(rd / "agent/raw-session.json").write_text('{"messages":[null]}')
self.add(kk=1)
result = ex.materialize(["r"], "data")
self.assertEqual(result["stats"]["sft_eligible"], 1)
self.assertEqual(result["stats"]["dropped"], {"invalid_trajectory:ValueError": 1})
def test_explicit_and_frozen_minimum_do_not_read_current_version_file(self):
self.add()
(self.root / "agent/VERSION").unlink()
first = ex.materialize(["r"], "data", min_agent_version=1)
(self.root / "agent/VERSION").write_text("broken")
refreshed = ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first["filters"], refreshed["filters"])
self.assertEqual(refreshed["stats"]["written"], 1)
def test_claim_is_not_an_outcome_gate(self):
self.assertTrue(ex.sft_eligibility({"terminal": "PASS"}, 4.0)[0])
self.assertTrue(ex.sft_eligibility({"terminal": "PASS", "claim_consistent": True}, 4.0)[0])
def test_redump_uses_configured_node_and_path(self):
rd = self.add()
(rd / "agent/tools.json").unlink()
(rd / "agent/cmd.json").write_text('{"cmd":["pi","--tools","bash"]}')
(self.root / "agent/dump_tools.js").write_text("// test")
with patch("harness.paths.config", return_value={"paths": {"node_bin": "/opt/bench-node/bin/node"}}), \
patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout='[{"name":"bash"}]')) as run:
definitions, source = ex.tool_defs(rd)
self.assertEqual(source, "redumped")
self.assertEqual(definitions[0]["name"], "bash")
self.assertEqual(run.call_args.args[0][0], "/opt/bench-node/bin/node")
self.assertTrue(run.call_args.kwargs["env"]["PATH"].startswith("/opt/bench-node/bin:"))
if __name__ == "__main__":
unittest.main()