data-gen-bpu/export/test_materialization.py

258 lines
14 KiB
Python

"""Training materialization regressions: evidence retention, pairing and refresh."""
import hashlib
import json
import sqlite3
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from export import to_agentrl as ex
from export.reward import compute
from harness.db import SCHEMA
from harness.verify import VERDICT_SCHEMA, VERIFICATION_RULE
def dialogue():
return [{"role": "user", "content": "Compile and evaluate."},
{"role": "assistant", "content": [{"type": "toolCall", "id": "t1", "name": "bash", "arguments": {"command": "bench stage status h"}}]},
{"role": "toolResult", "toolCallId": "t1", "content": [{"type": "text", "text": "DONE"}]}]
class MaterializationTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
(self.root / "agent").mkdir()
(self.root / "agent/VERSION").write_text("6")
(self.root / "state").mkdir()
self.con = sqlite3.connect(self.root / "state/jobs.sqlite")
self.con.executescript(SCHEMA)
self.patch = patch.object(ex, "BENCH", self.root)
self.patch.start()
self.env = patch.dict("os.environ", {"BENCH_STATE_DIR": str(self.root / "state")})
self.env.start()
def tearDown(self):
self.env.stop(); self.patch.stop(); self.con.close(); self.tmp.cleanup()
def add(self, kk=0, terminal="PASS", execution="DONE", review=False, ended=True, version=6, mode="e2e"):
jd = self.root / "runs/r/j"
rd = jd / "rollouts" / f"{kk:02d}"
(rd / "agent").mkdir(parents=True)
job = {"run_id": "r", "job_id": "j", "task": "cls", "model": "resnet18", "march": "nash-m", "backend": "ptq",
"gates": {"required": ["G1", "G3"], "G1": {}, "G2": {},
"G3": {"kind": "rel_pct", "tol": 5, "require_source": "board"}, "G4": {}}}
(jd / "job.json").write_text(json.dumps(job))
(rd / "rollout.json").write_text(json.dumps({"state": execution, "model": "glm-5.3", "policy": "teacher", "mode": mode,
"agent_config_version": version, "max_turns": 60, "max_tokens": 100}))
verdict = {"schema": VERDICT_SCHEMA, "verification_rule": VERIFICATION_RULE, "mode": mode, "run_id": "r", "job_id": "j", "rollout": kk, "terminal": terminal,
"execution_terminal": execution, "timeout_reason": "budget" if execution == "TIMEOUT" else None,
"claim_consistent": True, "claim": {"terminal": terminal}, "review": review,
"gates": {"G1": {"verdict": "pass"}, "G2": {"verdict": "pass"}, "G3": {"verdict": "pass-screen"}},
"cost": {"turns": 30, "tokens": 50}}
(rd / "verdict.json").write_text(json.dumps(verdict))
events = ([{"type": "agent_end", "messages": dialogue()}] if ended else
[{"type": "message_end", "message": message} for message in dialogue()])
(rd / "agent/events.jsonl").write_text("\n".join(json.dumps(e) for e in events))
(rd / "agent/system.md").write_text("Use tools and report measured results.")
(rd / "agent/tools.json").write_text('[{"name":"bash","parameters":{"type":"object"}}]')
self.con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','j',?,?)", (kk, terminal))
self.con.commit()
return rd
def test_completed_tool_tail_is_preserved(self):
msgs, stats = ex.clean(dialogue())
self.assertEqual(msgs[-1]["role"], "tool")
self.assertEqual(len(msgs), 3)
self.assertTrue(stats["tool_pairing"]["valid"])
def test_dangling_and_orphan_tool_evidence_is_labeled(self):
msgs, stats = ex.clean(dialogue()[:-1])
self.assertEqual(len(msgs), 2)
self.assertEqual(stats["tool_pairing"]["missing_results"], 1)
_, stats = ex.clean([dialogue()[0], dialogue()[2]])
self.assertEqual(stats["tool_pairing"]["orphan_results"], 1)
def test_timeout_keeps_verified_stage_rewards(self):
rd = self.add(execution="TIMEOUT")
reward = compute(json.loads((rd / "verdict.json").read_text()), 60, 100)
self.assertEqual(reward["recipe"], "v0.6")
self.assertEqual(reward["terms"]["G3"], 2.0)
self.assertEqual(reward["terms"]["timeout"], -1.0)
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertEqual(rec["verdict"]["terminal"], "PASS")
self.assertEqual(rec["execution"]["terminal"], "TIMEOUT")
def test_all_retains_timeout_failure_and_partial_without_inventing_messages(self):
self.add(terminal="TIMEOUT", execution="TIMEOUT", ended=False)
self.add(kk=1, terminal="FAIL")
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["written"], 2)
self.assertEqual(manifest["stats"]["partial"], 1)
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
recs = [json.loads(line) for line in (self.root / "datasets/data/teacher.jsonl").read_text().splitlines()]
self.assertEqual(recs[0]["trajectory"]["source"], "message_end")
self.assertEqual(sum(m["role"] == "assistant" for m in recs[0]["messages"]), 1)
def legacy_evidence(self, rd, *, mode="e2e", claim="PASS"):
original = json.loads((rd / "verdict.json").read_text())
original.update(schema="bench.verdict/2", review=True,
oracle_ref={"terminal": "FAIL", "G3_delta": -99})
(rd / "verdict.json").write_text(json.dumps(original))
(rd / "claim.json").write_text(json.dumps({"terminal": claim}))
for stage, measurements in {
"06-hbm": {"G1": {"hbm_exists": True, "march_ok": True, "io_ok": True,
"verifier_ok": True, "log_has_error": False}},
"08-eval": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}},
}.items():
sd = rd / "stages" / stage
sd.mkdir(parents=True, exist_ok=True)
(sd / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements}))
return original
def test_legacy_oracle_review_is_reverified_without_modifying_sources(self):
rd = self.add(review=True)
self.legacy_evidence(rd)
before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in rd.rglob("*") if p.is_file()}
before_db = list(self.con.execute("SELECT * FROM rollouts"))
manifest = ex.materialize(["r"], "data", purpose="sft")
self.assertEqual(manifest["stats"]["reverified"], 1)
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertTrue(rec["sft_eligible"])
self.assertTrue(rec["training_uses"]["reward_learning_candidate"])
self.assertEqual(rec["verification"]["method"], "readonly-reverification")
self.assertEqual(rec["verdict"]["schema"], VERDICT_SCHEMA)
self.assertFalse(rec["verdict"]["review"])
self.assertNotIn("oracle_ref", rec)
self.assertNotIn("history", rec)
self.assertTrue(rec["provenance"]["original_verdict"]["review"])
self.assertEqual(rec["provenance"]["original_verdict"]["sha256"], before[rd / "verdict.json"])
self.assertEqual(before, {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in before})
self.assertEqual(before_db, list(self.con.execute("SELECT * FROM rollouts")))
def test_failed_reverification_is_excluded_without_legacy_fallback(self):
rd = self.add(review=True)
self.legacy_evidence(rd)
(rd / "stages/08-eval/meta.json").write_text("broken evidence")
stats = ex.materialize(["r"], "data")["stats"]
self.assertEqual(stats["written"], 0)
self.assertEqual(stats["dropped"], {"reverification_failed:JSONDecodeError": 1})
def test_historical_repair_remains_analysis_only(self):
rd = self.add(mode="repair")
self.legacy_evidence(rd)
ex.materialize(["r"], "all")
rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text())
self.assertEqual(rec["mode"], "repair")
self.assertFalse(rec["sft_eligible"])
self.assertEqual(rec["sft_reason"], "historical-repair-context")
self.assertFalse(rec["training_uses"]["reward_learning_candidate"])
self.assertEqual(ex.materialize(["r"], "sft", purpose="sft")["stats"]["written"], 0)
self.assertEqual(ex.materialize(["r"], "rl", purpose="rl")["stats"]["written"], 0)
def test_blocked_is_not_a_success_or_reward_training_target(self):
rd = self.add(terminal="BLOCKED")
v = json.loads((rd / "verdict.json").read_text())
v.update(blocked_correct=True, oracle_ref={"terminal": "FAIL"})
(rd / "verdict.json").write_text(json.dumps(v))
ex.materialize(["r"], "all")
rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text())
self.assertFalse(rec["sft_eligible"])
self.assertFalse(rec["training_uses"]["reward_learning_candidate"])
self.assertNotIn("blocked", rec["reward_terms"])
def test_oracle_independence_does_not_bypass_transcript_or_annotation_checks(self):
first = self.add(ended=False)
self.legacy_evidence(first)
second = self.add(kk=1)
self.legacy_evidence(second)
(second / "claim.json").unlink()
third = self.add(kk=2)
self.legacy_evidence(third)
self.con.execute("INSERT INTO annotations(run_id,job_id,note) VALUES('r','j','rollout#2 contaminated: copied solution')")
self.con.commit()
stats = ex.materialize(["r"], "data", purpose="sft")["stats"]
self.assertEqual(stats["written"], 0)
self.assertEqual(stats["dropped"], {"sft:partial-transcript": 1, "sft:claim-missing-or-inconsistent": 1,
"contaminated": 1})
def test_old_version_and_missing_verdict_have_explicit_exclusions(self):
self.add(version=5)
rd = self.add(kk=1)
(rd / "verdict.json").unlink()
stats = ex.materialize(["r"], "data")["stats"]
self.assertEqual((stats["scanned"], stats["excluded"], stats["written"]), (2, 2, 0))
exclusions = (self.root / "datasets/data/exclusions.jsonl").read_text()
self.assertIn("old_version", exclusions)
self.assertIn("missing_verdict", exclusions)
def test_sft_and_rl_exports_do_not_read_historical_pipeline(self):
self.add()
with patch.object(ex, "pipeline_records", side_effect=OSError("unreadable old pipeline")) as pipeline:
for purpose in ("sft", "rl"):
with self.subTest(purpose=purpose):
result = ex.materialize(["r"], purpose, purpose=purpose)
self.assertEqual(result["stats"]["agent_written"], 1)
self.assertEqual(result["stats"]["pipeline_written"], 0)
self.assertEqual((self.root / "datasets" / purpose / "pipeline.jsonl").read_text(), "")
pipeline.assert_not_called()
def test_pipeline_is_real_separate_and_failed_jobs_are_retained(self):
self.add()
jd = self.root / "runs/r/j"
sd = jd / "oracle/stages/05-compile/a00"
sd.mkdir(parents=True)
(sd / "meta.json").write_text(json.dumps({"cmd": ["hb_compile", "--config", "compile.yaml"], "state": "FAIL", "error_class": "unsupported"}))
(jd / "oracle/result.json").write_text('{"terminal":"FAIL","error_class":"unsupported"}')
self.con.execute("INSERT INTO jobs(run_id,job_id,mode,state,finished_at) VALUES('r','j','oracle','FAIL',123)")
self.con.execute("INSERT INTO stages(run_id,job_id,rollout,stage,state) VALUES('r','j',-1,'05-compile','FAIL')")
self.con.commit()
stats = ex.materialize(["r"], "data")["stats"]
self.assertEqual((stats["agent_written"], stats["pipeline_written"]), (1, 1))
rec = json.loads((self.root / "datasets/data/pipeline.jsonl").read_text())
self.assertEqual(rec["source"], "deterministic_pipeline")
self.assertNotIn("messages", rec)
self.assertEqual(rec["stages"][0]["command"][0], "hb_compile")
def test_refresh_is_explicit_stable_and_hash_checked(self):
self.add()
ex.materialize(["r"], "data")
out = self.root / "datasets/data"
first = (out / "teacher.jsonl").read_bytes()
with self.assertRaises(FileExistsError):
ex.materialize(["r"], "data")
manifest = ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
self.assertEqual(manifest["files"]["teacher.jsonl"]["sha256"], hashlib.sha256(first).hexdigest())
self.assertFalse(list(out.glob(".refresh-*")))
with patch.object(ex, "_write_dataset", side_effect=RuntimeError("interrupted")):
with self.assertRaises(RuntimeError):
ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
self.assertFalse(list(out.glob(".refresh-*")))
def test_claim_must_be_explicitly_consistent(self):
self.assertFalse(ex.sft_eligibility({"terminal": "PASS"}, 4.0)[0])
self.assertTrue(ex.sft_eligibility({"terminal": "PASS", "claim_consistent": True}, 4.0)[0])
def test_redump_uses_configured_node_and_path(self):
rd = self.add()
(rd / "agent/tools.json").unlink()
(rd / "agent/cmd.json").write_text('{"cmd":["pi","--tools","bash"]}')
(self.root / "agent/dump_tools.js").write_text("// test")
with patch("harness.paths.config", return_value={"paths": {"node_bin": "/opt/bench-node/bin/node"}}), \
patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout='[{"name":"bash"}]')) as run:
definitions, source = ex.tool_defs(rd)
self.assertEqual(source, "redumped")
self.assertEqual(definitions[0]["name"], "bash")
self.assertEqual(run.call_args.args[0][0], "/opt/bench-node/bin/node")
self.assertTrue(run.call_args.kwargs["env"]["PATH"].startswith("/opt/bench-node/bin:"))
if __name__ == "__main__":
unittest.main()