258 lines
14 KiB
Python
258 lines
14 KiB
Python
"""Training materialization regressions: evidence retention, pairing and refresh."""
|
|
import hashlib
|
|
import json
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from export import to_agentrl as ex
|
|
from export.reward import compute
|
|
from harness.db import SCHEMA
|
|
from harness.verify import VERDICT_SCHEMA, VERIFICATION_RULE
|
|
|
|
|
|
def dialogue():
|
|
return [{"role": "user", "content": "Compile and evaluate."},
|
|
{"role": "assistant", "content": [{"type": "toolCall", "id": "t1", "name": "bash", "arguments": {"command": "bench stage status h"}}]},
|
|
{"role": "toolResult", "toolCallId": "t1", "content": [{"type": "text", "text": "DONE"}]}]
|
|
|
|
|
|
class MaterializationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
(self.root / "agent").mkdir()
|
|
(self.root / "agent/VERSION").write_text("6")
|
|
(self.root / "state").mkdir()
|
|
self.con = sqlite3.connect(self.root / "state/jobs.sqlite")
|
|
self.con.executescript(SCHEMA)
|
|
self.patch = patch.object(ex, "BENCH", self.root)
|
|
self.patch.start()
|
|
self.env = patch.dict("os.environ", {"BENCH_STATE_DIR": str(self.root / "state")})
|
|
self.env.start()
|
|
|
|
def tearDown(self):
|
|
self.env.stop(); self.patch.stop(); self.con.close(); self.tmp.cleanup()
|
|
|
|
def add(self, kk=0, terminal="PASS", execution="DONE", review=False, ended=True, version=6, mode="e2e"):
|
|
jd = self.root / "runs/r/j"
|
|
rd = jd / "rollouts" / f"{kk:02d}"
|
|
(rd / "agent").mkdir(parents=True)
|
|
job = {"run_id": "r", "job_id": "j", "task": "cls", "model": "resnet18", "march": "nash-m", "backend": "ptq",
|
|
"gates": {"required": ["G1", "G3"], "G1": {}, "G2": {},
|
|
"G3": {"kind": "rel_pct", "tol": 5, "require_source": "board"}, "G4": {}}}
|
|
(jd / "job.json").write_text(json.dumps(job))
|
|
(rd / "rollout.json").write_text(json.dumps({"state": execution, "model": "glm-5.3", "policy": "teacher", "mode": mode,
|
|
"agent_config_version": version, "max_turns": 60, "max_tokens": 100}))
|
|
verdict = {"schema": VERDICT_SCHEMA, "verification_rule": VERIFICATION_RULE, "mode": mode, "run_id": "r", "job_id": "j", "rollout": kk, "terminal": terminal,
|
|
"execution_terminal": execution, "timeout_reason": "budget" if execution == "TIMEOUT" else None,
|
|
"claim_consistent": True, "claim": {"terminal": terminal}, "review": review,
|
|
"gates": {"G1": {"verdict": "pass"}, "G2": {"verdict": "pass"}, "G3": {"verdict": "pass-screen"}},
|
|
"cost": {"turns": 30, "tokens": 50}}
|
|
(rd / "verdict.json").write_text(json.dumps(verdict))
|
|
events = ([{"type": "agent_end", "messages": dialogue()}] if ended else
|
|
[{"type": "message_end", "message": message} for message in dialogue()])
|
|
(rd / "agent/events.jsonl").write_text("\n".join(json.dumps(e) for e in events))
|
|
(rd / "agent/system.md").write_text("Use tools and report measured results.")
|
|
(rd / "agent/tools.json").write_text('[{"name":"bash","parameters":{"type":"object"}}]')
|
|
self.con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','j',?,?)", (kk, terminal))
|
|
self.con.commit()
|
|
return rd
|
|
|
|
def test_completed_tool_tail_is_preserved(self):
|
|
msgs, stats = ex.clean(dialogue())
|
|
self.assertEqual(msgs[-1]["role"], "tool")
|
|
self.assertEqual(len(msgs), 3)
|
|
self.assertTrue(stats["tool_pairing"]["valid"])
|
|
|
|
def test_dangling_and_orphan_tool_evidence_is_labeled(self):
|
|
msgs, stats = ex.clean(dialogue()[:-1])
|
|
self.assertEqual(len(msgs), 2)
|
|
self.assertEqual(stats["tool_pairing"]["missing_results"], 1)
|
|
_, stats = ex.clean([dialogue()[0], dialogue()[2]])
|
|
self.assertEqual(stats["tool_pairing"]["orphan_results"], 1)
|
|
|
|
def test_timeout_keeps_verified_stage_rewards(self):
|
|
rd = self.add(execution="TIMEOUT")
|
|
reward = compute(json.loads((rd / "verdict.json").read_text()), 60, 100)
|
|
self.assertEqual(reward["recipe"], "v0.6")
|
|
self.assertEqual(reward["terms"]["G3"], 2.0)
|
|
self.assertEqual(reward["terms"]["timeout"], -1.0)
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
|
|
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertEqual(rec["verdict"]["terminal"], "PASS")
|
|
self.assertEqual(rec["execution"]["terminal"], "TIMEOUT")
|
|
|
|
def test_all_retains_timeout_failure_and_partial_without_inventing_messages(self):
|
|
self.add(terminal="TIMEOUT", execution="TIMEOUT", ended=False)
|
|
self.add(kk=1, terminal="FAIL")
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["written"], 2)
|
|
self.assertEqual(manifest["stats"]["partial"], 1)
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
|
|
recs = [json.loads(line) for line in (self.root / "datasets/data/teacher.jsonl").read_text().splitlines()]
|
|
self.assertEqual(recs[0]["trajectory"]["source"], "message_end")
|
|
self.assertEqual(sum(m["role"] == "assistant" for m in recs[0]["messages"]), 1)
|
|
|
|
def legacy_evidence(self, rd, *, mode="e2e", claim="PASS"):
|
|
original = json.loads((rd / "verdict.json").read_text())
|
|
original.update(schema="bench.verdict/2", review=True,
|
|
oracle_ref={"terminal": "FAIL", "G3_delta": -99})
|
|
(rd / "verdict.json").write_text(json.dumps(original))
|
|
(rd / "claim.json").write_text(json.dumps({"terminal": claim}))
|
|
for stage, measurements in {
|
|
"06-hbm": {"G1": {"hbm_exists": True, "march_ok": True, "io_ok": True,
|
|
"verifier_ok": True, "log_has_error": False}},
|
|
"08-eval": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}},
|
|
}.items():
|
|
sd = rd / "stages" / stage
|
|
sd.mkdir(parents=True, exist_ok=True)
|
|
(sd / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements}))
|
|
return original
|
|
|
|
def test_legacy_oracle_review_is_reverified_without_modifying_sources(self):
|
|
rd = self.add(review=True)
|
|
self.legacy_evidence(rd)
|
|
before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in rd.rglob("*") if p.is_file()}
|
|
before_db = list(self.con.execute("SELECT * FROM rollouts"))
|
|
manifest = ex.materialize(["r"], "data", purpose="sft")
|
|
self.assertEqual(manifest["stats"]["reverified"], 1)
|
|
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertTrue(rec["sft_eligible"])
|
|
self.assertTrue(rec["training_uses"]["reward_learning_candidate"])
|
|
self.assertEqual(rec["verification"]["method"], "readonly-reverification")
|
|
self.assertEqual(rec["verdict"]["schema"], VERDICT_SCHEMA)
|
|
self.assertFalse(rec["verdict"]["review"])
|
|
self.assertNotIn("oracle_ref", rec)
|
|
self.assertNotIn("history", rec)
|
|
self.assertTrue(rec["provenance"]["original_verdict"]["review"])
|
|
self.assertEqual(rec["provenance"]["original_verdict"]["sha256"], before[rd / "verdict.json"])
|
|
self.assertEqual(before, {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in before})
|
|
self.assertEqual(before_db, list(self.con.execute("SELECT * FROM rollouts")))
|
|
|
|
def test_failed_reverification_is_excluded_without_legacy_fallback(self):
|
|
rd = self.add(review=True)
|
|
self.legacy_evidence(rd)
|
|
(rd / "stages/08-eval/meta.json").write_text("broken evidence")
|
|
stats = ex.materialize(["r"], "data")["stats"]
|
|
self.assertEqual(stats["written"], 0)
|
|
self.assertEqual(stats["dropped"], {"reverification_failed:JSONDecodeError": 1})
|
|
|
|
def test_historical_repair_remains_analysis_only(self):
|
|
rd = self.add(mode="repair")
|
|
self.legacy_evidence(rd)
|
|
ex.materialize(["r"], "all")
|
|
rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text())
|
|
self.assertEqual(rec["mode"], "repair")
|
|
self.assertFalse(rec["sft_eligible"])
|
|
self.assertEqual(rec["sft_reason"], "historical-repair-context")
|
|
self.assertFalse(rec["training_uses"]["reward_learning_candidate"])
|
|
self.assertEqual(ex.materialize(["r"], "sft", purpose="sft")["stats"]["written"], 0)
|
|
self.assertEqual(ex.materialize(["r"], "rl", purpose="rl")["stats"]["written"], 0)
|
|
|
|
def test_blocked_is_not_a_success_or_reward_training_target(self):
|
|
rd = self.add(terminal="BLOCKED")
|
|
v = json.loads((rd / "verdict.json").read_text())
|
|
v.update(blocked_correct=True, oracle_ref={"terminal": "FAIL"})
|
|
(rd / "verdict.json").write_text(json.dumps(v))
|
|
ex.materialize(["r"], "all")
|
|
rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text())
|
|
self.assertFalse(rec["sft_eligible"])
|
|
self.assertFalse(rec["training_uses"]["reward_learning_candidate"])
|
|
self.assertNotIn("blocked", rec["reward_terms"])
|
|
|
|
def test_oracle_independence_does_not_bypass_transcript_or_annotation_checks(self):
|
|
first = self.add(ended=False)
|
|
self.legacy_evidence(first)
|
|
second = self.add(kk=1)
|
|
self.legacy_evidence(second)
|
|
(second / "claim.json").unlink()
|
|
third = self.add(kk=2)
|
|
self.legacy_evidence(third)
|
|
self.con.execute("INSERT INTO annotations(run_id,job_id,note) VALUES('r','j','rollout#2 contaminated: copied solution')")
|
|
self.con.commit()
|
|
stats = ex.materialize(["r"], "data", purpose="sft")["stats"]
|
|
self.assertEqual(stats["written"], 0)
|
|
self.assertEqual(stats["dropped"], {"sft:partial-transcript": 1, "sft:claim-missing-or-inconsistent": 1,
|
|
"contaminated": 1})
|
|
|
|
def test_old_version_and_missing_verdict_have_explicit_exclusions(self):
|
|
self.add(version=5)
|
|
rd = self.add(kk=1)
|
|
(rd / "verdict.json").unlink()
|
|
stats = ex.materialize(["r"], "data")["stats"]
|
|
self.assertEqual((stats["scanned"], stats["excluded"], stats["written"]), (2, 2, 0))
|
|
exclusions = (self.root / "datasets/data/exclusions.jsonl").read_text()
|
|
self.assertIn("old_version", exclusions)
|
|
self.assertIn("missing_verdict", exclusions)
|
|
|
|
def test_sft_and_rl_exports_do_not_read_historical_pipeline(self):
|
|
self.add()
|
|
with patch.object(ex, "pipeline_records", side_effect=OSError("unreadable old pipeline")) as pipeline:
|
|
for purpose in ("sft", "rl"):
|
|
with self.subTest(purpose=purpose):
|
|
result = ex.materialize(["r"], purpose, purpose=purpose)
|
|
self.assertEqual(result["stats"]["agent_written"], 1)
|
|
self.assertEqual(result["stats"]["pipeline_written"], 0)
|
|
self.assertEqual((self.root / "datasets" / purpose / "pipeline.jsonl").read_text(), "")
|
|
pipeline.assert_not_called()
|
|
|
|
def test_pipeline_is_real_separate_and_failed_jobs_are_retained(self):
|
|
self.add()
|
|
jd = self.root / "runs/r/j"
|
|
sd = jd / "oracle/stages/05-compile/a00"
|
|
sd.mkdir(parents=True)
|
|
(sd / "meta.json").write_text(json.dumps({"cmd": ["hb_compile", "--config", "compile.yaml"], "state": "FAIL", "error_class": "unsupported"}))
|
|
(jd / "oracle/result.json").write_text('{"terminal":"FAIL","error_class":"unsupported"}')
|
|
self.con.execute("INSERT INTO jobs(run_id,job_id,mode,state,finished_at) VALUES('r','j','oracle','FAIL',123)")
|
|
self.con.execute("INSERT INTO stages(run_id,job_id,rollout,stage,state) VALUES('r','j',-1,'05-compile','FAIL')")
|
|
self.con.commit()
|
|
stats = ex.materialize(["r"], "data")["stats"]
|
|
self.assertEqual((stats["agent_written"], stats["pipeline_written"]), (1, 1))
|
|
rec = json.loads((self.root / "datasets/data/pipeline.jsonl").read_text())
|
|
self.assertEqual(rec["source"], "deterministic_pipeline")
|
|
self.assertNotIn("messages", rec)
|
|
self.assertEqual(rec["stages"][0]["command"][0], "hb_compile")
|
|
|
|
def test_refresh_is_explicit_stable_and_hash_checked(self):
|
|
self.add()
|
|
ex.materialize(["r"], "data")
|
|
out = self.root / "datasets/data"
|
|
first = (out / "teacher.jsonl").read_bytes()
|
|
with self.assertRaises(FileExistsError):
|
|
ex.materialize(["r"], "data")
|
|
manifest = ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
|
|
self.assertEqual(manifest["files"]["teacher.jsonl"]["sha256"], hashlib.sha256(first).hexdigest())
|
|
self.assertFalse(list(out.glob(".refresh-*")))
|
|
with patch.object(ex, "_write_dataset", side_effect=RuntimeError("interrupted")):
|
|
with self.assertRaises(RuntimeError):
|
|
ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
|
|
self.assertFalse(list(out.glob(".refresh-*")))
|
|
|
|
def test_claim_must_be_explicitly_consistent(self):
|
|
self.assertFalse(ex.sft_eligibility({"terminal": "PASS"}, 4.0)[0])
|
|
self.assertTrue(ex.sft_eligibility({"terminal": "PASS", "claim_consistent": True}, 4.0)[0])
|
|
|
|
def test_redump_uses_configured_node_and_path(self):
|
|
rd = self.add()
|
|
(rd / "agent/tools.json").unlink()
|
|
(rd / "agent/cmd.json").write_text('{"cmd":["pi","--tools","bash"]}')
|
|
(self.root / "agent/dump_tools.js").write_text("// test")
|
|
with patch("harness.paths.config", return_value={"paths": {"node_bin": "/opt/bench-node/bin/node"}}), \
|
|
patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout='[{"name":"bash"}]')) as run:
|
|
definitions, source = ex.tool_defs(rd)
|
|
self.assertEqual(source, "redumped")
|
|
self.assertEqual(definitions[0]["name"], "bash")
|
|
self.assertEqual(run.call_args.args[0][0], "/opt/bench-node/bin/node")
|
|
self.assertTrue(run.call_args.kwargs["env"]["PATH"].startswith("/opt/bench-node/bin:"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|