170 lines
9.0 KiB
Python
170 lines
9.0 KiB
Python
"""Training materialization regressions: evidence retention, pairing and refresh."""
|
|
import hashlib
|
|
import json
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from export import to_agentrl as ex
|
|
from export.reward import compute
|
|
from harness.db import SCHEMA
|
|
|
|
|
|
def dialogue():
|
|
return [{"role": "user", "content": "Compile and evaluate."},
|
|
{"role": "assistant", "content": [{"type": "toolCall", "id": "t1", "name": "bash", "arguments": {"command": "bench stage status h"}}]},
|
|
{"role": "toolResult", "toolCallId": "t1", "content": [{"type": "text", "text": "DONE"}]}]
|
|
|
|
|
|
class MaterializationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
(self.root / "agent").mkdir()
|
|
(self.root / "agent/VERSION").write_text("6")
|
|
(self.root / "state").mkdir()
|
|
self.con = sqlite3.connect(self.root / "state/jobs.sqlite")
|
|
self.con.executescript(SCHEMA)
|
|
self.patch = patch.object(ex, "BENCH", self.root)
|
|
self.patch.start()
|
|
self.env = patch.dict("os.environ", {"BENCH_STATE_DIR": str(self.root / "state")})
|
|
self.env.start()
|
|
|
|
def tearDown(self):
|
|
self.env.stop(); self.patch.stop(); self.con.close(); self.tmp.cleanup()
|
|
|
|
def add(self, kk=0, terminal="PASS", execution="DONE", review=False, ended=True, version=6):
|
|
jd = self.root / "runs/r/j"
|
|
rd = jd / "rollouts" / f"{kk:02d}"
|
|
(rd / "agent").mkdir(parents=True)
|
|
job = {"run_id": "r", "job_id": "j", "task": "cls", "model": "resnet18", "march": "nash-m", "backend": "ptq"}
|
|
(jd / "job.json").write_text(json.dumps(job))
|
|
(rd / "rollout.json").write_text(json.dumps({"state": execution, "model": "glm-5.3", "policy": "teacher",
|
|
"agent_config_version": version, "max_turns": 60, "max_tokens": 100}))
|
|
verdict = {"run_id": "r", "job_id": "j", "rollout": kk, "terminal": terminal,
|
|
"execution_terminal": execution, "timeout_reason": "budget" if execution == "TIMEOUT" else None,
|
|
"claim_consistent": True, "claim": {"terminal": terminal}, "review": review,
|
|
"gates": {"G1": {"verdict": "pass"}, "G2": {"verdict": "pass"}, "G3": {"verdict": "pass-screen"}},
|
|
"cost": {"turns": 30, "tokens": 50}}
|
|
(rd / "verdict.json").write_text(json.dumps(verdict))
|
|
events = ([{"type": "agent_end", "messages": dialogue()}] if ended else
|
|
[{"type": "message_end", "message": message} for message in dialogue()])
|
|
(rd / "agent/events.jsonl").write_text("\n".join(json.dumps(e) for e in events))
|
|
(rd / "agent/system.md").write_text("Use tools and report measured results.")
|
|
(rd / "agent/tools.json").write_text('[{"name":"bash","parameters":{"type":"object"}}]')
|
|
self.con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','j',?,?)", (kk, terminal))
|
|
self.con.commit()
|
|
return rd
|
|
|
|
def test_completed_tool_tail_is_preserved(self):
|
|
msgs, stats = ex.clean(dialogue())
|
|
self.assertEqual(msgs[-1]["role"], "tool")
|
|
self.assertEqual(len(msgs), 3)
|
|
self.assertTrue(stats["tool_pairing"]["valid"])
|
|
|
|
def test_dangling_and_orphan_tool_evidence_is_labeled(self):
|
|
msgs, stats = ex.clean(dialogue()[:-1])
|
|
self.assertEqual(len(msgs), 2)
|
|
self.assertEqual(stats["tool_pairing"]["missing_results"], 1)
|
|
_, stats = ex.clean([dialogue()[0], dialogue()[2]])
|
|
self.assertEqual(stats["tool_pairing"]["orphan_results"], 1)
|
|
|
|
def test_timeout_keeps_verified_stage_rewards(self):
|
|
rd = self.add(execution="TIMEOUT")
|
|
reward = compute(json.loads((rd / "verdict.json").read_text()), 60, 100)
|
|
self.assertEqual(reward["recipe"], "v0.5")
|
|
self.assertEqual(reward["terms"]["G3"], 2.0)
|
|
self.assertEqual(reward["terms"]["timeout"], -1.0)
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
|
|
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertEqual(rec["verdict"]["terminal"], "PASS")
|
|
self.assertEqual(rec["execution"]["terminal"], "TIMEOUT")
|
|
|
|
def test_all_retains_timeout_failure_and_partial_without_inventing_messages(self):
|
|
self.add(terminal="TIMEOUT", execution="TIMEOUT", ended=False)
|
|
self.add(kk=1, terminal="FAIL")
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["written"], 2)
|
|
self.assertEqual(manifest["stats"]["partial"], 1)
|
|
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
|
|
recs = [json.loads(line) for line in (self.root / "datasets/data/teacher.jsonl").read_text().splitlines()]
|
|
self.assertEqual(recs[0]["trajectory"]["source"], "message_end")
|
|
self.assertEqual(sum(m["role"] == "assistant" for m in recs[0]["messages"]), 1)
|
|
|
|
def test_review_is_retained_but_not_positive_training_candidate(self):
|
|
self.add(review=True)
|
|
manifest = ex.materialize(["r"], "data")
|
|
self.assertEqual(manifest["stats"]["review"], 1)
|
|
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
|
|
self.assertFalse(rec["sft_eligible"])
|
|
self.assertFalse(rec["training_uses"]["reward_learning_candidate"])
|
|
|
|
def test_old_version_and_missing_verdict_have_explicit_exclusions(self):
|
|
self.add(version=5)
|
|
rd = self.add(kk=1)
|
|
(rd / "verdict.json").unlink()
|
|
stats = ex.materialize(["r"], "data")["stats"]
|
|
self.assertEqual((stats["scanned"], stats["excluded"], stats["written"]), (2, 2, 0))
|
|
exclusions = (self.root / "datasets/data/exclusions.jsonl").read_text()
|
|
self.assertIn("old_version", exclusions)
|
|
self.assertIn("missing_verdict", exclusions)
|
|
|
|
def test_pipeline_is_real_separate_and_failed_jobs_are_retained(self):
|
|
self.add()
|
|
jd = self.root / "runs/r/j"
|
|
sd = jd / "oracle/stages/05-compile/a00"
|
|
sd.mkdir(parents=True)
|
|
(sd / "meta.json").write_text(json.dumps({"cmd": ["hb_compile", "--config", "compile.yaml"], "state": "FAIL", "error_class": "unsupported"}))
|
|
(jd / "oracle/result.json").write_text('{"terminal":"FAIL","error_class":"unsupported"}')
|
|
self.con.execute("INSERT INTO jobs(run_id,job_id,mode,state,finished_at) VALUES('r','j','oracle','FAIL',123)")
|
|
self.con.execute("INSERT INTO stages(run_id,job_id,rollout,stage,state) VALUES('r','j',-1,'05-compile','FAIL')")
|
|
self.con.commit()
|
|
stats = ex.materialize(["r"], "data")["stats"]
|
|
self.assertEqual((stats["agent_written"], stats["pipeline_written"]), (1, 1))
|
|
rec = json.loads((self.root / "datasets/data/pipeline.jsonl").read_text())
|
|
self.assertEqual(rec["source"], "deterministic_pipeline")
|
|
self.assertNotIn("messages", rec)
|
|
self.assertEqual(rec["stages"][0]["command"][0], "hb_compile")
|
|
|
|
def test_refresh_is_explicit_stable_and_hash_checked(self):
|
|
self.add()
|
|
ex.materialize(["r"], "data")
|
|
out = self.root / "datasets/data"
|
|
first = (out / "teacher.jsonl").read_bytes()
|
|
with self.assertRaises(FileExistsError):
|
|
ex.materialize(["r"], "data")
|
|
manifest = ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
|
|
self.assertEqual(manifest["files"]["teacher.jsonl"]["sha256"], hashlib.sha256(first).hexdigest())
|
|
self.assertFalse(list(out.glob(".refresh-*")))
|
|
with patch.object(ex, "_write_dataset", side_effect=RuntimeError("interrupted")):
|
|
with self.assertRaises(RuntimeError):
|
|
ex.materialize(["r"], "data", refresh=True)
|
|
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
|
|
self.assertFalse(list(out.glob(".refresh-*")))
|
|
|
|
def test_claim_must_be_explicitly_consistent(self):
|
|
self.assertFalse(ex.sft_eligibility({"terminal": "PASS"}, 4.0)[0])
|
|
self.assertTrue(ex.sft_eligibility({"terminal": "PASS", "claim_consistent": True}, 4.0)[0])
|
|
|
|
def test_redump_uses_configured_node_and_path(self):
|
|
rd = self.add()
|
|
(rd / "agent/tools.json").unlink()
|
|
(rd / "agent/cmd.json").write_text('{"cmd":["pi","--tools","bash"]}')
|
|
(self.root / "agent/dump_tools.js").write_text("// test")
|
|
with patch("harness.paths.config", return_value={"paths": {"node_bin": "/opt/bench-node/bin/node"}}), \
|
|
patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout='[{"name":"bash"}]')) as run:
|
|
definitions, source = ex.tool_defs(rd)
|
|
self.assertEqual(source, "redumped")
|
|
self.assertEqual(definitions[0]["name"], "bash")
|
|
self.assertEqual(run.call_args.args[0][0], "/opt/bench-node/bin/node")
|
|
self.assertTrue(run.call_args.kwargs["env"]["PATH"].startswith("/opt/bench-node/bin:"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|