data-gen-bpu/export/test_materialization.py
bench ee4c7fce1c reward v0.4: board_offline 不加分只免罚;board-backfill 支持 --rollout 用现成 HBM 补测
- export/reward.py: G3 unverified 且最后一次 07/08 为 board_offline → G2/G3 权重 0、G1 保持 1.0、免 TIMEOUT 罚分,
  返回 unverifiable=board_offline;RECIPE v0.4。实测 r20260916-01 regnet.y.800mf −0.97 → +0.03
- export/to_agentrl.py: 记录 reward_unverifiable
- harness/pipeline.py: board_backfill_rollout:用 rollouts/kk/artifacts 的 HBM 补跑 07/08(lane=backfill)后 verify_rollout
- harness/cli.py: bench board-backfill --rollout JOB_ID[:KK]
- 单测 scripts/test_reward_board_offline.py、scripts/test_rollout_backfill.py;CONTRACT §9 / README 同步
2026-09-16 15:58:24 +08:00

170 lines
9.0 KiB
Python

"""Training materialization regressions: evidence retention, pairing and refresh."""
import hashlib
import json
import sqlite3
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from export import to_agentrl as ex
from export.reward import compute
from harness.db import SCHEMA
def dialogue():
return [{"role": "user", "content": "Compile and evaluate."},
{"role": "assistant", "content": [{"type": "toolCall", "id": "t1", "name": "bash", "arguments": {"command": "bench stage status h"}}]},
{"role": "toolResult", "toolCallId": "t1", "content": [{"type": "text", "text": "DONE"}]}]
class MaterializationTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
(self.root / "agent").mkdir()
(self.root / "agent/VERSION").write_text("6")
(self.root / "state").mkdir()
self.con = sqlite3.connect(self.root / "state/jobs.sqlite")
self.con.executescript(SCHEMA)
self.patch = patch.object(ex, "BENCH", self.root)
self.patch.start()
self.env = patch.dict("os.environ", {"BENCH_STATE_DIR": str(self.root / "state")})
self.env.start()
def tearDown(self):
self.env.stop(); self.patch.stop(); self.con.close(); self.tmp.cleanup()
def add(self, kk=0, terminal="PASS", execution="DONE", review=False, ended=True, version=6):
jd = self.root / "runs/r/j"
rd = jd / "rollouts" / f"{kk:02d}"
(rd / "agent").mkdir(parents=True)
job = {"run_id": "r", "job_id": "j", "task": "cls", "model": "resnet18", "march": "nash-m", "backend": "ptq"}
(jd / "job.json").write_text(json.dumps(job))
(rd / "rollout.json").write_text(json.dumps({"state": execution, "model": "glm-5.3", "policy": "teacher",
"agent_config_version": version, "max_turns": 60, "max_tokens": 100}))
verdict = {"run_id": "r", "job_id": "j", "rollout": kk, "terminal": terminal,
"execution_terminal": execution, "timeout_reason": "budget" if execution == "TIMEOUT" else None,
"claim_consistent": True, "claim": {"terminal": terminal}, "review": review,
"gates": {"G1": {"verdict": "pass"}, "G2": {"verdict": "pass"}, "G3": {"verdict": "pass-screen"}},
"cost": {"turns": 30, "tokens": 50}}
(rd / "verdict.json").write_text(json.dumps(verdict))
events = ([{"type": "agent_end", "messages": dialogue()}] if ended else
[{"type": "message_end", "message": message} for message in dialogue()])
(rd / "agent/events.jsonl").write_text("\n".join(json.dumps(e) for e in events))
(rd / "agent/system.md").write_text("Use tools and report measured results.")
(rd / "agent/tools.json").write_text('[{"name":"bash","parameters":{"type":"object"}}]')
self.con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','j',?,?)", (kk, terminal))
self.con.commit()
return rd
def test_completed_tool_tail_is_preserved(self):
msgs, stats = ex.clean(dialogue())
self.assertEqual(msgs[-1]["role"], "tool")
self.assertEqual(len(msgs), 3)
self.assertTrue(stats["tool_pairing"]["valid"])
def test_dangling_and_orphan_tool_evidence_is_labeled(self):
msgs, stats = ex.clean(dialogue()[:-1])
self.assertEqual(len(msgs), 2)
self.assertEqual(stats["tool_pairing"]["missing_results"], 1)
_, stats = ex.clean([dialogue()[0], dialogue()[2]])
self.assertEqual(stats["tool_pairing"]["orphan_results"], 1)
def test_timeout_keeps_verified_stage_rewards(self):
rd = self.add(execution="TIMEOUT")
reward = compute(json.loads((rd / "verdict.json").read_text()), 60, 100)
self.assertEqual(reward["recipe"], "v0.4")
self.assertEqual(reward["terms"]["G3"], 2.0)
self.assertEqual(reward["terms"]["timeout"], -1.0)
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["sft_eligible"], 1)
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertEqual(rec["verdict"]["terminal"], "PASS")
self.assertEqual(rec["execution"]["terminal"], "TIMEOUT")
def test_all_retains_timeout_failure_and_partial_without_inventing_messages(self):
self.add(terminal="TIMEOUT", execution="TIMEOUT", ended=False)
self.add(kk=1, terminal="FAIL")
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["written"], 2)
self.assertEqual(manifest["stats"]["partial"], 1)
self.assertEqual(manifest["stats"]["sft_eligible"], 0)
recs = [json.loads(line) for line in (self.root / "datasets/data/teacher.jsonl").read_text().splitlines()]
self.assertEqual(recs[0]["trajectory"]["source"], "message_end")
self.assertEqual(sum(m["role"] == "assistant" for m in recs[0]["messages"]), 1)
def test_review_is_retained_but_not_positive_training_candidate(self):
self.add(review=True)
manifest = ex.materialize(["r"], "data")
self.assertEqual(manifest["stats"]["review"], 1)
rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text())
self.assertFalse(rec["sft_eligible"])
self.assertFalse(rec["training_uses"]["reward_learning_candidate"])
def test_old_version_and_missing_verdict_have_explicit_exclusions(self):
self.add(version=5)
rd = self.add(kk=1)
(rd / "verdict.json").unlink()
stats = ex.materialize(["r"], "data")["stats"]
self.assertEqual((stats["scanned"], stats["excluded"], stats["written"]), (2, 2, 0))
exclusions = (self.root / "datasets/data/exclusions.jsonl").read_text()
self.assertIn("old_version", exclusions)
self.assertIn("missing_verdict", exclusions)
def test_pipeline_is_real_separate_and_failed_jobs_are_retained(self):
self.add()
jd = self.root / "runs/r/j"
sd = jd / "oracle/stages/05-compile/a00"
sd.mkdir(parents=True)
(sd / "meta.json").write_text(json.dumps({"cmd": ["hb_compile", "--config", "compile.yaml"], "state": "FAIL", "error_class": "unsupported"}))
(jd / "oracle/result.json").write_text('{"terminal":"FAIL","error_class":"unsupported"}')
self.con.execute("INSERT INTO jobs(run_id,job_id,mode,state,finished_at) VALUES('r','j','oracle','FAIL',123)")
self.con.execute("INSERT INTO stages(run_id,job_id,rollout,stage,state) VALUES('r','j',-1,'05-compile','FAIL')")
self.con.commit()
stats = ex.materialize(["r"], "data")["stats"]
self.assertEqual((stats["agent_written"], stats["pipeline_written"]), (1, 1))
rec = json.loads((self.root / "datasets/data/pipeline.jsonl").read_text())
self.assertEqual(rec["source"], "deterministic_pipeline")
self.assertNotIn("messages", rec)
self.assertEqual(rec["stages"][0]["command"][0], "hb_compile")
def test_refresh_is_explicit_stable_and_hash_checked(self):
self.add()
ex.materialize(["r"], "data")
out = self.root / "datasets/data"
first = (out / "teacher.jsonl").read_bytes()
with self.assertRaises(FileExistsError):
ex.materialize(["r"], "data")
manifest = ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
self.assertEqual(manifest["files"]["teacher.jsonl"]["sha256"], hashlib.sha256(first).hexdigest())
self.assertFalse(list(out.glob(".refresh-*")))
with patch.object(ex, "_write_dataset", side_effect=RuntimeError("interrupted")):
with self.assertRaises(RuntimeError):
ex.materialize(["r"], "data", refresh=True)
self.assertEqual(first, (out / "teacher.jsonl").read_bytes())
self.assertFalse(list(out.glob(".refresh-*")))
def test_claim_must_be_explicitly_consistent(self):
self.assertFalse(ex.sft_eligibility({"terminal": "PASS"}, 4.0)[0])
self.assertTrue(ex.sft_eligibility({"terminal": "PASS", "claim_consistent": True}, 4.0)[0])
def test_redump_uses_configured_node_and_path(self):
rd = self.add()
(rd / "agent/tools.json").unlink()
(rd / "agent/cmd.json").write_text('{"cmd":["pi","--tools","bash"]}')
(self.root / "agent/dump_tools.js").write_text("// test")
with patch("harness.paths.config", return_value={"paths": {"node_bin": "/opt/bench-node/bin/node"}}), \
patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout='[{"name":"bash"}]')) as run:
definitions, source = ex.tool_defs(rd)
self.assertEqual(source, "redumped")
self.assertEqual(definitions[0]["name"], "bash")
self.assertEqual(run.call_args.args[0][0], "/opt/bench-node/bin/node")
self.assertTrue(run.call_args.kwargs["env"]["PATH"].startswith("/opt/bench-node/bin:"))
if __name__ == "__main__":
unittest.main()