data-gen-bpu/scripts/test_eval_provenance.py

171 lines
9.0 KiB
Python

"""A continued Agent keeps its source evidence without mixing evaluation sets."""
import copy
import json
import os
import sqlite3
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from export import reward, to_agentrl
from harness import db, paths, verify
def evaluation(n=100):
return {"dataset": "imagenet", "n": n, "content_sha": f"content-{n}",
"manifest_sha": f"archive-{n}", "evaluator_version": "v1"}
def verdict():
return {"terminal": "PASS", "eval_set": evaluation(),
"gates": {"G3": {"verdict": "pass-screen", "kind": "rel_pct", "delta": 1, "tolerance": 5}},
"oracle_ref": {"G3_kind": "rel_pct", "G3_delta": 3, "eval_set": evaluation()}}
class RewardEvaluationIdentity(unittest.TestCase):
def test_oracle_comparison_and_old_review_do_not_affect_reward(self):
v = verdict()
expected = reward.compute({k: value for k, value in v.items() if k != "oracle_ref"})
for oracle in (None, {}, {"terminal": "PASS", "G3_delta": -99},
{"terminal": "FAIL", "G3_delta": 99, "latency_ms_mean": 999}, "damaged"):
with self.subTest(oracle=oracle):
candidate = {**v, "oracle_ref": oracle, "review": True, "blocked_correct": True}
self.assertEqual(reward.compute(candidate), expected)
self.assertEqual(expected["recipe"], "v0.7")
self.assertNotIn("acc_rel", expected["terms"])
self.assertNotIn("lat_rel", expected["terms"])
self.assertNotIn("blocked", expected["terms"])
def test_eval_identity_still_does_not_enable_relative_reward(self):
v = verdict()
expected = reward.compute(v)
for key, value in (("n", 2000), ("content_sha", "different"), ("evaluator_version", "v2")):
v["oracle_ref"]["eval_set"][key] = value
self.assertEqual(reward.compute(v), expected)
v.pop("eval_set")
v["continuation"] = {"source_run_id": "old"}
self.assertEqual(reward.compute(v), expected)
class ExportEvaluationIdentity(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.root = Path(tmp.name)
self.addCleanup(patch.stopall)
patch.object(paths, "RUNS", self.root / "runs").start()
patch.object(to_agentrl, "BENCH", self.root).start()
patch.dict(os.environ, {"BENCH_STATE_DIR": str(self.root / "state")}).start()
(self.root / "state").mkdir()
self.database = self.root / "state/jobs.sqlite"
with sqlite3.connect(self.database) as con:
con.executescript(db.SCHEMA)
con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('new','job',2,'repair','teacher','PASS')")
con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('old','job',2,'repair','teacher','TRANSFERRED')")
con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','pending','oracle','PENDING')")
self.job = {"run_id": "new", "job_id": "job", "task": "cls", "march": "nash-m", "backend": "ptq",
"data": {"dataset": "imagenet", "t1": evaluation()}, "evaluator_version": "v1",
"gates": {"required": [], "G1": {}, "G2": {}, "G3": {"kind": "rel_pct", "tol": 5}, "G4": {}},
"oracle_ref": {"run_id": "old", "job_id": "job"},
"continuation": {"source_run_id": "old", "source_job_id": "job"}}
self.jd = self.root / "runs/new/job"
self.rd = self.jd / "rollouts/02"
self.rd.mkdir(parents=True)
self.save(self.jd / "job.json", self.job)
old = copy.deepcopy(self.job)
old.update(run_id="old", data={"dataset": "imagenet", "t1": evaluation(2000)})
old.pop("oracle_ref")
old.pop("continuation")
self.save(self.root / "runs/old/job/job.json", old)
self.save(self.root / "runs/old/job/oracle/result.json", {
"terminal": "PASS", "gates": {"G3": {"kind": "rel_pct", "delta": 3}}})
self.save(self.rd / "rollout.json", {"state": "DONE", "mode": "repair", "policy": "teacher", "agent_config_version": 1})
self.save(self.rd / "claim.json", {"terminal": "PASS"})
self.save(self.rd / "stages/08-eval/meta.json", {
"state": "OK", "measurements": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}}})
self.save(self.rd / "agent/events.jsonl", {"type": "agent_end", "messages": [
{"role": "user", "content": "Test this model."}, {"role": "assistant", "content": "Completed."}]})
self.save(self.rd / "agent/tools.json", [{"name": "bash"}])
(self.rd / "agent/system.md").write_text("Run the model evaluation.")
@staticmethod
def save(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value))
def test_verdict_is_independent_of_absent_pass_fail_or_damaged_oracle(self):
expected = verify.verify_rollout(self.rd, write=False)
expected.pop("verified_at")
old = self.root / "runs/old/job"
# Even a damaged historical job must never be opened by verification.
(old / "job.json").write_text("damaged historical job")
op = old / "oracle/result.json"
for value in (None, '{"terminal":"PASS"}', '{"terminal":"FAIL"}', "damaged"):
with self.subTest(oracle=value):
if value is None:
op.unlink(missing_ok=True)
else:
op.write_text(value)
result = verify.verify_rollout(self.rd, write=False)
result.pop("verified_at")
self.assertEqual(result, expected)
self.assertEqual(result["eval_set"]["n"], 100)
self.assertNotIn("oracle_ref", result)
self.assertFalse(result["review"])
self.assertEqual(result["schema"], "bench.verdict/4")
self.assertFalse((self.jd / "oracle").exists())
def test_export_keeps_source_rollout_and_eval_set_and_excludes_transfer(self):
self.save(self.rd / "verdict.json", verify.verify_rollout(self.rd, write=False))
manifest = to_agentrl.materialize(["old", "new"], "test", min_agent_version=1)
record = json.loads((self.root / "datasets/test/teacher.jsonl").read_text())
self.assertEqual(record["rollout"], 2)
self.assertEqual(record["continuation"], {"source_run_id": "old", "source_job_id": "job", "source_rollout": 2})
# A job requirement cannot fabricate a missing measured receipt.
self.assertIsNone(record["eval_set"])
self.assertEqual(record["required_eval_set"]["n"], 100)
self.assertNotIn("oracle_ref", record)
self.assertNotIn("oracle_ref", record["verdict"])
self.assertNotIn("acc_rel", record["reward_terms"])
self.assertFalse(record["sft_eligible"]) # historical repair stays analysis
self.assertEqual(manifest["stats"]["scanned"], 1)
self.assertEqual(manifest["stats"]["pipeline_written"], 0)
self.assertEqual(manifest["stats"]["excluded"], 0)
def test_pipeline_export_records_eval_identity(self):
with sqlite3.connect(self.database) as con:
con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','job','oracle','FAIL')")
self.save(self.jd / "oracle/result.json", {"terminal": "FAIL"})
record = next(to_agentrl.pipeline_records(["new"]))
self.assertEqual(record["eval_set"]["n"], 100)
self.assertEqual(record["continuation"]["source_run_id"], "old")
def test_missing_measured_identity_keeps_job_requirement_separate(self):
result = verify.verify_rollout(self.rd, write=False)
result.pop("eval_set")
result.pop("continuation")
self.save(self.rd / "verdict.json", result)
to_agentrl.materialize(["new"], "missing-identity", min_agent_version=1)
record = json.loads((self.root / "datasets/missing-identity/teacher.jsonl").read_text())
self.assertIsNone(record["eval_set"])
self.assertEqual(record["required_eval_set"]["n"], 100)
self.assertNotIn("acc_rel", record["reward_terms"])
self.assertNotIn("reward_acc_rel_reason", record)
def test_measured_identity_is_preserved_even_when_requirement_differs(self):
path = self.rd / 'stages/08-eval/meta.json'
recorded = json.loads(path.read_text())
recorded['measurements']['G3']['eval_set'] = evaluation(2000)
self.save(path, recorded)
self.save(self.rd / 'verdict.json', verify.verify_rollout(self.rd, write=False))
to_agentrl.materialize(['new'], 'different-measured-set', min_agent_version=1)
record = json.loads((self.root / 'datasets/different-measured-set/teacher.jsonl').read_text())
self.assertEqual(record['eval_set']['n'], 2000)
self.assertEqual(record['required_eval_set']['n'], 100)
self.assertFalse(record['sft_eligible'])
if __name__ == "__main__":
unittest.main()