data-gen-bpu/scripts/test_eval_provenance.py

147 lines
7.6 KiB
Python

"""A continued Agent keeps its source evidence without mixing evaluation sets."""
import copy
import json
import os
import sqlite3
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from export import reward, to_agentrl
from harness import db, paths, verify
def evaluation(n=100):
return {"dataset": "imagenet", "n": n, "content_sha": f"content-{n}",
"manifest_sha": f"archive-{n}", "evaluator_version": "v1"}
def verdict():
return {"terminal": "PASS", "eval_set": evaluation(),
"gates": {"G3": {"verdict": "pass-screen", "kind": "rel_pct", "delta": 1, "tolerance": 5}},
"oracle_ref": {"G3_kind": "rel_pct", "G3_delta": 3, "eval_set": evaluation()}}
class RewardEvaluationIdentity(unittest.TestCase):
def test_same_content_is_comparable_even_when_archive_changes(self):
v = verdict()
v["oracle_ref"]["eval_set"]["manifest_sha"] = "repacked"
result = reward.compute(v)
self.assertEqual(result["terms"]["acc_rel"], .4)
self.assertIsNone(result["acc_rel_reason"])
def test_different_set_or_evaluator_does_not_reward_accuracy_improvement(self):
for field, value in (("n", 2000), ("content_sha", "another-100"), ("dataset", "coco"),
("evaluator_version", "v2")):
with self.subTest(field=field):
v = verdict()
v["oracle_ref"]["eval_set"][field] = value
result = reward.compute(v)
self.assertEqual(result["terms"]["acc_rel"], 0)
self.assertEqual(result["acc_rel_reason"], "eval_set_mismatch")
def test_missing_identity_is_not_comparable_for_continuation(self):
v = verdict()
v.pop("eval_set")
v["oracle_ref"].pop("eval_set")
self.assertEqual(reward.compute(v)["terms"]["acc_rel"], .4) # legacy
v["continuation"] = {"source_run_id": "old", "source_job_id": "job"}
result = reward.compute(v)
self.assertEqual(result["terms"]["acc_rel"], 0)
self.assertEqual(result["acc_rel_reason"], "missing_eval_set")
v["eval_set"] = evaluation()
self.assertEqual(reward.compute(v)["acc_rel_reason"], "missing_eval_set")
class ExportEvaluationIdentity(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.root = Path(tmp.name)
self.addCleanup(patch.stopall)
patch.object(paths, "RUNS", self.root / "runs").start()
patch.object(to_agentrl, "BENCH", self.root).start()
patch.dict(os.environ, {"BENCH_STATE_DIR": str(self.root / "state")}).start()
(self.root / "state").mkdir()
self.database = self.root / "state/jobs.sqlite"
with sqlite3.connect(self.database) as con:
con.executescript(db.SCHEMA)
con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('new','job',2,'repair','teacher','PASS')")
con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('old','job',2,'repair','teacher','TRANSFERRED')")
con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','pending','oracle','PENDING')")
self.job = {"run_id": "new", "job_id": "job", "task": "cls", "march": "nash-m", "backend": "ptq",
"data": {"dataset": "imagenet", "t1": evaluation()}, "evaluator_version": "v1",
"gates": {"required": [], "G1": {}, "G2": {}, "G3": {"kind": "rel_pct", "tol": 5}, "G4": {}},
"oracle_ref": {"run_id": "old", "job_id": "job"},
"continuation": {"source_run_id": "old", "source_job_id": "job"}}
self.jd = self.root / "runs/new/job"
self.rd = self.jd / "rollouts/02"
self.rd.mkdir(parents=True)
self.save(self.jd / "job.json", self.job)
old = copy.deepcopy(self.job)
old.update(run_id="old", data={"dataset": "imagenet", "t1": evaluation(2000)})
old.pop("oracle_ref")
old.pop("continuation")
self.save(self.root / "runs/old/job/job.json", old)
self.save(self.root / "runs/old/job/oracle/result.json", {
"terminal": "PASS", "gates": {"G3": {"kind": "rel_pct", "delta": 3}}})
self.save(self.rd / "rollout.json", {"state": "DONE", "mode": "repair", "policy": "teacher", "agent_config_version": 1})
self.save(self.rd / "claim.json", {"terminal": "PASS"})
self.save(self.rd / "stages/08-eval/meta.json", {
"state": "OK", "measurements": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}}})
self.save(self.rd / "agent/events.jsonl", {"type": "agent_end", "messages": [
{"role": "user", "content": "Test this model."}, {"role": "assistant", "content": "Completed."}]})
self.save(self.rd / "agent/tools.json", [{"name": "bash"}])
(self.rd / "agent/system.md").write_text("Run the model evaluation.")
@staticmethod
def save(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value))
def test_verdict_preserves_source_identity_without_copying_oracle(self):
result = verify.verify_rollout(self.rd, write=False)
self.assertEqual(result["eval_set"]["n"], 100)
self.assertEqual(result["oracle_ref"]["eval_set"]["n"], 2000)
self.assertEqual((result["oracle_ref"]["run_id"], result["oracle_ref"]["job_id"]), ("old", "job"))
self.assertFalse((self.jd / "oracle").exists())
self.assertEqual(reward.compute(result)["acc_rel_reason"], "eval_set_mismatch")
def test_export_keeps_source_rollout_and_eval_set_and_excludes_transfer(self):
self.save(self.rd / "verdict.json", verify.verify_rollout(self.rd, write=False))
manifest = to_agentrl.materialize(["old", "new"], "test", min_agent_version=1)
record = json.loads((self.root / "datasets/test/teacher.jsonl").read_text())
self.assertEqual(record["rollout"], 2)
self.assertEqual(record["continuation"], {"source_run_id": "old", "source_job_id": "job", "source_rollout": 2})
self.assertEqual(record["eval_set"]["n"], 100)
self.assertEqual(record["oracle_ref"]["eval_set"]["n"], 2000)
self.assertEqual(record["reward_terms"]["acc_rel"], 0)
self.assertEqual(record["reward_acc_rel_reason"], "eval_set_mismatch")
self.assertEqual(manifest["stats"]["scanned"], 1)
self.assertEqual(manifest["stats"]["pipeline_written"], 0)
self.assertEqual(manifest["stats"]["excluded"], 0)
def test_pipeline_export_records_eval_identity(self):
with sqlite3.connect(self.database) as con:
con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','job','oracle','FAIL')")
self.save(self.jd / "oracle/result.json", {"terminal": "FAIL"})
record = next(to_agentrl.pipeline_records(["new"]))
self.assertEqual(record["eval_set"]["n"], 100)
self.assertEqual(record["continuation"]["source_run_id"], "old")
def test_export_missing_verdict_identity_cannot_use_legacy_comparison(self):
result = verify.verify_rollout(self.rd, write=False)
result.pop("eval_set")
result.pop("continuation")
result["oracle_ref"].pop("eval_set")
self.save(self.rd / "verdict.json", result)
to_agentrl.materialize(["new"], "missing-identity", min_agent_version=1)
record = json.loads((self.root / "datasets/missing-identity/teacher.jsonl").read_text())
self.assertEqual(record["reward_terms"]["acc_rel"], 0)
self.assertEqual(record["reward_acc_rel_reason"], "missing_eval_set")
if __name__ == "__main__":
unittest.main()