"""A continued Agent keeps its source evidence without mixing evaluation sets.""" import copy import json import os import sqlite3 import tempfile import unittest from pathlib import Path from unittest.mock import patch from export import reward, to_agentrl from harness import db, paths, verify def evaluation(n=100): return {"dataset": "imagenet", "n": n, "content_sha": f"content-{n}", "manifest_sha": f"archive-{n}", "evaluator_version": "v1"} def verdict(): return {"terminal": "PASS", "eval_set": evaluation(), "gates": {"G3": {"verdict": "pass-screen", "kind": "rel_pct", "delta": 1, "tolerance": 5}}, "oracle_ref": {"G3_kind": "rel_pct", "G3_delta": 3, "eval_set": evaluation()}} class RewardEvaluationIdentity(unittest.TestCase): def test_oracle_comparison_and_old_review_do_not_affect_reward(self): v = verdict() expected = reward.compute({k: value for k, value in v.items() if k != "oracle_ref"}) for oracle in (None, {}, {"terminal": "PASS", "G3_delta": -99}, {"terminal": "FAIL", "G3_delta": 99, "latency_ms_mean": 999}, "damaged"): with self.subTest(oracle=oracle): candidate = {**v, "oracle_ref": oracle, "review": True, "blocked_correct": True} self.assertEqual(reward.compute(candidate), expected) self.assertEqual(expected["recipe"], "v0.7") self.assertNotIn("acc_rel", expected["terms"]) self.assertNotIn("lat_rel", expected["terms"]) self.assertNotIn("blocked", expected["terms"]) def test_eval_identity_still_does_not_enable_relative_reward(self): v = verdict() expected = reward.compute(v) for key, value in (("n", 2000), ("content_sha", "different"), ("evaluator_version", "v2")): v["oracle_ref"]["eval_set"][key] = value self.assertEqual(reward.compute(v), expected) v.pop("eval_set") v["continuation"] = {"source_run_id": "old"} self.assertEqual(reward.compute(v), expected) class ExportEvaluationIdentity(unittest.TestCase): def setUp(self): tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) self.root = Path(tmp.name) self.addCleanup(patch.stopall) patch.object(paths, "RUNS", self.root / "runs").start() patch.object(to_agentrl, "BENCH", self.root).start() patch.dict(os.environ, {"BENCH_STATE_DIR": str(self.root / "state")}).start() (self.root / "state").mkdir() self.database = self.root / "state/jobs.sqlite" with sqlite3.connect(self.database) as con: con.executescript(db.SCHEMA) con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('new','job',2,'repair','teacher','PASS')") con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('old','job',2,'repair','teacher','TRANSFERRED')") con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','pending','oracle','PENDING')") self.job = {"run_id": "new", "job_id": "job", "task": "cls", "march": "nash-m", "backend": "ptq", "data": {"dataset": "imagenet", "t1": evaluation()}, "evaluator_version": "v1", "gates": {"required": [], "G1": {}, "G2": {}, "G3": {"kind": "rel_pct", "tol": 5}, "G4": {}}, "oracle_ref": {"run_id": "old", "job_id": "job"}, "continuation": {"source_run_id": "old", "source_job_id": "job"}} self.jd = self.root / "runs/new/job" self.rd = self.jd / "rollouts/02" self.rd.mkdir(parents=True) self.save(self.jd / "job.json", self.job) old = copy.deepcopy(self.job) old.update(run_id="old", data={"dataset": "imagenet", "t1": evaluation(2000)}) old.pop("oracle_ref") old.pop("continuation") self.save(self.root / "runs/old/job/job.json", old) self.save(self.root / "runs/old/job/oracle/result.json", { "terminal": "PASS", "gates": {"G3": {"kind": "rel_pct", "delta": 3}}}) self.save(self.rd / "rollout.json", {"state": "DONE", "mode": "repair", "policy": "teacher", "agent_config_version": 1}) self.save(self.rd / "claim.json", {"terminal": "PASS"}) self.save(self.rd / "stages/08-eval/meta.json", { "state": "OK", "measurements": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}}}) self.save(self.rd / "agent/events.jsonl", {"type": "agent_end", "messages": [ {"role": "user", "content": "Test this model."}, {"role": "assistant", "content": "Completed."}]}) self.save(self.rd / "agent/tools.json", [{"name": "bash"}]) (self.rd / "agent/system.md").write_text("Run the model evaluation.") @staticmethod def save(path, value): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value)) def test_verdict_is_independent_of_absent_pass_fail_or_damaged_oracle(self): expected = verify.verify_rollout(self.rd, write=False) expected.pop("verified_at") old = self.root / "runs/old/job" # Even a damaged historical job must never be opened by verification. (old / "job.json").write_text("damaged historical job") op = old / "oracle/result.json" for value in (None, '{"terminal":"PASS"}', '{"terminal":"FAIL"}', "damaged"): with self.subTest(oracle=value): if value is None: op.unlink(missing_ok=True) else: op.write_text(value) result = verify.verify_rollout(self.rd, write=False) result.pop("verified_at") self.assertEqual(result, expected) self.assertEqual(result["eval_set"]["n"], 100) self.assertNotIn("oracle_ref", result) self.assertFalse(result["review"]) self.assertEqual(result["schema"], "bench.verdict/4") self.assertFalse((self.jd / "oracle").exists()) def test_export_keeps_source_rollout_and_eval_set_and_excludes_transfer(self): self.save(self.rd / "verdict.json", verify.verify_rollout(self.rd, write=False)) manifest = to_agentrl.materialize(["old", "new"], "test", min_agent_version=1) record = json.loads((self.root / "datasets/test/teacher.jsonl").read_text()) self.assertEqual(record["rollout"], 2) self.assertEqual(record["continuation"], {"source_run_id": "old", "source_job_id": "job", "source_rollout": 2}) # A job requirement cannot fabricate a missing measured receipt. self.assertIsNone(record["eval_set"]) self.assertEqual(record["required_eval_set"]["n"], 100) self.assertNotIn("oracle_ref", record) self.assertNotIn("oracle_ref", record["verdict"]) self.assertNotIn("acc_rel", record["reward_terms"]) self.assertFalse(record["sft_eligible"]) # historical repair stays analysis self.assertEqual(manifest["stats"]["scanned"], 1) self.assertEqual(manifest["stats"]["pipeline_written"], 0) self.assertEqual(manifest["stats"]["excluded"], 0) def test_pipeline_export_records_eval_identity(self): with sqlite3.connect(self.database) as con: con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','job','oracle','FAIL')") self.save(self.jd / "oracle/result.json", {"terminal": "FAIL"}) record = next(to_agentrl.pipeline_records(["new"])) self.assertEqual(record["eval_set"]["n"], 100) self.assertEqual(record["continuation"]["source_run_id"], "old") def test_missing_measured_identity_keeps_job_requirement_separate(self): result = verify.verify_rollout(self.rd, write=False) result.pop("eval_set") result.pop("continuation") self.save(self.rd / "verdict.json", result) to_agentrl.materialize(["new"], "missing-identity", min_agent_version=1) record = json.loads((self.root / "datasets/missing-identity/teacher.jsonl").read_text()) self.assertIsNone(record["eval_set"]) self.assertEqual(record["required_eval_set"]["n"], 100) self.assertNotIn("acc_rel", record["reward_terms"]) self.assertNotIn("reward_acc_rel_reason", record) def test_measured_identity_is_preserved_even_when_requirement_differs(self): path = self.rd / 'stages/08-eval/meta.json' recorded = json.loads(path.read_text()) recorded['measurements']['G3']['eval_set'] = evaluation(2000) self.save(path, recorded) self.save(self.rd / 'verdict.json', verify.verify_rollout(self.rd, write=False)) to_agentrl.materialize(['new'], 'different-measured-set', min_agent_version=1) record = json.loads((self.root / 'datasets/different-measured-set/teacher.jsonl').read_text()) self.assertEqual(record['eval_set']['n'], 2000) self.assertEqual(record['required_eval_set']['n'], 100) self.assertFalse(record['sft_eligible']) if __name__ == "__main__": unittest.main()