147 lines
7.6 KiB
Python
147 lines
7.6 KiB
Python
"""A continued Agent keeps its source evidence without mixing evaluation sets."""
|
|
import copy
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from export import reward, to_agentrl
|
|
from harness import db, paths, verify
|
|
|
|
|
|
def evaluation(n=100):
|
|
return {"dataset": "imagenet", "n": n, "content_sha": f"content-{n}",
|
|
"manifest_sha": f"archive-{n}", "evaluator_version": "v1"}
|
|
|
|
|
|
def verdict():
|
|
return {"terminal": "PASS", "eval_set": evaluation(),
|
|
"gates": {"G3": {"verdict": "pass-screen", "kind": "rel_pct", "delta": 1, "tolerance": 5}},
|
|
"oracle_ref": {"G3_kind": "rel_pct", "G3_delta": 3, "eval_set": evaluation()}}
|
|
|
|
|
|
class RewardEvaluationIdentity(unittest.TestCase):
|
|
def test_same_content_is_comparable_even_when_archive_changes(self):
|
|
v = verdict()
|
|
v["oracle_ref"]["eval_set"]["manifest_sha"] = "repacked"
|
|
result = reward.compute(v)
|
|
self.assertEqual(result["terms"]["acc_rel"], .4)
|
|
self.assertIsNone(result["acc_rel_reason"])
|
|
|
|
def test_different_set_or_evaluator_does_not_reward_accuracy_improvement(self):
|
|
for field, value in (("n", 2000), ("content_sha", "another-100"), ("dataset", "coco"),
|
|
("evaluator_version", "v2")):
|
|
with self.subTest(field=field):
|
|
v = verdict()
|
|
v["oracle_ref"]["eval_set"][field] = value
|
|
result = reward.compute(v)
|
|
self.assertEqual(result["terms"]["acc_rel"], 0)
|
|
self.assertEqual(result["acc_rel_reason"], "eval_set_mismatch")
|
|
|
|
def test_missing_identity_is_not_comparable_for_continuation(self):
|
|
v = verdict()
|
|
v.pop("eval_set")
|
|
v["oracle_ref"].pop("eval_set")
|
|
self.assertEqual(reward.compute(v)["terms"]["acc_rel"], .4) # legacy
|
|
v["continuation"] = {"source_run_id": "old", "source_job_id": "job"}
|
|
result = reward.compute(v)
|
|
self.assertEqual(result["terms"]["acc_rel"], 0)
|
|
self.assertEqual(result["acc_rel_reason"], "missing_eval_set")
|
|
v["eval_set"] = evaluation()
|
|
self.assertEqual(reward.compute(v)["acc_rel_reason"], "missing_eval_set")
|
|
|
|
|
|
class ExportEvaluationIdentity(unittest.TestCase):
|
|
def setUp(self):
|
|
tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(tmp.cleanup)
|
|
self.root = Path(tmp.name)
|
|
self.addCleanup(patch.stopall)
|
|
patch.object(paths, "RUNS", self.root / "runs").start()
|
|
patch.object(to_agentrl, "BENCH", self.root).start()
|
|
patch.dict(os.environ, {"BENCH_STATE_DIR": str(self.root / "state")}).start()
|
|
(self.root / "state").mkdir()
|
|
self.database = self.root / "state/jobs.sqlite"
|
|
with sqlite3.connect(self.database) as con:
|
|
con.executescript(db.SCHEMA)
|
|
con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('new','job',2,'repair','teacher','PASS')")
|
|
con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('old','job',2,'repair','teacher','TRANSFERRED')")
|
|
con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','pending','oracle','PENDING')")
|
|
self.job = {"run_id": "new", "job_id": "job", "task": "cls", "march": "nash-m", "backend": "ptq",
|
|
"data": {"dataset": "imagenet", "t1": evaluation()}, "evaluator_version": "v1",
|
|
"gates": {"required": [], "G1": {}, "G2": {}, "G3": {"kind": "rel_pct", "tol": 5}, "G4": {}},
|
|
"oracle_ref": {"run_id": "old", "job_id": "job"},
|
|
"continuation": {"source_run_id": "old", "source_job_id": "job"}}
|
|
self.jd = self.root / "runs/new/job"
|
|
self.rd = self.jd / "rollouts/02"
|
|
self.rd.mkdir(parents=True)
|
|
self.save(self.jd / "job.json", self.job)
|
|
old = copy.deepcopy(self.job)
|
|
old.update(run_id="old", data={"dataset": "imagenet", "t1": evaluation(2000)})
|
|
old.pop("oracle_ref")
|
|
old.pop("continuation")
|
|
self.save(self.root / "runs/old/job/job.json", old)
|
|
self.save(self.root / "runs/old/job/oracle/result.json", {
|
|
"terminal": "PASS", "gates": {"G3": {"kind": "rel_pct", "delta": 3}}})
|
|
self.save(self.rd / "rollout.json", {"state": "DONE", "mode": "repair", "policy": "teacher", "agent_config_version": 1})
|
|
self.save(self.rd / "claim.json", {"terminal": "PASS"})
|
|
self.save(self.rd / "stages/08-eval/meta.json", {
|
|
"state": "OK", "measurements": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}}})
|
|
self.save(self.rd / "agent/events.jsonl", {"type": "agent_end", "messages": [
|
|
{"role": "user", "content": "Test this model."}, {"role": "assistant", "content": "Completed."}]})
|
|
self.save(self.rd / "agent/tools.json", [{"name": "bash"}])
|
|
(self.rd / "agent/system.md").write_text("Run the model evaluation.")
|
|
|
|
@staticmethod
|
|
def save(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value))
|
|
|
|
def test_verdict_preserves_source_identity_without_copying_oracle(self):
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(result["eval_set"]["n"], 100)
|
|
self.assertEqual(result["oracle_ref"]["eval_set"]["n"], 2000)
|
|
self.assertEqual((result["oracle_ref"]["run_id"], result["oracle_ref"]["job_id"]), ("old", "job"))
|
|
self.assertFalse((self.jd / "oracle").exists())
|
|
self.assertEqual(reward.compute(result)["acc_rel_reason"], "eval_set_mismatch")
|
|
|
|
def test_export_keeps_source_rollout_and_eval_set_and_excludes_transfer(self):
|
|
self.save(self.rd / "verdict.json", verify.verify_rollout(self.rd, write=False))
|
|
manifest = to_agentrl.materialize(["old", "new"], "test", min_agent_version=1)
|
|
record = json.loads((self.root / "datasets/test/teacher.jsonl").read_text())
|
|
self.assertEqual(record["rollout"], 2)
|
|
self.assertEqual(record["continuation"], {"source_run_id": "old", "source_job_id": "job", "source_rollout": 2})
|
|
self.assertEqual(record["eval_set"]["n"], 100)
|
|
self.assertEqual(record["oracle_ref"]["eval_set"]["n"], 2000)
|
|
self.assertEqual(record["reward_terms"]["acc_rel"], 0)
|
|
self.assertEqual(record["reward_acc_rel_reason"], "eval_set_mismatch")
|
|
self.assertEqual(manifest["stats"]["scanned"], 1)
|
|
self.assertEqual(manifest["stats"]["pipeline_written"], 0)
|
|
self.assertEqual(manifest["stats"]["excluded"], 0)
|
|
|
|
def test_pipeline_export_records_eval_identity(self):
|
|
with sqlite3.connect(self.database) as con:
|
|
con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','job','oracle','FAIL')")
|
|
self.save(self.jd / "oracle/result.json", {"terminal": "FAIL"})
|
|
record = next(to_agentrl.pipeline_records(["new"]))
|
|
self.assertEqual(record["eval_set"]["n"], 100)
|
|
self.assertEqual(record["continuation"]["source_run_id"], "old")
|
|
|
|
def test_export_missing_verdict_identity_cannot_use_legacy_comparison(self):
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
result.pop("eval_set")
|
|
result.pop("continuation")
|
|
result["oracle_ref"].pop("eval_set")
|
|
self.save(self.rd / "verdict.json", result)
|
|
to_agentrl.materialize(["new"], "missing-identity", min_agent_version=1)
|
|
record = json.loads((self.root / "datasets/missing-identity/teacher.jsonl").read_text())
|
|
self.assertEqual(record["reward_terms"]["acc_rel"], 0)
|
|
self.assertEqual(record["reward_acc_rel_reason"], "missing_eval_set")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|