156 lines
8.1 KiB
Python
156 lines
8.1 KiB
Python
"""A continued Agent keeps its source evidence without mixing evaluation sets."""
|
|
import copy
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from export import reward, to_agentrl
|
|
from harness import db, paths, verify
|
|
|
|
|
|
def evaluation(n=100):
|
|
return {"dataset": "imagenet", "n": n, "content_sha": f"content-{n}",
|
|
"manifest_sha": f"archive-{n}", "evaluator_version": "v1"}
|
|
|
|
|
|
def verdict():
|
|
return {"terminal": "PASS", "eval_set": evaluation(),
|
|
"gates": {"G3": {"verdict": "pass-screen", "kind": "rel_pct", "delta": 1, "tolerance": 5}},
|
|
"oracle_ref": {"G3_kind": "rel_pct", "G3_delta": 3, "eval_set": evaluation()}}
|
|
|
|
|
|
class RewardEvaluationIdentity(unittest.TestCase):
|
|
def test_oracle_comparison_and_old_review_do_not_affect_reward(self):
|
|
v = verdict()
|
|
expected = reward.compute({k: value for k, value in v.items() if k != "oracle_ref"})
|
|
for oracle in (None, {}, {"terminal": "PASS", "G3_delta": -99},
|
|
{"terminal": "FAIL", "G3_delta": 99, "latency_ms_mean": 999}, "damaged"):
|
|
with self.subTest(oracle=oracle):
|
|
candidate = {**v, "oracle_ref": oracle, "review": True, "blocked_correct": True}
|
|
self.assertEqual(reward.compute(candidate), expected)
|
|
self.assertEqual(expected["recipe"], "v0.6")
|
|
self.assertNotIn("acc_rel", expected["terms"])
|
|
self.assertNotIn("lat_rel", expected["terms"])
|
|
self.assertNotIn("blocked", expected["terms"])
|
|
|
|
def test_eval_identity_still_does_not_enable_relative_reward(self):
|
|
v = verdict()
|
|
expected = reward.compute(v)
|
|
for key, value in (("n", 2000), ("content_sha", "different"), ("evaluator_version", "v2")):
|
|
v["oracle_ref"]["eval_set"][key] = value
|
|
self.assertEqual(reward.compute(v), expected)
|
|
v.pop("eval_set")
|
|
v["continuation"] = {"source_run_id": "old"}
|
|
self.assertEqual(reward.compute(v), expected)
|
|
|
|
|
|
class ExportEvaluationIdentity(unittest.TestCase):
|
|
def setUp(self):
|
|
tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(tmp.cleanup)
|
|
self.root = Path(tmp.name)
|
|
self.addCleanup(patch.stopall)
|
|
patch.object(paths, "RUNS", self.root / "runs").start()
|
|
patch.object(to_agentrl, "BENCH", self.root).start()
|
|
patch.dict(os.environ, {"BENCH_STATE_DIR": str(self.root / "state")}).start()
|
|
(self.root / "state").mkdir()
|
|
self.database = self.root / "state/jobs.sqlite"
|
|
with sqlite3.connect(self.database) as con:
|
|
con.executescript(db.SCHEMA)
|
|
con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('new','job',2,'repair','teacher','PASS')")
|
|
con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('old','job',2,'repair','teacher','TRANSFERRED')")
|
|
con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','pending','oracle','PENDING')")
|
|
self.job = {"run_id": "new", "job_id": "job", "task": "cls", "march": "nash-m", "backend": "ptq",
|
|
"data": {"dataset": "imagenet", "t1": evaluation()}, "evaluator_version": "v1",
|
|
"gates": {"required": [], "G1": {}, "G2": {}, "G3": {"kind": "rel_pct", "tol": 5}, "G4": {}},
|
|
"oracle_ref": {"run_id": "old", "job_id": "job"},
|
|
"continuation": {"source_run_id": "old", "source_job_id": "job"}}
|
|
self.jd = self.root / "runs/new/job"
|
|
self.rd = self.jd / "rollouts/02"
|
|
self.rd.mkdir(parents=True)
|
|
self.save(self.jd / "job.json", self.job)
|
|
old = copy.deepcopy(self.job)
|
|
old.update(run_id="old", data={"dataset": "imagenet", "t1": evaluation(2000)})
|
|
old.pop("oracle_ref")
|
|
old.pop("continuation")
|
|
self.save(self.root / "runs/old/job/job.json", old)
|
|
self.save(self.root / "runs/old/job/oracle/result.json", {
|
|
"terminal": "PASS", "gates": {"G3": {"kind": "rel_pct", "delta": 3}}})
|
|
self.save(self.rd / "rollout.json", {"state": "DONE", "mode": "repair", "policy": "teacher", "agent_config_version": 1})
|
|
self.save(self.rd / "claim.json", {"terminal": "PASS"})
|
|
self.save(self.rd / "stages/08-eval/meta.json", {
|
|
"state": "OK", "measurements": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}}})
|
|
self.save(self.rd / "agent/events.jsonl", {"type": "agent_end", "messages": [
|
|
{"role": "user", "content": "Test this model."}, {"role": "assistant", "content": "Completed."}]})
|
|
self.save(self.rd / "agent/tools.json", [{"name": "bash"}])
|
|
(self.rd / "agent/system.md").write_text("Run the model evaluation.")
|
|
|
|
@staticmethod
|
|
def save(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value))
|
|
|
|
def test_verdict_is_independent_of_absent_pass_fail_or_damaged_oracle(self):
|
|
expected = verify.verify_rollout(self.rd, write=False)
|
|
expected.pop("verified_at")
|
|
old = self.root / "runs/old/job"
|
|
# Even a damaged historical job must never be opened by verification.
|
|
(old / "job.json").write_text("damaged historical job")
|
|
op = old / "oracle/result.json"
|
|
for value in (None, '{"terminal":"PASS"}', '{"terminal":"FAIL"}', "damaged"):
|
|
with self.subTest(oracle=value):
|
|
if value is None:
|
|
op.unlink(missing_ok=True)
|
|
else:
|
|
op.write_text(value)
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
result.pop("verified_at")
|
|
self.assertEqual(result, expected)
|
|
self.assertEqual(result["eval_set"]["n"], 100)
|
|
self.assertNotIn("oracle_ref", result)
|
|
self.assertFalse(result["review"])
|
|
self.assertEqual(result["schema"], "bench.verdict/3")
|
|
self.assertFalse((self.jd / "oracle").exists())
|
|
|
|
def test_export_keeps_source_rollout_and_eval_set_and_excludes_transfer(self):
|
|
self.save(self.rd / "verdict.json", verify.verify_rollout(self.rd, write=False))
|
|
manifest = to_agentrl.materialize(["old", "new"], "test", min_agent_version=1)
|
|
record = json.loads((self.root / "datasets/test/teacher.jsonl").read_text())
|
|
self.assertEqual(record["rollout"], 2)
|
|
self.assertEqual(record["continuation"], {"source_run_id": "old", "source_job_id": "job", "source_rollout": 2})
|
|
self.assertEqual(record["eval_set"]["n"], 100)
|
|
self.assertNotIn("oracle_ref", record)
|
|
self.assertNotIn("oracle_ref", record["verdict"])
|
|
self.assertNotIn("acc_rel", record["reward_terms"])
|
|
self.assertFalse(record["sft_eligible"]) # historical repair stays analysis
|
|
self.assertEqual(manifest["stats"]["scanned"], 1)
|
|
self.assertEqual(manifest["stats"]["pipeline_written"], 0)
|
|
self.assertEqual(manifest["stats"]["excluded"], 0)
|
|
|
|
def test_pipeline_export_records_eval_identity(self):
|
|
with sqlite3.connect(self.database) as con:
|
|
con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','job','oracle','FAIL')")
|
|
self.save(self.jd / "oracle/result.json", {"terminal": "FAIL"})
|
|
record = next(to_agentrl.pipeline_records(["new"]))
|
|
self.assertEqual(record["eval_set"]["n"], 100)
|
|
self.assertEqual(record["continuation"]["source_run_id"], "old")
|
|
|
|
def test_export_missing_verdict_identity_uses_current_job_identity(self):
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
result.pop("eval_set")
|
|
result.pop("continuation")
|
|
self.save(self.rd / "verdict.json", result)
|
|
to_agentrl.materialize(["new"], "missing-identity", min_agent_version=1)
|
|
record = json.loads((self.root / "datasets/missing-identity/teacher.jsonl").read_text())
|
|
self.assertEqual(record["eval_set"]["n"], 100)
|
|
self.assertNotIn("acc_rel", record["reward_terms"])
|
|
self.assertNotIn("reward_acc_rel_reason", record)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|