data-gen-ebd/tests/test_catalog.py
amos fad2b1d204 Recalibrate the task catalog on trace6
Calibrate all 128 non-blocked tasks on the trace6 image (128/128 PASS,
cleanup confirmed) and regenerate docs/task-catalog.{json,md} against
those receipts. Count each controller attempt once in the catalog: batch
6 now sits in both the hot and cold run roots, which doubled nine tasks'
Qwen counts. The trace6 Qwen smoke is Codex's batch-6 run, re-verified
read-only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 01:04:33 +08:00

93 lines
5.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import json
import shutil
import unittest
import test_core as core_fixture
from rollout import catalog, core
IMAGE = "sha256:" + "a" * 64
class CatalogTests(unittest.TestCase):
def setUp(self):
helper = core_fixture.CoreTests("runTest")
helper.setUp()
self.addCleanup(helper.doCleanups)
self.root = helper.root
self.tasks = self.root / "tasks"
self.tasks.mkdir()
for name in ("good", "held", "blocked", "stale"):
shutil.copytree(helper.task, self.tasks / name)
manifest = json.loads((self.tasks / name / "task.json").read_text())
manifest["id"] = name
(self.tasks / name / "task.json").write_text(json.dumps(manifest))
(self.tasks / "blocked" / "isolation-blocked.txt").write_text("hidden tests run in the candidate process\n")
self.calibrations = self.root / "calibrations"
for name in ("good", "held", "blocked"):
task = core.load_task(self.tasks / name)
for image, status in ((IMAGE, "PASS"), ("sha256:" + "b" * 64, "PASS"), (IMAGE, "FAIL")):
attempt = self.calibrations / ("calibration-%s-%s-%s" % (name, image[-1], status))
attempt.mkdir(parents=True)
attempt.joinpath("receipt.json").write_text(json.dumps(dict(
task_digest=task["_digest"], image_id=image, status=status, cleanup="confirmed",
trusted_local=False, trials=[{}] * 20)))
self.digest = core.load_task(self.tasks / "good")["_digest"]
self.runs = self.root / "runs"
for index, (model, status, evidence) in enumerate((
(catalog.QWEN, "PASS", "VERIFIED"), (catalog.QWEN, "FAIL", "VERIFIED"),
(catalog.QWEN, "FAIL", "VERIFIED"), (catalog.QWEN, "PASS", "UNVERIFIED"),
("glm-5.3", "PASS", "VERIFIED"))):
control = self.runs / "batch" / ("pi-%d" % index)
control.joinpath("run-x", "workspace", "pi-nested").mkdir(parents=True)
control.joinpath("receipt.json").write_text(json.dumps(dict(
task_digest=self.digest, model_id=model, status=status, cleanup="confirmed", agent_exit=0,
pi_evidence=dict(status=evidence))))
self.holds = self.tasks / "catalog-holds.json"
self.holds.write_text(json.dumps(dict(held=dict(reason="ambiguous public contract"))))
def test_status_comes_from_blocks_holds_and_matching_pass_calibrations(self):
cold = self.root / "cold"
shutil.copytree(self.runs, cold)
result = catalog.build(self.tasks, [self.calibrations], [self.runs, cold], self.holds, IMAGE)
rows = {row["id"]: row for row in result["tasks"]}
self.assertEqual({name: row["status"] for name, row in rows.items()},
dict(good="USABLE", held="HOLD", blocked="BLOCKED", stale="UNCALIBRATED"))
self.assertEqual(rows["good"]["calibration"]["path"], str(self.calibrations / "calibration-good-a-PASS"))
self.assertEqual(rows["good"]["results"], {catalog.QWEN: dict(PASS=1, FAIL=2), "glm-5.3": dict(PASS=1, FAIL=0)})
self.assertEqual(rows["good"]["difficulty"], "HARD")
self.assertEqual(rows["held"]["reason"], "ambiguous public contract")
self.assertEqual(list(catalog.jobs(result, {"USABLE"}, set())), ["good " + rows["good"]["calibration"]["path"]])
self.assertEqual(list(catalog.jobs(result, {"USABLE"}, {"ROUTINE"})), [])
text = catalog.markdown(result)
self.assertIn("| good | HARD | 1/2 | 1/0 |", text)
self.assertIn("## HOLD:暂缓使用(1 道)", text)
self.assertIn("| held | protocol | ambiguous public contract | - |", text)
self.assertIn("## BLOCKED:判据待重写(1 道)", text)
self.assertIn("按 RUNNER 方式重写判据", text)
self.assertNotIn("| stale | HARD", text)
def test_reviews_apply_only_to_the_reviewed_identity(self):
reviews = self.root / "reviews.json"
digest = core.load_task(self.tasks / "held")["_digest"]
reviews.write_text(json.dumps(dict(
good=dict(task_digest=self.digest, difficulty="ROUTINE", results="P,P,P", basis="reviewed"),
held=dict(task_digest="0" * 64, difficulty="HARD"))))
rows = {row["id"]: row for row in catalog.build(self.tasks, [self.calibrations], [self.runs], self.holds, IMAGE,
reviews)["tasks"]}
self.assertEqual((rows["good"]["difficulty"], rows["good"]["difficulty_source"]), ("ROUTINE", "reviewed"))
self.assertEqual(rows["good"]["results"][catalog.QWEN], dict(PASS=1, FAIL=2))
self.assertEqual((rows["held"]["difficulty"], rows["held"]["difficulty_source"]), ("UNMEASURED", "measured"))
self.assertEqual(rows["held"]["stale_review"]["task_digest"], "0" * 64)
self.assertNotEqual(digest, "0" * 64)
def test_difficulty_buckets(self):
for counts, expected in ((dict(), "UNMEASURED"), (dict(PASS=1), "PARTIAL"), (dict(PASS=2), "ROUTINE"),
(dict(PASS=2, FAIL=1), "MEDIUM"), (dict(PASS=1, FAIL=2), "HARD"),
(dict(FAIL=2), "PARTIAL"), (dict(FAIL=3), "HARD")):
with self.subTest(counts=counts):
self.assertEqual(catalog.difficulty(counts), expected)
if __name__ == "__main__":
unittest.main()