data-gen-ebd/tests/test_catalog.py
amos 913d3c367f Add the usable task catalog and admit batch 6 tasks
All 161 task packages, including the 21 batch 6 C++17 tasks that passed
gates 1/2, are now in Git. rollout.catalog builds docs/task-catalog.{json,md}
from the task packages, a fresh trace5 calibration (128/128 PASS), every
verified controller receipt on the current task digest, documented holds
and digest-bound difficulty reviews. 123 tasks are usable (6 HARD),
5 are on hold and 33 still need a subprocess-only grader.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 17:43:15 +08:00

91 lines
5.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import json
import shutil
import unittest
import test_core as core_fixture
from rollout import catalog, core
IMAGE = "sha256:" + "a" * 64
class CatalogTests(unittest.TestCase):
def setUp(self):
helper = core_fixture.CoreTests("runTest")
helper.setUp()
self.addCleanup(helper.doCleanups)
self.root = helper.root
self.tasks = self.root / "tasks"
self.tasks.mkdir()
for name in ("good", "held", "blocked", "stale"):
shutil.copytree(helper.task, self.tasks / name)
manifest = json.loads((self.tasks / name / "task.json").read_text())
manifest["id"] = name
(self.tasks / name / "task.json").write_text(json.dumps(manifest))
(self.tasks / "blocked" / "isolation-blocked.txt").write_text("hidden tests run in the candidate process\n")
self.calibrations = self.root / "calibrations"
for name in ("good", "held", "blocked"):
task = core.load_task(self.tasks / name)
for image, status in ((IMAGE, "PASS"), ("sha256:" + "b" * 64, "PASS"), (IMAGE, "FAIL")):
attempt = self.calibrations / ("calibration-%s-%s-%s" % (name, image[-1], status))
attempt.mkdir(parents=True)
attempt.joinpath("receipt.json").write_text(json.dumps(dict(
task_digest=task["_digest"], image_id=image, status=status, cleanup="confirmed",
trusted_local=False, trials=[{}] * 20)))
self.digest = core.load_task(self.tasks / "good")["_digest"]
self.runs = self.root / "runs"
for index, (model, status, evidence) in enumerate((
(catalog.QWEN, "PASS", "VERIFIED"), (catalog.QWEN, "FAIL", "VERIFIED"),
(catalog.QWEN, "FAIL", "VERIFIED"), (catalog.QWEN, "PASS", "UNVERIFIED"),
("glm-5.3", "PASS", "VERIFIED"))):
control = self.runs / "batch" / ("pi-%d" % index)
control.joinpath("run-x", "workspace", "pi-nested").mkdir(parents=True)
control.joinpath("receipt.json").write_text(json.dumps(dict(
task_digest=self.digest, model_id=model, status=status, cleanup="confirmed", agent_exit=0,
pi_evidence=dict(status=evidence))))
self.holds = self.tasks / "catalog-holds.json"
self.holds.write_text(json.dumps(dict(held=dict(reason="ambiguous public contract"))))
def test_status_comes_from_blocks_holds_and_matching_pass_calibrations(self):
result = catalog.build(self.tasks, [self.calibrations], [self.runs], self.holds, IMAGE)
rows = {row["id"]: row for row in result["tasks"]}
self.assertEqual({name: row["status"] for name, row in rows.items()},
dict(good="USABLE", held="HOLD", blocked="BLOCKED", stale="UNCALIBRATED"))
self.assertEqual(rows["good"]["calibration"]["path"], str(self.calibrations / "calibration-good-a-PASS"))
self.assertEqual(rows["good"]["results"], {catalog.QWEN: dict(PASS=1, FAIL=2), "glm-5.3": dict(PASS=1, FAIL=0)})
self.assertEqual(rows["good"]["difficulty"], "HARD")
self.assertEqual(rows["held"]["reason"], "ambiguous public contract")
self.assertEqual(list(catalog.jobs(result, {"USABLE"}, set())), ["good " + rows["good"]["calibration"]["path"]])
self.assertEqual(list(catalog.jobs(result, {"USABLE"}, {"ROUTINE"})), [])
text = catalog.markdown(result)
self.assertIn("| good | HARD | 1/2 | 1/0 |", text)
self.assertIn("## HOLD:暂缓使用(1 道)", text)
self.assertIn("| held | protocol | ambiguous public contract | - |", text)
self.assertIn("## BLOCKED:判据待重写(1 道)", text)
self.assertIn("按 RUNNER 方式重写判据", text)
self.assertNotIn("| stale | HARD", text)
def test_reviews_apply_only_to_the_reviewed_identity(self):
reviews = self.root / "reviews.json"
digest = core.load_task(self.tasks / "held")["_digest"]
reviews.write_text(json.dumps(dict(
good=dict(task_digest=self.digest, difficulty="ROUTINE", results="P,P,P", basis="reviewed"),
held=dict(task_digest="0" * 64, difficulty="HARD"))))
rows = {row["id"]: row for row in catalog.build(self.tasks, [self.calibrations], [self.runs], self.holds, IMAGE,
reviews)["tasks"]}
self.assertEqual((rows["good"]["difficulty"], rows["good"]["difficulty_source"]), ("ROUTINE", "reviewed"))
self.assertEqual(rows["good"]["results"][catalog.QWEN], dict(PASS=1, FAIL=2))
self.assertEqual((rows["held"]["difficulty"], rows["held"]["difficulty_source"]), ("UNMEASURED", "measured"))
self.assertEqual(rows["held"]["stale_review"]["task_digest"], "0" * 64)
self.assertNotEqual(digest, "0" * 64)
def test_difficulty_buckets(self):
for counts, expected in ((dict(), "UNMEASURED"), (dict(PASS=1), "PARTIAL"), (dict(PASS=2), "ROUTINE"),
(dict(PASS=2, FAIL=1), "MEDIUM"), (dict(PASS=1, FAIL=2), "HARD"),
(dict(FAIL=2), "PARTIAL"), (dict(FAIL=3), "HARD")):
with self.subTest(counts=counts):
self.assertEqual(catalog.difficulty(counts), expected)
if __name__ == "__main__":
unittest.main()