All 161 task packages, including the 21 batch 6 C++17 tasks that passed
gates 1/2, are now in Git. rollout.catalog builds docs/task-catalog.{json,md}
from the task packages, a fresh trace5 calibration (128/128 PASS), every
verified controller receipt on the current task digest, documented holds
and digest-bound difficulty reviews. 123 tasks are usable (6 HARD),
5 are on hold and 33 still need a subprocess-only grader.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
91 lines
5.2 KiB
Python
91 lines
5.2 KiB
Python
import json
|
||
import shutil
|
||
import unittest
|
||
|
||
import test_core as core_fixture
|
||
from rollout import catalog, core
|
||
|
||
IMAGE = "sha256:" + "a" * 64
|
||
|
||
|
||
class CatalogTests(unittest.TestCase):
|
||
def setUp(self):
|
||
helper = core_fixture.CoreTests("runTest")
|
||
helper.setUp()
|
||
self.addCleanup(helper.doCleanups)
|
||
self.root = helper.root
|
||
self.tasks = self.root / "tasks"
|
||
self.tasks.mkdir()
|
||
for name in ("good", "held", "blocked", "stale"):
|
||
shutil.copytree(helper.task, self.tasks / name)
|
||
manifest = json.loads((self.tasks / name / "task.json").read_text())
|
||
manifest["id"] = name
|
||
(self.tasks / name / "task.json").write_text(json.dumps(manifest))
|
||
(self.tasks / "blocked" / "isolation-blocked.txt").write_text("hidden tests run in the candidate process\n")
|
||
self.calibrations = self.root / "calibrations"
|
||
for name in ("good", "held", "blocked"):
|
||
task = core.load_task(self.tasks / name)
|
||
for image, status in ((IMAGE, "PASS"), ("sha256:" + "b" * 64, "PASS"), (IMAGE, "FAIL")):
|
||
attempt = self.calibrations / ("calibration-%s-%s-%s" % (name, image[-1], status))
|
||
attempt.mkdir(parents=True)
|
||
attempt.joinpath("receipt.json").write_text(json.dumps(dict(
|
||
task_digest=task["_digest"], image_id=image, status=status, cleanup="confirmed",
|
||
trusted_local=False, trials=[{}] * 20)))
|
||
self.digest = core.load_task(self.tasks / "good")["_digest"]
|
||
self.runs = self.root / "runs"
|
||
for index, (model, status, evidence) in enumerate((
|
||
(catalog.QWEN, "PASS", "VERIFIED"), (catalog.QWEN, "FAIL", "VERIFIED"),
|
||
(catalog.QWEN, "FAIL", "VERIFIED"), (catalog.QWEN, "PASS", "UNVERIFIED"),
|
||
("glm-5.3", "PASS", "VERIFIED"))):
|
||
control = self.runs / "batch" / ("pi-%d" % index)
|
||
control.joinpath("run-x", "workspace", "pi-nested").mkdir(parents=True)
|
||
control.joinpath("receipt.json").write_text(json.dumps(dict(
|
||
task_digest=self.digest, model_id=model, status=status, cleanup="confirmed", agent_exit=0,
|
||
pi_evidence=dict(status=evidence))))
|
||
self.holds = self.tasks / "catalog-holds.json"
|
||
self.holds.write_text(json.dumps(dict(held=dict(reason="ambiguous public contract"))))
|
||
|
||
def test_status_comes_from_blocks_holds_and_matching_pass_calibrations(self):
|
||
result = catalog.build(self.tasks, [self.calibrations], [self.runs], self.holds, IMAGE)
|
||
rows = {row["id"]: row for row in result["tasks"]}
|
||
self.assertEqual({name: row["status"] for name, row in rows.items()},
|
||
dict(good="USABLE", held="HOLD", blocked="BLOCKED", stale="UNCALIBRATED"))
|
||
self.assertEqual(rows["good"]["calibration"]["path"], str(self.calibrations / "calibration-good-a-PASS"))
|
||
self.assertEqual(rows["good"]["results"], {catalog.QWEN: dict(PASS=1, FAIL=2), "glm-5.3": dict(PASS=1, FAIL=0)})
|
||
self.assertEqual(rows["good"]["difficulty"], "HARD")
|
||
self.assertEqual(rows["held"]["reason"], "ambiguous public contract")
|
||
self.assertEqual(list(catalog.jobs(result, {"USABLE"}, set())), ["good " + rows["good"]["calibration"]["path"]])
|
||
self.assertEqual(list(catalog.jobs(result, {"USABLE"}, {"ROUTINE"})), [])
|
||
text = catalog.markdown(result)
|
||
self.assertIn("| good | HARD | 1/2 | 1/0 |", text)
|
||
self.assertIn("## HOLD:暂缓使用(1 道)", text)
|
||
self.assertIn("| held | protocol | ambiguous public contract | - |", text)
|
||
self.assertIn("## BLOCKED:判据待重写(1 道)", text)
|
||
self.assertIn("按 RUNNER 方式重写判据", text)
|
||
self.assertNotIn("| stale | HARD", text)
|
||
|
||
def test_reviews_apply_only_to_the_reviewed_identity(self):
|
||
reviews = self.root / "reviews.json"
|
||
digest = core.load_task(self.tasks / "held")["_digest"]
|
||
reviews.write_text(json.dumps(dict(
|
||
good=dict(task_digest=self.digest, difficulty="ROUTINE", results="P,P,P", basis="reviewed"),
|
||
held=dict(task_digest="0" * 64, difficulty="HARD"))))
|
||
rows = {row["id"]: row for row in catalog.build(self.tasks, [self.calibrations], [self.runs], self.holds, IMAGE,
|
||
reviews)["tasks"]}
|
||
self.assertEqual((rows["good"]["difficulty"], rows["good"]["difficulty_source"]), ("ROUTINE", "reviewed"))
|
||
self.assertEqual(rows["good"]["results"][catalog.QWEN], dict(PASS=1, FAIL=2))
|
||
self.assertEqual((rows["held"]["difficulty"], rows["held"]["difficulty_source"]), ("UNMEASURED", "measured"))
|
||
self.assertEqual(rows["held"]["stale_review"]["task_digest"], "0" * 64)
|
||
self.assertNotEqual(digest, "0" * 64)
|
||
|
||
def test_difficulty_buckets(self):
|
||
for counts, expected in ((dict(), "UNMEASURED"), (dict(PASS=1), "PARTIAL"), (dict(PASS=2), "ROUTINE"),
|
||
(dict(PASS=2, FAIL=1), "MEDIUM"), (dict(PASS=1, FAIL=2), "HARD"),
|
||
(dict(FAIL=2), "PARTIAL"), (dict(FAIL=3), "HARD")):
|
||
with self.subTest(counts=counts):
|
||
self.assertEqual(catalog.difficulty(counts), expected)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|