import json import shutil import unittest import test_core as core_fixture from rollout import catalog, core IMAGE = "sha256:" + "a" * 64 class CatalogTests(unittest.TestCase): def setUp(self): helper = core_fixture.CoreTests("runTest") helper.setUp() self.addCleanup(helper.doCleanups) self.root = helper.root self.tasks = self.root / "tasks" self.tasks.mkdir() for name in ("good", "held", "blocked", "stale"): shutil.copytree(helper.task, self.tasks / name) manifest = json.loads((self.tasks / name / "task.json").read_text()) manifest["id"] = name (self.tasks / name / "task.json").write_text(json.dumps(manifest)) (self.tasks / "blocked" / "isolation-blocked.txt").write_text("hidden tests run in the candidate process\n") self.calibrations = self.root / "calibrations" for name in ("good", "held", "blocked"): task = core.load_task(self.tasks / name) for image, status in ((IMAGE, "PASS"), ("sha256:" + "b" * 64, "PASS"), (IMAGE, "FAIL")): attempt = self.calibrations / ("calibration-%s-%s-%s" % (name, image[-1], status)) attempt.mkdir(parents=True) attempt.joinpath("receipt.json").write_text(json.dumps(dict( task_digest=task["_digest"], image_id=image, status=status, cleanup="confirmed", trusted_local=False, trials=[{}] * 20))) self.digest = core.load_task(self.tasks / "good")["_digest"] self.runs = self.root / "runs" for index, (model, status, evidence) in enumerate(( (catalog.QWEN, "PASS", "VERIFIED"), (catalog.QWEN, "FAIL", "VERIFIED"), (catalog.QWEN, "FAIL", "VERIFIED"), (catalog.QWEN, "PASS", "UNVERIFIED"), ("glm-5.3", "PASS", "VERIFIED"))): control = self.runs / "batch" / ("pi-%d" % index) control.joinpath("run-x", "workspace", "pi-nested").mkdir(parents=True) control.joinpath("receipt.json").write_text(json.dumps(dict( task_digest=self.digest, model_id=model, status=status, cleanup="confirmed", agent_exit=0, pi_evidence=dict(status=evidence)))) self.holds = self.tasks / "catalog-holds.json" self.holds.write_text(json.dumps(dict(held=dict(reason="ambiguous public contract")))) def test_status_comes_from_blocks_holds_and_matching_pass_calibrations(self): result = catalog.build(self.tasks, [self.calibrations], [self.runs], self.holds, IMAGE) rows = {row["id"]: row for row in result["tasks"]} self.assertEqual({name: row["status"] for name, row in rows.items()}, dict(good="USABLE", held="HOLD", blocked="BLOCKED", stale="UNCALIBRATED")) self.assertEqual(rows["good"]["calibration"]["path"], str(self.calibrations / "calibration-good-a-PASS")) self.assertEqual(rows["good"]["results"], {catalog.QWEN: dict(PASS=1, FAIL=2), "glm-5.3": dict(PASS=1, FAIL=0)}) self.assertEqual(rows["good"]["difficulty"], "HARD") self.assertEqual(rows["held"]["reason"], "ambiguous public contract") self.assertEqual(list(catalog.jobs(result, {"USABLE"}, set())), ["good " + rows["good"]["calibration"]["path"]]) self.assertEqual(list(catalog.jobs(result, {"USABLE"}, {"ROUTINE"})), []) text = catalog.markdown(result) self.assertIn("| good | HARD | 1/2 | 1/0 |", text) self.assertIn("## HOLD:暂缓使用(1 道)", text) self.assertIn("| held | protocol | ambiguous public contract | - |", text) self.assertIn("## BLOCKED:判据待重写(1 道)", text) self.assertIn("按 RUNNER 方式重写判据", text) self.assertNotIn("| stale | HARD", text) def test_reviews_apply_only_to_the_reviewed_identity(self): reviews = self.root / "reviews.json" digest = core.load_task(self.tasks / "held")["_digest"] reviews.write_text(json.dumps(dict( good=dict(task_digest=self.digest, difficulty="ROUTINE", results="P,P,P", basis="reviewed"), held=dict(task_digest="0" * 64, difficulty="HARD")))) rows = {row["id"]: row for row in catalog.build(self.tasks, [self.calibrations], [self.runs], self.holds, IMAGE, reviews)["tasks"]} self.assertEqual((rows["good"]["difficulty"], rows["good"]["difficulty_source"]), ("ROUTINE", "reviewed")) self.assertEqual(rows["good"]["results"][catalog.QWEN], dict(PASS=1, FAIL=2)) self.assertEqual((rows["held"]["difficulty"], rows["held"]["difficulty_source"]), ("UNMEASURED", "measured")) self.assertEqual(rows["held"]["stale_review"]["task_digest"], "0" * 64) self.assertNotEqual(digest, "0" * 64) def test_difficulty_buckets(self): for counts, expected in ((dict(), "UNMEASURED"), (dict(PASS=1), "PARTIAL"), (dict(PASS=2), "ROUTINE"), (dict(PASS=2, FAIL=1), "MEDIUM"), (dict(PASS=1, FAIL=2), "HARD"), (dict(FAIL=2), "PARTIAL"), (dict(FAIL=3), "HARD")): with self.subTest(counts=counts): self.assertEqual(catalog.difficulty(counts), expected) if __name__ == "__main__": unittest.main()