"""Coverage reflects generated plans and exported Agent episodes for the selected run.""" from contextlib import contextmanager import csv import json from pathlib import Path import tempfile from types import SimpleNamespace import unittest from unittest.mock import patch from harness import coverage, db class Coverage(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.root = Path(self.tmp.name) self.db_path = self.root / "jobs.sqlite" self.rows = [{"model_id": name, "task": "cls", "priority": priority, "status": "ready"} for name, priority in [("a", "A"), ("b", "B"), ("c", "C")]] self.addCleanup(patch.stopall) patch.object(db, "connect", self.connect).start() patch.object(coverage, "models", lambda: self.rows).start() patch.object(coverage, "expand_matrix", lambda: [ SimpleNamespace(job_id="a", model="a", mode="e2e", k=4)]).start() patch.object(coverage.paths, "DATASETS", self.root / "datasets").start() patch.object(coverage, "live_line", lambda: []).start() with self.connect() as con: for row in self.rows: con.execute("INSERT INTO jobs (run_id,job_id,model,task,priority,mode,state)" " VALUES ('run',?,?,'cls',?,'oracle','PASS')", (row["model_id"], row["model_id"], row["priority"])) @contextmanager def connect(self): con = db._open(self.db_path) con.executescript(db.SCHEMA) try: yield con con.commit() finally: con.close() def dataset(self, directory, filename, records): path = self.root / "datasets" / directory / filename path.parent.mkdir(parents=True, exist_ok=True) path.write_text("".join(json.dumps(record) + "\n" for record in records)) return path def test_counts_stable_and_legacy_agent_exports_only(self): def record(kk): return {"run_id": "run", "job_id": "b", "rollout": kk, "source": "agent_events", "messages": [{"role": "assistant", "content": "done"}]} self.dataset("run-training", "teacher.jsonl", [record(0)]) self.dataset("run-training", "student.jsonl", [record(1)]) legacy = record(0) legacy.pop("source") self.dataset("ds20260916-01", "teacher.jsonl", [legacy]) self.dataset("run-training", "pipeline.jsonl", [record(2)]) self.dataset("run-training", "exclusions.jsonl", [record(3)]) void = self.dataset("ds20260915-01", "teacher.jsonl", [record(4)]) (void.parent / "VOID").touch() self.dataset("nested/deeper", "teacher.jsonl", [record(5)]) self.assertEqual(coverage.exported_keys(), {("run", "b", 0), ("run", "b", 1)}) def test_run_counts_database_plans_for_b_and_c_without_matrix_regeneration(self): with self.connect() as con: for name in ("b", "c"): for kk in range(4): db.rollout_upsert(con, "run", name, kk, "e2e", "teacher") for kk in (4, 5): db.rollout_upsert(con, "run", "b", kk, "repair", "teacher") db.rollout_upsert(con, "other-run", "c", 7, "e2e", "teacher") actual = coverage.collect("run") self.assertEqual({name: item["planned_rollouts"] for name, item in actual.items()}, {"a": 0, "b": 6, "c": 4}) expected = coverage.collect(None) self.assertEqual({name: item["planned_rollouts"] for name, item in expected.items()}, {"a": 4, "b": 0, "c": 0}) def test_unverified_is_collected_and_verifying_remains_active_in_display_and_csv(self): with self.connect() as con: for kk, state in enumerate(("PASS", "UNVERIFIED", "VERIFYING", "QUEUED")): db.rollout_upsert(con, "run", "b", kk, "e2e", "teacher", state) rendered = coverage.render("run") self.assertIn("计划 4 条 · 已采 2", rendered) self.assertIn("UNVERIFIED 1", rendered) self.assertIn("在采 1(RUNNING 0 / VERIFYING 1)", rendered) path = self.root / "coverage.csv" coverage.to_csv("run", str(path)) with path.open() as stream: row = next(row for row in csv.DictReader(stream) if row["model"] == "b") self.assertEqual(row["rollout_planned"], "4") self.assertEqual(row["rollout_done"], "2") if __name__ == "__main__": unittest.main()