100 lines
4.5 KiB
Python
100 lines
4.5 KiB
Python
"""Coverage reflects generated plans and exported Agent episodes for the selected run."""
|
||
from contextlib import contextmanager
|
||
import csv
|
||
import json
|
||
from pathlib import Path
|
||
import tempfile
|
||
from types import SimpleNamespace
|
||
import unittest
|
||
from unittest.mock import patch
|
||
|
||
from harness import coverage, db
|
||
|
||
|
||
class Coverage(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(self.tmp.cleanup)
|
||
self.root = Path(self.tmp.name)
|
||
self.db_path = self.root / "jobs.sqlite"
|
||
self.rows = [{"model_id": name, "task": "cls", "priority": priority, "status": "ready"}
|
||
for name, priority in [("a", "A"), ("b", "B"), ("c", "C")]]
|
||
self.addCleanup(patch.stopall)
|
||
patch.object(db, "connect", self.connect).start()
|
||
patch.object(coverage, "models", lambda: self.rows).start()
|
||
patch.object(coverage, "expand_matrix", lambda: [
|
||
SimpleNamespace(job_id="a", model="a", mode="e2e", k=4)]).start()
|
||
patch.object(coverage.paths, "DATASETS", self.root / "datasets").start()
|
||
patch.object(coverage, "live_line", lambda: []).start()
|
||
with self.connect() as con:
|
||
for row in self.rows:
|
||
con.execute("INSERT INTO jobs (run_id,job_id,model,task,priority,mode,state)"
|
||
" VALUES ('run',?,?,'cls',?,'oracle','PASS')",
|
||
(row["model_id"], row["model_id"], row["priority"]))
|
||
|
||
@contextmanager
|
||
def connect(self):
|
||
con = db._open(self.db_path)
|
||
con.executescript(db.SCHEMA)
|
||
try:
|
||
yield con
|
||
con.commit()
|
||
finally:
|
||
con.close()
|
||
|
||
def dataset(self, directory, filename, records):
|
||
path = self.root / "datasets" / directory / filename
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text("".join(json.dumps(record) + "\n" for record in records))
|
||
return path
|
||
|
||
def test_counts_stable_and_legacy_agent_exports_only(self):
|
||
def record(kk):
|
||
return {"run_id": "run", "job_id": "b", "rollout": kk,
|
||
"source": "agent_events", "messages": [{"role": "assistant", "content": "done"}]}
|
||
self.dataset("run-training", "teacher.jsonl", [record(0)])
|
||
self.dataset("run-training", "student.jsonl", [record(1)])
|
||
legacy = record(0)
|
||
legacy.pop("source")
|
||
self.dataset("ds20260916-01", "teacher.jsonl", [legacy])
|
||
self.dataset("run-training", "pipeline.jsonl", [record(2)])
|
||
self.dataset("run-training", "exclusions.jsonl", [record(3)])
|
||
void = self.dataset("ds20260915-01", "teacher.jsonl", [record(4)])
|
||
(void.parent / "VOID").touch()
|
||
self.dataset("nested/deeper", "teacher.jsonl", [record(5)])
|
||
self.assertEqual(coverage.exported_keys(), {("run", "b", 0), ("run", "b", 1)})
|
||
|
||
def test_run_counts_database_plans_for_b_and_c_without_matrix_regeneration(self):
|
||
with self.connect() as con:
|
||
for name in ("b", "c"):
|
||
for kk in range(4):
|
||
db.rollout_upsert(con, "run", name, kk, "e2e", "teacher")
|
||
for kk in (4, 5):
|
||
db.rollout_upsert(con, "run", "b", kk, "repair", "teacher")
|
||
db.rollout_upsert(con, "other-run", "c", 7, "e2e", "teacher")
|
||
actual = coverage.collect("run")
|
||
self.assertEqual({name: item["planned_rollouts"] for name, item in actual.items()},
|
||
{"a": 0, "b": 6, "c": 4})
|
||
expected = coverage.collect(None)
|
||
self.assertEqual({name: item["planned_rollouts"] for name, item in expected.items()},
|
||
{"a": 4, "b": 0, "c": 0})
|
||
|
||
def test_unverified_is_collected_and_verifying_remains_active_in_display_and_csv(self):
|
||
with self.connect() as con:
|
||
for kk, state in enumerate(("PASS", "UNVERIFIED", "VERIFYING", "QUEUED")):
|
||
db.rollout_upsert(con, "run", "b", kk, "e2e", "teacher", state)
|
||
rendered = coverage.render("run")
|
||
self.assertIn("计划 4 条 · 已采 2", rendered)
|
||
self.assertIn("UNVERIFIED 1", rendered)
|
||
self.assertIn("在采 1(RUNNING 0 / VERIFYING 1)", rendered)
|
||
path = self.root / "coverage.csv"
|
||
coverage.to_csv("run", str(path))
|
||
with path.open() as stream:
|
||
row = next(row for row in csv.DictReader(stream) if row["model"] == "b")
|
||
self.assertEqual(row["rollout_planned"], "4")
|
||
self.assertEqual(row["rollout_done"], "2")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|