data-gen-bpu/scripts/test_coverage.py

120 lines
5.7 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Coverage reflects generated plans and exported Agent episodes for the selected run."""
from contextlib import contextmanager
import csv
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import coverage, db
class Coverage(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.db_path = self.root / "jobs.sqlite"
self.rows = [{"model_id": name, "task": "cls", "priority": priority, "status": "ready"}
for name, priority in [("a", "A"), ("b", "B"), ("c", "C")]]
self.addCleanup(patch.stopall)
patch.object(db, "connect", self.connect).start()
patch.object(coverage, "models", lambda: self.rows).start()
patch.object(coverage, "expand_matrix", lambda: [
SimpleNamespace(job_id="a", model="a", mode="e2e", k=4)]).start()
patch.object(coverage.paths, "DATASETS", self.root / "datasets").start()
patch.object(coverage, "live_line", lambda: []).start()
with self.connect() as con:
for row in self.rows:
con.execute("INSERT INTO jobs (run_id,job_id,model,task,priority,mode,state)"
" VALUES ('run',?,?,'cls',?,'oracle','PASS')",
(row["model_id"], row["model_id"], row["priority"]))
@contextmanager
def connect(self):
con = db._open(self.db_path)
con.executescript(db.SCHEMA)
try:
yield con
con.commit()
finally:
con.close()
def dataset(self, directory, filename, records):
path = self.root / "datasets" / directory / filename
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("".join(json.dumps(record) + "\n" for record in records))
return path
def test_counts_stable_and_legacy_agent_exports_only(self):
def record(kk):
return {"run_id": "run", "job_id": "b", "rollout": kk,
"source": "agent_events", "messages": [{"role": "assistant", "content": "done"}]}
self.dataset("run-training", "teacher.jsonl", [record(0)])
self.dataset("run-training", "student.jsonl", [record(1)])
legacy = record(0)
legacy.pop("source")
self.dataset("ds20260916-01", "teacher.jsonl", [legacy])
self.dataset("run-training", "pipeline.jsonl", [record(2)])
self.dataset("run-training", "exclusions.jsonl", [record(3)])
void = self.dataset("ds20260915-01", "teacher.jsonl", [record(4)])
(void.parent / "VOID").touch()
self.dataset("nested/deeper", "teacher.jsonl", [record(5)])
self.assertEqual(coverage.exported_keys(), {("run", "b", 0), ("run", "b", 1)})
def test_run_counts_database_plans_for_b_and_c_without_matrix_regeneration(self):
with self.connect() as con:
for name in ("b", "c"):
for kk in range(4):
db.rollout_upsert(con, "run", name, kk, "e2e", "teacher")
for kk in (4, 5):
db.rollout_upsert(con, "run", "b", kk, "repair", "teacher")
db.rollout_upsert(con, "other-run", "c", 7, "e2e", "teacher")
actual = coverage.collect("run")
self.assertEqual({name: item["planned_rollouts"] for name, item in actual.items()},
{"a": 0, "b": 6, "c": 4})
expected = coverage.collect(None)
self.assertEqual({name: item["planned_rollouts"] for name, item in expected.items()},
{"a": 4, "b": 0, "c": 0})
def test_unverified_is_collected_and_verifying_remains_active_in_display_and_csv(self):
with self.connect() as con:
for kk, state in enumerate(("PASS", "UNVERIFIED", "VERIFYING", "QUEUED")):
db.rollout_upsert(con, "run", "b", kk, "e2e", "teacher", state)
rendered = coverage.render("run")
self.assertIn("计划 4 条 · 已采 2", rendered)
self.assertIn("UNVERIFIED 1", rendered)
self.assertIn("在采 1(RUNNING 0 / VERIFYING 1)", rendered)
path = self.root / "coverage.csv"
coverage.to_csv("run", str(path))
with path.open() as stream:
row = next(row for row in csv.DictReader(stream) if row["model"] == "b")
self.assertEqual(row["rollout_planned"], "4")
self.assertEqual(row["rollout_done"], "2")
def test_all_model_master_preserves_legacy_oracle_evidence(self):
# Existing result/1 rows store gates/stages, without a completion object.
result = {"gates": {"G1": {"hbm_exists": True},
"G3": {"source": "board", "fp32": 80, "quant": 79, "verdict": "pass-screen"}},
"stages": {"05-compile": {"state": "OK"}}}
with self.connect() as con:
con.execute("UPDATE jobs SET result=? WHERE model='a'", (json.dumps(result),))
with patch.object(db, "readonly", self.connect), \
patch.object(coverage, "model_blockers", lambda *args: []), \
patch.object(coverage.paths, "CACHE", self.root / "cache"):
path = self.root / "master.csv"
coverage.write_master(str(path))
rows = {r["model"]: r for r in csv.DictReader(path.open())}
self.assertEqual(set(rows), {"a", "b", "c"})
for key in ("compile", "hbm_generated", "board", "accuracy", "accuracy_passed"):
self.assertEqual(rows["a"][key], "1")
self.assertEqual(rows["b"][key], "0")
self.assertEqual(rows["a"]["first_glm53_trace"], "")
self.assertEqual(rows["a"]["glm53_exported"], "0")
if __name__ == "__main__":
unittest.main()