data-gen-ebd/tests/test_dashboard.py
amos 6166b7624f Add the read-only live dashboard for the task catalog and rollout progress
rollout.dashboard serves catalog counts, attempts and complete trajectories, and the public
prompt and template of each task. catalog.controllers()/complete() are split out so the
dashboard can reuse receipts it has already read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 17:45:28 +08:00

91 lines
4.6 KiB
Python

import base64
import json
import shutil
import threading
import time
import types
import unittest
import urllib.error
import urllib.request
import test_core as core_fixture
from rollout import catalog, core, dashboard
IMAGE = "sha256:" + "a" * 64
class DashboardTests(unittest.TestCase):
def setUp(self):
helper = core_fixture.CoreTests("runTest")
helper.setUp()
self.addCleanup(helper.doCleanups)
repo = helper.root / "repo"
shutil.copytree(helper.task, repo / "tasks" / "parser")
task = core.load_task(repo / "tasks" / "parser")
calibration = helper.root / "calibrations" / "calibration-1"
calibration.mkdir(parents=True)
calibration.joinpath("receipt.json").write_text(json.dumps(dict(
task_digest=task["_digest"], image_id=IMAGE, status="PASS", cleanup="confirmed", trusted_local=False,
trials=[{}] * 20)))
runs = helper.root / "runs" / "batch"
done = runs / "pi-done"
done.mkdir(parents=True)
done.joinpath("receipt.json").write_text(json.dumps(dict(
task_id="parser", task_digest=task["_digest"], model_id=catalog.QWEN, status="PASS", cleanup="confirmed",
agent_exit=0, pi_evidence=dict(status="VERIFIED"))))
running = runs / "pi-running" / "run-x"
running.mkdir(parents=True)
running.joinpath("events.jsonl").write_text(json.dumps(dict(time=time.time(), event="created",
task_id="parser")) + "\n")
self.calibrations = calibration.parent
self.args = types.SimpleNamespace(repo=repo, calibrations=[], runs=[helper.root / "runs"], cold_runs=[],
image_id=IMAGE, refresh=60, host="127.0.0.1", port=0)
self.state = dashboard.State()
self.state.cold_calibrated = catalog.calibrations([self.calibrations], IMAGE) # Read as the cold thread would.
dashboard.refresh(self.args, self.state)
self.httpd = dashboard.server(self.args, self.state, "secret")
threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
self.addCleanup(self.httpd.server_close)
self.addCleanup(self.httpd.shutdown)
def get(self, path, password="secret"):
request = urllib.request.Request("http://127.0.0.1:%d%s" % (self.httpd.server_port, path))
if password:
request.add_header("Authorization", "Basic " + base64.b64encode(b"viewer:" + password.encode()).decode())
try:
with urllib.request.urlopen(request) as response:
return response.status, json.loads(response.read())
except urllib.error.HTTPError as error:
return error.code, json.loads(error.read())
def test_counts_tasks_and_attempts_and_serves_only_public_material(self):
status, summary = self.get("/api/summary")
self.assertEqual(status, 200)
self.assertEqual((summary["tasks"]["total"], summary["tasks"]["status"], summary["tasks"]["ran"]),
(1, {"USABLE": 1}, 1))
self.assertEqual({key: summary["attempts"][key] for key in ("total", "complete", "passed", "running")},
dict(total=2, complete=1, passed=1, running=1))
self.assertEqual(summary["running"][0]["task_id"], "parser")
status, detail = self.get("/api/task/parser")
self.assertEqual((status, detail["prompt"], detail["files"]), (200, "Repair decoder.c",
[dict(path="decoder.c", size=5)]))
self.assertEqual(self.get("/api/file/parser/decoder.c")[1]["text"], "buggy")
for path in ("/api/file/parser/verify.py", "/api/file/parser/../gold/decoder.c", "/api/task/..%2Fparser",
"/api/file/other/decoder.c"):
self.assertEqual(self.get(path)[0], 404, path)
self.assertEqual(self.get("/api/summary", password=None)[0], 401)
self.assertEqual(self.get("/api/summary", password="wrong")[0], 401)
def test_cold_cache_round_trip_keeps_the_counted_facts(self):
cold = dashboard.scan(self.args.runs, time.time())
cache = self.args.repo / "cold.json"
dashboard.save_cold(cache, 1.0, cold, self.state.cold_calibrated)
at, (pairs, rows), calibrated = dashboard.load_cold(cache)
self.assertEqual((at, rows, calibrated), (1.0, cold[1], self.state.cold_calibrated))
self.assertEqual(catalog.outcomes(pairs), catalog.outcomes(cold[0]))
self.assertEqual(sorted(str(path) for path, _ in pairs), sorted(str(path) for path, _ in cold[0]))
if __name__ == "__main__":
unittest.main()