rollout.dashboard serves catalog counts, attempts and complete trajectories, and the public prompt and template of each task. catalog.controllers()/complete() are split out so the dashboard can reuse receipts it has already read. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
91 lines
4.6 KiB
Python
91 lines
4.6 KiB
Python
import base64
|
|
import json
|
|
import shutil
|
|
import threading
|
|
import time
|
|
import types
|
|
import unittest
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
import test_core as core_fixture
|
|
from rollout import catalog, core, dashboard
|
|
|
|
IMAGE = "sha256:" + "a" * 64
|
|
|
|
|
|
class DashboardTests(unittest.TestCase):
|
|
def setUp(self):
|
|
helper = core_fixture.CoreTests("runTest")
|
|
helper.setUp()
|
|
self.addCleanup(helper.doCleanups)
|
|
repo = helper.root / "repo"
|
|
shutil.copytree(helper.task, repo / "tasks" / "parser")
|
|
task = core.load_task(repo / "tasks" / "parser")
|
|
calibration = helper.root / "calibrations" / "calibration-1"
|
|
calibration.mkdir(parents=True)
|
|
calibration.joinpath("receipt.json").write_text(json.dumps(dict(
|
|
task_digest=task["_digest"], image_id=IMAGE, status="PASS", cleanup="confirmed", trusted_local=False,
|
|
trials=[{}] * 20)))
|
|
runs = helper.root / "runs" / "batch"
|
|
done = runs / "pi-done"
|
|
done.mkdir(parents=True)
|
|
done.joinpath("receipt.json").write_text(json.dumps(dict(
|
|
task_id="parser", task_digest=task["_digest"], model_id=catalog.QWEN, status="PASS", cleanup="confirmed",
|
|
agent_exit=0, pi_evidence=dict(status="VERIFIED"))))
|
|
running = runs / "pi-running" / "run-x"
|
|
running.mkdir(parents=True)
|
|
running.joinpath("events.jsonl").write_text(json.dumps(dict(time=time.time(), event="created",
|
|
task_id="parser")) + "\n")
|
|
self.calibrations = calibration.parent
|
|
self.args = types.SimpleNamespace(repo=repo, calibrations=[], runs=[helper.root / "runs"], cold_runs=[],
|
|
image_id=IMAGE, refresh=60, host="127.0.0.1", port=0)
|
|
self.state = dashboard.State()
|
|
self.state.cold_calibrated = catalog.calibrations([self.calibrations], IMAGE) # Read as the cold thread would.
|
|
dashboard.refresh(self.args, self.state)
|
|
self.httpd = dashboard.server(self.args, self.state, "secret")
|
|
threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
|
|
self.addCleanup(self.httpd.server_close)
|
|
self.addCleanup(self.httpd.shutdown)
|
|
|
|
def get(self, path, password="secret"):
|
|
request = urllib.request.Request("http://127.0.0.1:%d%s" % (self.httpd.server_port, path))
|
|
if password:
|
|
request.add_header("Authorization", "Basic " + base64.b64encode(b"viewer:" + password.encode()).decode())
|
|
try:
|
|
with urllib.request.urlopen(request) as response:
|
|
return response.status, json.loads(response.read())
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, json.loads(error.read())
|
|
|
|
def test_counts_tasks_and_attempts_and_serves_only_public_material(self):
|
|
status, summary = self.get("/api/summary")
|
|
self.assertEqual(status, 200)
|
|
self.assertEqual((summary["tasks"]["total"], summary["tasks"]["status"], summary["tasks"]["ran"]),
|
|
(1, {"USABLE": 1}, 1))
|
|
self.assertEqual({key: summary["attempts"][key] for key in ("total", "complete", "passed", "running")},
|
|
dict(total=2, complete=1, passed=1, running=1))
|
|
self.assertEqual(summary["running"][0]["task_id"], "parser")
|
|
status, detail = self.get("/api/task/parser")
|
|
self.assertEqual((status, detail["prompt"], detail["files"]), (200, "Repair decoder.c",
|
|
[dict(path="decoder.c", size=5)]))
|
|
self.assertEqual(self.get("/api/file/parser/decoder.c")[1]["text"], "buggy")
|
|
for path in ("/api/file/parser/verify.py", "/api/file/parser/../gold/decoder.c", "/api/task/..%2Fparser",
|
|
"/api/file/other/decoder.c"):
|
|
self.assertEqual(self.get(path)[0], 404, path)
|
|
self.assertEqual(self.get("/api/summary", password=None)[0], 401)
|
|
self.assertEqual(self.get("/api/summary", password="wrong")[0], 401)
|
|
|
|
def test_cold_cache_round_trip_keeps_the_counted_facts(self):
|
|
cold = dashboard.scan(self.args.runs, time.time())
|
|
cache = self.args.repo / "cold.json"
|
|
dashboard.save_cold(cache, 1.0, cold, self.state.cold_calibrated)
|
|
at, (pairs, rows), calibrated = dashboard.load_cold(cache)
|
|
self.assertEqual((at, rows, calibrated), (1.0, cold[1], self.state.cold_calibrated))
|
|
self.assertEqual(catalog.outcomes(pairs), catalog.outcomes(cold[0]))
|
|
self.assertEqual(sorted(str(path) for path, _ in pairs), sorted(str(path) for path, _ in cold[0]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|