data-gen-bpu/scripts/test_monitor.py
ming01.wang 498187569c fix: close rollout finalization and review publication gaps
Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
2026-09-29 16:09:01 +08:00

151 lines
8.0 KiB
Python

"""Read-only monitor: truthful process identities and no request-side sampling."""
import json
import sqlite3
import tempfile
import threading
import unittest
import urllib.request
from pathlib import Path
from harness import db, monitor
class MonitorTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
(self.root / 'state').mkdir()
with sqlite3.connect(self.root / 'state/jobs.sqlite') as con:
con.executescript(db.SCHEMA)
con.execute("INSERT INTO runs VALUES ('r', 1, '{}', '')")
for k in range(2):
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES ('r',?,?,?)", (f'j{k}',0,'RUNNING'))
self.reads = []
def probe(identity):
self.reads.append(identity['pid'])
return {'status': 'ALIVE', 'cpu_ticks': 1, 'io_bytes': 2}
self.observer = monitor.Observer(self.root, process_probe=probe, clock=lambda: 100.0)
for k in range(2):
rd = self.root / f'runs/r/j{k}/rollouts/00'
rd.mkdir(parents=True)
(rd/'rollout.json').write_text(json.dumps({'state':'RUNNING'}))
records = [{'pid': 11+k, 'role':'agent','host':'host','boot_id':'boot','proc_start_ticks':1}]
if k == 0: records.append({'pid':21,'role':'compiler','host':'host','boot_id':'boot','proc_start_ticks':2})
(rd/'execution.json').write_text(json.dumps({'execution_id':f'e{k}','execution_state':'ACTIVE','activity':'WAITING_LLM','epoch':1,
'owner':{'pid':31,'host':'host','boot_id':'boot','proc_start_ticks':3},'processes':records}))
def test_identity_counts_separate_and_database_is_readonly(self):
before = (self.root/'state/jobs.sqlite').read_bytes()
self.observer.sample()
snapshot = self.observer.snapshot('r')
self.assertEqual(snapshot['counts']['agent_processes'],2)
self.assertEqual(snapshot['counts']['stage_processes'],1)
self.assertEqual(snapshot['counts']['collector_processes'],1)
self.assertEqual(snapshot['counts']['db_active'],2)
self.assertEqual((self.root/'state/jobs.sqlite').read_bytes(),before)
self.assertEqual(snapshot['source_status'],'FRESH')
def test_missing_identity_unknown_not_zero(self):
(self.root/'runs/r/j1/rollouts/00/execution.json').unlink()
self.observer.sample()
self.assertIsNone(self.observer.snapshot('r')['counts']['agent_processes'])
self.assertEqual(self.observer.detail('r','j1',0)['health'],'UNKNOWN')
def test_supervisor_is_not_an_agent_or_compiler(self):
path = self.root / 'runs/r/j0/rollouts/00/execution.json'
record = json.loads(path.read_text())
record['processes'].append({'pid': 41, 'role': 'supervisor', 'host': 'host',
'boot_id': 'boot', 'proc_start_ticks': 4})
path.write_text(json.dumps(record))
self.observer.sample()
counts = self.observer.snapshot('r')['counts']
self.assertEqual((counts['agent_processes'], counts['stage_processes'], counts['supervisor_processes']), (2, 1, 1))
def test_descendants_and_unknown_roles_never_become_stages(self):
path = self.root / 'runs/r/j0/rollouts/00/execution.json'
record = json.loads(path.read_text())
added = [{'pid': 40 + i, 'role': role, 'host': 'host', 'boot_id': 'boot',
'proc_start_ticks': 10 + i}
for i, role in enumerate(('stage', 'compiler', 'board', 'descendant', 'unknown', 'tool'))]
# Duplicate observations of the same birth identity count only once.
record['processes'].extend(added + [dict(added[3])])
path.write_text(json.dumps(record))
self.observer.sample()
counts = self.observer.snapshot('r')['counts']
self.assertEqual(counts['verified_processes'],
{'agent': 2, 'stage': 4, 'collector': 1, 'supervisor': 0, 'descendant': 1, 'other': 2})
self.assertEqual((counts['stage_processes'], counts['descendant_processes'], counts['other_processes']), (4, 1, 2))
details = self.observer.detail('r', 'j0', 0)['processes']
for process in added:
self.assertTrue(any(monitor.identity_key(observed) == monitor.identity_key(process)
and observed['role'] == process['role'] for observed in details))
def test_unknown_progress_remains_unknown_even_when_process_alive(self):
path = self.root / 'runs/r/j1/rollouts/00/execution.json'
record = json.loads(path.read_text())
record.update(health='UNKNOWN', progress={'reason': 'IO_SOURCE_UNAVAILABLE'})
path.write_text(json.dumps(record))
self.observer.sample()
observed = self.observer.detail('r', 'j1', 0)
self.assertEqual(observed['health'], 'UNKNOWN')
self.assertEqual(observed['reason_code'], 'IO_SOURCE_UNAVAILABLE')
def test_quarantine_preserves_specific_stop_and_cleanup_reason(self):
path = self.root / 'runs/r/j0/rollouts/00/execution.json'
record = json.loads(path.read_text())
record.update(cleanup_state='QUARANTINED', stop_reason='INFRA_STAGE_REGISTRATION_PENDING',
cleanup_errors=['board receipt remains unknown'])
path.write_text(json.dumps(record))
self.observer.sample()
row = self.observer.detail('r', 'j0', 0)
self.assertEqual(row['reason_code'], 'INFRA_STAGE_REGISTRATION_PENDING')
self.assertEqual(row['cleanup_errors'], ['board receipt remains unknown'])
def test_registration_pending_is_visible_without_forging_done(self):
path = self.root / 'runs/r/j0/rollouts/00/execution.json'
record = json.loads(path.read_text())
record.update(activity='VERIFYING', health='OK', cleanup_state='NOT_NEEDED', progress={'sampled_at': 100})
path.write_text(json.dumps(record))
handle = path.parent / 'handles/h1/status.json'
handle.parent.mkdir(parents=True)
handle.write_text('{"state":"RUNNING","registration":"PENDING"}')
self.observer.sample()
row = self.observer.detail('r', 'j0', 0)
self.assertEqual(row['reason_code'], 'STAGE_REGISTRATION_PENDING')
self.assertEqual(row['pending_registration'], ['h1'])
self.assertEqual(row['state'], 'RUNNING')
def test_legal_phase_without_agent_does_not_signal_mismatch(self):
path=self.root/'runs/r/j1/rollouts/00/execution.json'
record=json.loads(path.read_text());record.update(activity='VERIFYING',processes=[])
path.write_text(json.dumps(record))
self.observer.sample()
self.assertNotEqual(self.observer.detail('r','j1',0)['health'],'STATE_MISMATCH')
def test_refresh_only_reads_cached_snapshot_and_rejects_paths(self):
self.observer.sample()
seen=len(self.reads)
server=monitor.make_server(self.observer,('127.0.0.1',0))
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
self.addCleanup(server.server_close);self.addCleanup(server.shutdown)
url=f'http://127.0.0.1:{server.server_port}'
for _ in range(2):
with urllib.request.urlopen(url+'/api/runs/r/snapshot') as r:self.assertEqual(r.status,200)
self.assertEqual(len(self.reads),seen)
with self.assertRaises(urllib.error.HTTPError):urllib.request.urlopen(url+'/api/runs/../snapshot')
with urllib.request.urlopen(url+'/') as r:self.assertIn(b'Agent',r.read())
def test_stale_and_database_failure_never_fake_zero(self):
self.observer.sample()
self.observer.clock=lambda:116.0
self.assertEqual(self.observer.snapshot('r')['source_status'],'STALE')
(self.root/'state/jobs.sqlite').unlink()
self.observer.sample()
snapshot=self.observer.snapshot('r')
self.assertEqual(snapshot['source_status'],'UNAVAILABLE')
self.assertEqual(snapshot['counts']['agent_processes'],2)
self.assertEqual(snapshot['sampled_at'],100.0)
if __name__ == '__main__': unittest.main()