data-gen-bpu/scripts/test_monitor.py

126 lines
6.6 KiB
Python

"""Read-only monitor: truthful process identities and no request-side sampling."""
import json
import sqlite3
import tempfile
import threading
import unittest
import urllib.request
from pathlib import Path
from harness import db, monitor
class MonitorTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
(self.root / 'state').mkdir()
with sqlite3.connect(self.root / 'state/jobs.sqlite') as con:
con.executescript(db.SCHEMA)
con.execute("INSERT INTO runs VALUES ('r', 1, '{}', '')")
for k in range(2):
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES ('r',?,?,?)", (f'j{k}',0,'RUNNING'))
self.reads = []
def probe(identity):
self.reads.append(identity['pid'])
return {'status': 'ALIVE', 'cpu_ticks': 1, 'io_bytes': 2}
self.observer = monitor.Observer(self.root, process_probe=probe, clock=lambda: 100.0)
for k in range(2):
rd = self.root / f'runs/r/j{k}/rollouts/00'
rd.mkdir(parents=True)
(rd/'rollout.json').write_text(json.dumps({'state':'RUNNING'}))
records = [{'pid': 11+k, 'role':'agent','host':'host','boot_id':'boot','proc_start_ticks':1}]
if k == 0: records.append({'pid':21,'role':'compiler','host':'host','boot_id':'boot','proc_start_ticks':2})
(rd/'execution.json').write_text(json.dumps({'execution_id':f'e{k}','execution_state':'ACTIVE','activity':'WAITING_LLM','epoch':1,
'owner':{'pid':31,'host':'host','boot_id':'boot','proc_start_ticks':3},'processes':records}))
def test_identity_counts_separate_and_database_is_readonly(self):
before = (self.root/'state/jobs.sqlite').read_bytes()
self.observer.sample()
snapshot = self.observer.snapshot('r')
self.assertEqual(snapshot['counts']['agent_processes'],2)
self.assertEqual(snapshot['counts']['stage_processes'],1)
self.assertEqual(snapshot['counts']['collector_processes'],1)
self.assertEqual(snapshot['counts']['db_active'],2)
self.assertEqual((self.root/'state/jobs.sqlite').read_bytes(),before)
self.assertEqual(snapshot['source_status'],'FRESH')
def test_missing_identity_unknown_not_zero(self):
(self.root/'runs/r/j1/rollouts/00/execution.json').unlink()
self.observer.sample()
self.assertIsNone(self.observer.snapshot('r')['counts']['agent_processes'])
self.assertEqual(self.observer.detail('r','j1',0)['health'],'UNKNOWN')
def test_supervisor_is_not_an_agent_or_compiler(self):
path = self.root / 'runs/r/j0/rollouts/00/execution.json'
record = json.loads(path.read_text())
record['processes'].append({'pid': 41, 'role': 'supervisor', 'host': 'host',
'boot_id': 'boot', 'proc_start_ticks': 4})
path.write_text(json.dumps(record))
self.observer.sample()
counts = self.observer.snapshot('r')['counts']
self.assertEqual((counts['agent_processes'], counts['stage_processes'], counts['supervisor_processes']), (2, 1, 1))
def test_descendants_and_unknown_roles_never_become_stages(self):
path = self.root / 'runs/r/j0/rollouts/00/execution.json'
record = json.loads(path.read_text())
added = [{'pid': 40 + i, 'role': role, 'host': 'host', 'boot_id': 'boot',
'proc_start_ticks': 10 + i}
for i, role in enumerate(('stage', 'compiler', 'board', 'descendant', 'unknown', 'tool'))]
# Duplicate observations of the same birth identity count only once.
record['processes'].extend(added + [dict(added[3])])
path.write_text(json.dumps(record))
self.observer.sample()
counts = self.observer.snapshot('r')['counts']
self.assertEqual(counts['verified_processes'],
{'agent': 2, 'stage': 4, 'collector': 1, 'supervisor': 0, 'descendant': 1, 'other': 2})
self.assertEqual((counts['stage_processes'], counts['descendant_processes'], counts['other_processes']), (4, 1, 2))
details = self.observer.detail('r', 'j0', 0)['processes']
for process in added:
self.assertTrue(any(monitor.identity_key(observed) == monitor.identity_key(process)
and observed['role'] == process['role'] for observed in details))
def test_unknown_progress_remains_unknown_even_when_process_alive(self):
path = self.root / 'runs/r/j1/rollouts/00/execution.json'
record = json.loads(path.read_text())
record.update(health='UNKNOWN', progress={'reason': 'IO_SOURCE_UNAVAILABLE'})
path.write_text(json.dumps(record))
self.observer.sample()
observed = self.observer.detail('r', 'j1', 0)
self.assertEqual(observed['health'], 'UNKNOWN')
self.assertEqual(observed['reason_code'], 'IO_SOURCE_UNAVAILABLE')
def test_legal_phase_without_agent_does_not_signal_mismatch(self):
path=self.root/'runs/r/j1/rollouts/00/execution.json'
record=json.loads(path.read_text());record.update(activity='VERIFYING',processes=[])
path.write_text(json.dumps(record))
self.observer.sample()
self.assertNotEqual(self.observer.detail('r','j1',0)['health'],'STATE_MISMATCH')
def test_refresh_only_reads_cached_snapshot_and_rejects_paths(self):
self.observer.sample()
seen=len(self.reads)
server=monitor.make_server(self.observer,('127.0.0.1',0))
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
self.addCleanup(server.server_close);self.addCleanup(server.shutdown)
url=f'http://127.0.0.1:{server.server_port}'
for _ in range(2):
with urllib.request.urlopen(url+'/api/runs/r/snapshot') as r:self.assertEqual(r.status,200)
self.assertEqual(len(self.reads),seen)
with self.assertRaises(urllib.error.HTTPError):urllib.request.urlopen(url+'/api/runs/../snapshot')
with urllib.request.urlopen(url+'/') as r:self.assertIn(b'Agent',r.read())
def test_stale_and_database_failure_never_fake_zero(self):
self.observer.sample()
self.observer.clock=lambda:116.0
self.assertEqual(self.observer.snapshot('r')['source_status'],'STALE')
(self.root/'state/jobs.sqlite').unlink()
self.observer.sample()
snapshot=self.observer.snapshot('r')
self.assertEqual(snapshot['source_status'],'UNAVAILABLE')
self.assertEqual(snapshot['counts']['agent_processes'],2)
self.assertEqual(snapshot['sampled_at'],100.0)
if __name__ == '__main__': unittest.main()