"""Read-only monitor: truthful process identities and no request-side sampling.""" import json import sqlite3 import tempfile import threading import unittest import urllib.request from pathlib import Path from harness import db, monitor class MonitorTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.root = Path(self.tmp.name) (self.root / 'state').mkdir() with sqlite3.connect(self.root / 'state/jobs.sqlite') as con: con.executescript(db.SCHEMA) con.execute("INSERT INTO runs VALUES ('r', 1, '{}', '')") for k in range(2): con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES ('r',?,?,?)", (f'j{k}',0,'RUNNING')) self.reads = [] def probe(identity): self.reads.append(identity['pid']) return {'status': 'ALIVE', 'cpu_ticks': 1, 'io_bytes': 2} self.observer = monitor.Observer(self.root, process_probe=probe, clock=lambda: 100.0) for k in range(2): rd = self.root / f'runs/r/j{k}/rollouts/00' rd.mkdir(parents=True) (rd/'rollout.json').write_text(json.dumps({'state':'RUNNING'})) records = [{'pid': 11+k, 'role':'agent','host':'host','boot_id':'boot','proc_start_ticks':1}] if k == 0: records.append({'pid':21,'role':'compiler','host':'host','boot_id':'boot','proc_start_ticks':2}) (rd/'execution.json').write_text(json.dumps({'execution_id':f'e{k}','execution_state':'ACTIVE','activity':'WAITING_LLM','epoch':1, 'owner':{'pid':31,'host':'host','boot_id':'boot','proc_start_ticks':3},'processes':records})) def test_identity_counts_separate_and_database_is_readonly(self): before = (self.root/'state/jobs.sqlite').read_bytes() self.observer.sample() snapshot = self.observer.snapshot('r') self.assertEqual(snapshot['counts']['agent_processes'],2) self.assertEqual(snapshot['counts']['stage_processes'],1) self.assertEqual(snapshot['counts']['collector_processes'],1) self.assertEqual(snapshot['counts']['db_active'],2) self.assertEqual((self.root/'state/jobs.sqlite').read_bytes(),before) self.assertEqual(snapshot['source_status'],'FRESH') def test_missing_identity_unknown_not_zero(self): (self.root/'runs/r/j1/rollouts/00/execution.json').unlink() self.observer.sample() self.assertIsNone(self.observer.snapshot('r')['counts']['agent_processes']) self.assertEqual(self.observer.detail('r','j1',0)['health'],'UNKNOWN') def test_supervisor_is_not_an_agent_or_compiler(self): path = self.root / 'runs/r/j0/rollouts/00/execution.json' record = json.loads(path.read_text()) record['processes'].append({'pid': 41, 'role': 'supervisor', 'host': 'host', 'boot_id': 'boot', 'proc_start_ticks': 4}) path.write_text(json.dumps(record)) self.observer.sample() counts = self.observer.snapshot('r')['counts'] self.assertEqual((counts['agent_processes'], counts['stage_processes'], counts['supervisor_processes']), (2, 1, 1)) def test_descendants_and_unknown_roles_never_become_stages(self): path = self.root / 'runs/r/j0/rollouts/00/execution.json' record = json.loads(path.read_text()) added = [{'pid': 40 + i, 'role': role, 'host': 'host', 'boot_id': 'boot', 'proc_start_ticks': 10 + i} for i, role in enumerate(('stage', 'compiler', 'board', 'descendant', 'unknown', 'tool'))] # Duplicate observations of the same birth identity count only once. record['processes'].extend(added + [dict(added[3])]) path.write_text(json.dumps(record)) self.observer.sample() counts = self.observer.snapshot('r')['counts'] self.assertEqual(counts['verified_processes'], {'agent': 2, 'stage': 4, 'collector': 1, 'supervisor': 0, 'descendant': 1, 'other': 2}) self.assertEqual((counts['stage_processes'], counts['descendant_processes'], counts['other_processes']), (4, 1, 2)) details = self.observer.detail('r', 'j0', 0)['processes'] for process in added: self.assertTrue(any(monitor.identity_key(observed) == monitor.identity_key(process) and observed['role'] == process['role'] for observed in details)) def test_unknown_progress_remains_unknown_even_when_process_alive(self): path = self.root / 'runs/r/j1/rollouts/00/execution.json' record = json.loads(path.read_text()) record.update(health='UNKNOWN', progress={'reason': 'IO_SOURCE_UNAVAILABLE'}) path.write_text(json.dumps(record)) self.observer.sample() observed = self.observer.detail('r', 'j1', 0) self.assertEqual(observed['health'], 'UNKNOWN') self.assertEqual(observed['reason_code'], 'IO_SOURCE_UNAVAILABLE') def test_quarantine_preserves_specific_stop_and_cleanup_reason(self): path = self.root / 'runs/r/j0/rollouts/00/execution.json' record = json.loads(path.read_text()) record.update(cleanup_state='QUARANTINED', stop_reason='INFRA_STAGE_REGISTRATION_PENDING', cleanup_errors=['board receipt remains unknown']) path.write_text(json.dumps(record)) self.observer.sample() row = self.observer.detail('r', 'j0', 0) self.assertEqual(row['reason_code'], 'INFRA_STAGE_REGISTRATION_PENDING') self.assertEqual(row['cleanup_errors'], ['board receipt remains unknown']) def test_registration_pending_is_visible_without_forging_done(self): path = self.root / 'runs/r/j0/rollouts/00/execution.json' record = json.loads(path.read_text()) record.update(activity='VERIFYING', health='OK', cleanup_state='NOT_NEEDED', progress={'sampled_at': 100}) path.write_text(json.dumps(record)) handle = path.parent / 'handles/h1/status.json' handle.parent.mkdir(parents=True) handle.write_text('{"state":"RUNNING","registration":"PENDING"}') self.observer.sample() row = self.observer.detail('r', 'j0', 0) self.assertEqual(row['reason_code'], 'STAGE_REGISTRATION_PENDING') self.assertEqual(row['pending_registration'], ['h1']) self.assertEqual(row['state'], 'RUNNING') def test_legal_phase_without_agent_does_not_signal_mismatch(self): path=self.root/'runs/r/j1/rollouts/00/execution.json' record=json.loads(path.read_text());record.update(activity='VERIFYING',processes=[]) path.write_text(json.dumps(record)) self.observer.sample() self.assertNotEqual(self.observer.detail('r','j1',0)['health'],'STATE_MISMATCH') def test_refresh_only_reads_cached_snapshot_and_rejects_paths(self): self.observer.sample() seen=len(self.reads) server=monitor.make_server(self.observer,('127.0.0.1',0)) thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start() self.addCleanup(server.server_close);self.addCleanup(server.shutdown) url=f'http://127.0.0.1:{server.server_port}' for _ in range(2): with urllib.request.urlopen(url+'/api/runs/r/snapshot') as r:self.assertEqual(r.status,200) self.assertEqual(len(self.reads),seen) with self.assertRaises(urllib.error.HTTPError):urllib.request.urlopen(url+'/api/runs/../snapshot') with urllib.request.urlopen(url+'/') as r:self.assertIn(b'Agent',r.read()) def test_stale_and_database_failure_never_fake_zero(self): self.observer.sample() self.observer.clock=lambda:116.0 self.assertEqual(self.observer.snapshot('r')['source_status'],'STALE') (self.root/'state/jobs.sqlite').unlink() self.observer.sample() snapshot=self.observer.snapshot('r') self.assertEqual(snapshot['source_status'],'UNAVAILABLE') self.assertEqual(snapshot['counts']['agent_processes'],2) self.assertEqual(snapshot['sampled_at'],100.0) if __name__ == '__main__': unittest.main()