Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
349 lines
20 KiB
Python
349 lines
20 KiB
Python
"""Cached, read-only collection monitor. HTTP handlers never inspect or run jobs."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import collections
|
|
from contextlib import closing
|
|
import copy
|
|
import json
|
|
import hashlib
|
|
import re
|
|
import socket
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qs, unquote, urlsplit
|
|
|
|
ACTIVE = {'RUNNING', 'VERIFYING', 'PAUSED_INFRA', 'CLEANING'}
|
|
AGENT_PHASES = {'WAITING_LLM', 'RUNNING_TOOL', 'COMPILING', 'RUNNING_BOARD'}
|
|
IDENTIFIER = re.compile(r'^[A-Za-z0-9_.-]+$')
|
|
|
|
|
|
def read_json(path: Path, default=None):
|
|
try:
|
|
return json.loads(path.read_text())
|
|
except (OSError, ValueError):
|
|
return default
|
|
|
|
|
|
def process_probe(identity: dict) -> dict:
|
|
"""PID reuse, another host and unreadable procfs are distinct from dead."""
|
|
try:
|
|
if identity.get('host') != socket.gethostname():
|
|
return {'status': 'UNKNOWN'}
|
|
if not identity.get('boot_id') or not isinstance(identity.get('proc_start_ticks'), int):
|
|
return {'status': 'UNKNOWN'}
|
|
boot = Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
|
if identity.get('boot_id') != boot:
|
|
return {'status': 'DEAD'}
|
|
proc = Path('/proc') / str(int(identity['pid']))
|
|
try:
|
|
stat = (proc / 'stat').read_text().rsplit(')', 1)[1].split()
|
|
except FileNotFoundError:
|
|
return {'status': 'DEAD'}
|
|
if int(stat[19]) != int(identity['proc_start_ticks']):
|
|
return {'status': 'REUSED'}
|
|
io = {}
|
|
try:
|
|
io = dict(line.split(': ', 1) for line in (proc / 'io').read_text().splitlines())
|
|
except OSError:
|
|
pass
|
|
return {'status': 'DEAD' if stat[0] == 'Z' else 'ALIVE', 'state': stat[0],
|
|
'cpu_ticks': int(stat[11]) + int(stat[12]),
|
|
'io_bytes': sum(int(io.get(k, 0)) for k in ('read_bytes', 'write_bytes')) if io else None}
|
|
except (OSError, ValueError, KeyError, IndexError, TypeError):
|
|
return {'status': 'UNKNOWN'}
|
|
|
|
|
|
def identity_key(record):
|
|
return tuple(record.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks'))
|
|
|
|
|
|
class Observer:
|
|
def __init__(self, root: Path, *, state_dir=None, process_probe=process_probe,
|
|
clock=time.time, stale_after=15, quota_source=None):
|
|
self.root = Path(root).resolve()
|
|
self.state = Path(state_dir) if state_dir else self.root / 'state'
|
|
self.probe, self.clock, self.stale_after = process_probe, clock, stale_after
|
|
self.quota_source = quota_source
|
|
self._lock = threading.Lock()
|
|
self._snapshots, self._rows, self._previous = {}, {}, {}
|
|
self._error = None
|
|
try:
|
|
self.local_boot = Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
|
except OSError:
|
|
self.local_boot = None
|
|
|
|
def _execution(self, row, probes, now):
|
|
rd = self.root / 'runs' / row['run_id'] / row['job_id'] / 'rollouts' / f"{row['kk']:02d}"
|
|
record = read_json(rd / 'execution.json')
|
|
rj = read_json(rd / 'rollout.json', {})
|
|
result = {k: row.get(k) for k in ('run_id', 'job_id', 'kk', 'state', 'started_at', 'finished_at', 'verdict_terminal')}
|
|
result.update(execution_state=rj.get('state'), execution_outcome=None, health='UNKNOWN',
|
|
processes=[], activity='UNKNOWN', cleanup_state='UNKNOWN')
|
|
if not isinstance(record, dict):
|
|
result['reason_code'] = 'EXECUTION_IDENTITY_NOT_RECORDED'
|
|
return result
|
|
for key in ('phases', 'execution_id', 'epoch', 'activity', 'execution_state', 'execution_outcome',
|
|
'cleanup_state', 'stop_reason', 'cancel_requested', 'config_version', 'updated_at',
|
|
'deadline_monotonic', 'activity_details', 'progress', 'health', 'cleanup_errors'):
|
|
result[key] = record.get(key)
|
|
identities = list(record.get('processes', []))
|
|
if record.get('owner'):
|
|
identities.append({**record['owner'], 'role': 'collector'})
|
|
for item in identities:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
identity = item.get('identity', item)
|
|
key = identity_key(identity)
|
|
if key not in probes:
|
|
probes[key] = self.probe(identity)
|
|
observation = probes[key]
|
|
safe = {k: identity.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks')}
|
|
safe.update(role=item.get('role', 'unknown'), **observation)
|
|
previous = self._previous.get(key)
|
|
if previous and observation.get('status') == previous.get('status') == 'ALIVE':
|
|
for metric in ('cpu_ticks', 'io_bytes'):
|
|
if isinstance(observation.get(metric), (int, float)) and isinstance(previous.get(metric), (int, float)):
|
|
safe[metric + '_delta'] = max(0, observation[metric] - previous[metric])
|
|
result['processes'].append(safe)
|
|
unknown = not record.get('owner') or any(p['status'] == 'UNKNOWN' for p in result['processes'])
|
|
result['identity_status'] = 'UNKNOWN' if unknown else 'CONFIRMED'
|
|
result['sources'] = {name: self.freshness(at, now) for name, at in
|
|
[('execution', record.get('updated_at')), ('progress', (record.get('progress') or {}).get('sampled_at'))]}
|
|
active = record.get('execution_state') == 'ACTIVE'
|
|
expected_agent = active and record.get('activity') in AGENT_PHASES
|
|
agents = [p for p in result['processes'] if p['role'] == 'agent']
|
|
started = row.get('started_at') or now
|
|
if unknown:
|
|
result.update(health='UNKNOWN', reason_code='PROCESS_SOURCE_UNAVAILABLE')
|
|
elif expected_agent and now - started > 30 and not any(p['status'] == 'ALIVE' for p in agents):
|
|
result.update(health='STATE_MISMATCH', reason_code='EXPECTED_AGENT_MISSING')
|
|
elif record.get('cleanup_state') in ('UNKNOWN', 'QUARANTINED'):
|
|
result.update(health='QUARANTINED', reason_code=record.get('stop_reason') or 'CLEANUP_UNCONFIRMED')
|
|
elif active and result['sources']['progress']['status'] == 'STALE':
|
|
result.update(health='STALE', reason_code='PROGRESS_SNAPSHOT_STALE')
|
|
elif record.get('health') in ('SUSPECT', 'UNKNOWN'):
|
|
result.update(health=record['health'], reason_code=(record.get('progress') or {}).get('reason', 'PROGRESS_NOT_CONFIRMED'))
|
|
else:
|
|
result.update(health='WAITING' if str(record.get('activity', '')).startswith('WAITING') else 'OBSERVED',
|
|
reason_code='IDENTITY_CHECKED')
|
|
result['pending_registration'] = [p.parent.name for p in sorted((rd / 'handles').glob('*/status.json'))
|
|
if isinstance(status := read_json(p), dict) and status.get('registration') == 'PENDING']
|
|
if result['pending_registration'] and result['health'] not in ('QUARANTINED', 'STATE_MISMATCH', 'UNKNOWN'):
|
|
result.update(health='WAITING', reason_code='STAGE_REGISTRATION_PENDING')
|
|
# Exported and actual execution facts stay separate. No inference from exit 0.
|
|
verdict = read_json(rd / 'verdict.json', {})
|
|
result['hard_gates'] = {k: {'verdict': v.get('verdict'), 'reason': v.get('reason')}
|
|
for k, v in verdict.get('gates', {}).items() if isinstance(v, dict)}
|
|
result['claim_consistent'] = verdict.get('claim_consistent')
|
|
result['evidence'] = [name for name in ('rollout.json', 'execution.json', 'verdict.json') if (rd / name).is_file()]
|
|
return result
|
|
|
|
def freshness(self, at, now):
|
|
if not isinstance(at, (float, int)):
|
|
return {'status': 'UNKNOWN', 'sampled_at': None}
|
|
age = max(0, now - at)
|
|
return {'status': 'STALE' if age > self.stale_after else 'FRESH', 'sampled_at': at, 'age_s': age}
|
|
|
|
def resources(self, observed, now):
|
|
locks = self.state / 'locks'
|
|
leases = []
|
|
for suffix, state in (('lease', 'HELD'), ('quarantine', 'QUARANTINED')):
|
|
for path in sorted(locks.glob('*.' + suffix + '.json'))[:1000]:
|
|
row = read_json(path)
|
|
if not isinstance(row, dict):
|
|
leases.append({'resource': path.name, 'state': 'UNKNOWN'})
|
|
continue
|
|
leases.append({'resource': path.name.split('.' + suffix)[0], 'state': state,
|
|
**{k: row.get(k) for k in ('execution_id', 'epoch', 'pool', 'slot', 'phase', 'operation_id', 'acquired_at', 'reason')},
|
|
'holder': {k: row.get('holder', {}).get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks')}})
|
|
waits = []
|
|
for row in observed:
|
|
for operation, phase in (row.get('phases') or {}).items():
|
|
if phase.get('state') == 'QUEUED':
|
|
waits.append({'job_id': row['job_id'], 'kk': row['kk'], 'operation_id': operation,
|
|
**phase, 'wait_s': max(0, time.monotonic() - phase['queued_monotonic'])
|
|
if phase.get('boot_id') == self.local_boot and isinstance(phase.get('queued_monotonic'), (int, float)) else None})
|
|
return {'status': 'OBSERVED' if locks.is_dir() else 'UNKNOWN', 'sampled_at': now,
|
|
'leases': leases, 'waiting': waits, 'waiting_count': len(waits)}
|
|
|
|
def published(self, now):
|
|
pointer = read_json(self.root / 'datasets/CURRENT.json')
|
|
if not isinstance(pointer, dict):
|
|
return {'status': 'UNKNOWN', 'reason': 'NO_PUBLISHED_GENERATION'}
|
|
version = pointer.get('dataset_version', '')
|
|
if not re.fullmatch(r'sft-[0-9a-f]{24}', version):
|
|
return {'status': 'UNKNOWN', 'reason': 'INVALID_POINTER'}
|
|
folder = self.root / 'datasets' / version
|
|
try:
|
|
release_bytes = (folder / 'release.json').read_bytes()
|
|
release = json.loads(release_bytes)
|
|
data = (folder / 'acceptance.json').read_bytes()
|
|
manifest_bytes = (folder / 'dataset-manifest.json').read_bytes()
|
|
if (hashlib.sha256(release_bytes).hexdigest() != pointer.get('release_sha256')
|
|
or hashlib.sha256(data).hexdigest() != release.get('acceptance_sha256')
|
|
or hashlib.sha256(manifest_bytes).hexdigest() != release.get('manifest_sha256')):
|
|
raise ValueError('hash mismatch')
|
|
accepted, manifest = json.loads(data), json.loads(manifest_bytes)
|
|
revocation_path = self.root / 'datasets/revocations.json'
|
|
revoked = read_json(revocation_path) if revocation_path.exists() else {'versions': {}}
|
|
if not isinstance(revoked, dict) or not isinstance(revoked.get('versions'), dict):
|
|
raise ValueError('revocations unavailable')
|
|
return {'status': 'REVOKED' if version in revoked.get('versions', {}) else 'VERIFIED_MANIFEST',
|
|
'scope': 'published_generation', 'dataset_version': version, 'rules_version': manifest['rules_version'],
|
|
'accepted': accepted['accepted'], 'retained': accepted.get('retained'),
|
|
'quality': accepted.get('data_quality', {'status': 'UNKNOWN'}),
|
|
'source': 'immutable acceptance.json', 'sampled_at': now}
|
|
except (OSError, ValueError, KeyError, TypeError):
|
|
return {'status': 'UNKNOWN', 'reason': 'PUBLISHED_METADATA_UNVERIFIED', 'sampled_at': now}
|
|
|
|
def sample(self):
|
|
now = self.clock()
|
|
try:
|
|
with closing(sqlite3.connect(f'file:{self.state / "jobs.sqlite"}?mode=ro', uri=True, timeout=2)) as con:
|
|
con.row_factory = sqlite3.Row
|
|
runs = [r[0] for r in con.execute('SELECT run_id FROM runs ORDER BY created_at DESC')]
|
|
rows = [dict(r) for r in con.execute('SELECT * FROM rollouts ORDER BY run_id,job_id,kk')]
|
|
probes, details, snapshots = {}, {}, {}
|
|
quota = self.quota_source() if self.quota_source else {'status': 'UNKNOWN'}
|
|
for rid in runs:
|
|
if not IDENTIFIER.fullmatch(rid) or rid in ('.', '..'):
|
|
continue
|
|
selected = [r for r in rows if r['run_id'] == rid and IDENTIFIER.fullmatch(r['job_id']) and r['job_id'] not in ('.', '..')]
|
|
observed = [self._execution(row, probes, now) for row in selected]
|
|
details.update({(rid, r['job_id'], r['kk']): r for r in observed})
|
|
counts = {'db_active': sum(r['state'] in ACTIVE for r in selected),
|
|
'db_queued': sum(r['state'] == 'QUEUED' for r in selected), 'attempts': len(selected)}
|
|
identities = {name: set() for name in ('agent', 'stage', 'collector', 'supervisor', 'descendant', 'other')}
|
|
unknown = any(r.get('identity_status', 'UNKNOWN') == 'UNKNOWN' and r['state'] in ACTIVE for r in observed)
|
|
for row in observed:
|
|
for p in row['processes']:
|
|
if p['status'] == 'ALIVE':
|
|
role = p['role']
|
|
kind = ('stage' if role in ('stage', 'compiler', 'board') else
|
|
role if role in identities else 'other')
|
|
identities[kind].add(identity_key(p))
|
|
counts.update({kind + '_processes': None if unknown else len(ids) for kind, ids in identities.items()})
|
|
counts['verified_processes'] = {kind: len(ids) for kind, ids in identities.items()}
|
|
legacy = read_json(self.root / 'datasets' / f'{rid}-training' / 'manifest.json', {})
|
|
current = read_json(self.root / 'datasets' / 'CURRENT.json')
|
|
snapshots[rid] = {'run_id': rid, 'snapshot_id': f'{now:.6f}:{rid}', 'sampled_at': now,
|
|
'source_status': 'FRESH', 'counts': counts,
|
|
'db_states': dict(collections.Counter(r['state'] for r in selected)),
|
|
'data': {'mixed_records': legacy.get('stats', {}).get('written'),
|
|
'legacy_candidates': legacy.get('stats', {}).get('sft_eligible'),
|
|
'published_release': {k: current.get(k) for k in ('dataset_version', 'rules_version', 'revision')} if isinstance(current, dict) else None},
|
|
'quota': quota, 'resources': self.resources(observed, now),
|
|
'final_dataset': self.published(now),
|
|
'sources': {'database': {'status': 'FRESH', 'sampled_at': now}, 'quota': {'status': quota.get('status', 'UNKNOWN'), 'sampled_at': now}},
|
|
'alerts': [{'job_id': r['job_id'], 'kk': r['kk'], 'health': r['health'], 'reason': r.get('reason_code')}
|
|
for r in observed if r['health'] in ('UNKNOWN', 'STATE_MISMATCH', 'QUARANTINED', 'SUSPECT', 'STALE')]}
|
|
with self._lock:
|
|
self._snapshots, self._rows, self._previous, self._error = snapshots, details, probes, None
|
|
except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as exc:
|
|
with self._lock:
|
|
self._error = type(exc).__name__
|
|
|
|
def snapshot(self, run_id=None):
|
|
with self._lock:
|
|
if run_id is None:
|
|
run_id = next(iter(self._snapshots), None)
|
|
value = copy.deepcopy(self._snapshots.get(run_id))
|
|
error = self._error
|
|
if value is None:
|
|
return {'run_id': run_id, 'source_status': 'UNAVAILABLE', 'sampled_at': None, 'counts': {}, 'error': error or 'NO_SNAPSHOT'}
|
|
value['age_s'] = max(0, self.clock() - value['sampled_at'])
|
|
if error:
|
|
value.update(source_status='UNAVAILABLE', error=error)
|
|
elif value['age_s'] > self.stale_after:
|
|
value['source_status'] = 'STALE'
|
|
return value
|
|
|
|
def rows(self, run_id, state=None, offset=0, limit=100):
|
|
with self._lock:
|
|
selected = [copy.deepcopy(v) for key, v in self._rows.items() if key[0] == run_id and (not state or v['state'] == state)]
|
|
return {'items': selected[offset:offset+limit], 'total': len(selected), 'snapshot': self.snapshot(run_id)}
|
|
|
|
def detail(self, run_id, job_id, kk):
|
|
with self._lock:
|
|
value = copy.deepcopy(self._rows.get((run_id, job_id, kk)))
|
|
if value is None:
|
|
raise KeyError('ROLLOUT_NOT_FOUND')
|
|
return value
|
|
|
|
|
|
def make_server(observer, address=('127.0.0.1', 8787)):
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
parsed = urlsplit(self.path)
|
|
route = [unquote(s) for s in parsed.path.split('/') if s]
|
|
try:
|
|
if any(not IDENTIFIER.fullmatch(s) or s in ('.', '..') for s in route):
|
|
raise KeyError('INVALID_PATH')
|
|
if not route:
|
|
body = Path(__file__).with_name('monitor.html').read_bytes()
|
|
kind = 'text/html; charset=utf-8'
|
|
else:
|
|
query = parse_qs(parsed.query)
|
|
if route == ['api', 'snapshot']:
|
|
value = observer.snapshot()
|
|
elif len(route) == 4 and route[:2] == ['api', 'runs'] and route[3] == 'snapshot':
|
|
value = observer.snapshot(route[2])
|
|
elif len(route) == 4 and route[:2] == ['api', 'runs'] and route[3] == 'rollouts':
|
|
offset = max(0, int(query.get('cursor', ['0'])[0]))
|
|
limit = min(200, max(1, int(query.get('limit', ['100'])[0])))
|
|
value = observer.rows(route[2], query.get('state', [None])[0], offset, limit)
|
|
elif len(route) == 6 and route[:2] == ['api', 'runs'] and route[3] == 'rollouts':
|
|
value = observer.detail(route[2], route[4], int(route[5]))
|
|
else:
|
|
raise KeyError('NOT_FOUND')
|
|
body, kind = json.dumps(value, ensure_ascii=False).encode(), 'application/json; charset=utf-8'
|
|
self.send_response(200)
|
|
self.send_header('Content-Type', kind)
|
|
self.send_header('Content-Length', str(len(body)))
|
|
self.send_header('Cache-Control', 'no-store')
|
|
self.send_header('X-Content-Type-Options', 'nosniff')
|
|
self.end_headers(); self.wfile.write(body)
|
|
except (KeyError, ValueError):
|
|
self.send_error(404)
|
|
return ThreadingHTTPServer(address, Handler)
|
|
|
|
|
|
def serve(root, *, state_dir=None, host='127.0.0.1', port=8787, interval=5, quota_source=None):
|
|
if quota_source is None:
|
|
def quota_source():
|
|
import yaml
|
|
from .quota import quota_snapshot
|
|
try:
|
|
config = yaml.safe_load((Path(root) / 'config.yaml').read_text())
|
|
ledger = config.get('collection_v1', {}).get('quota_ledger')
|
|
return quota_snapshot(ledger) if ledger else {'status': 'UNKNOWN'}
|
|
except (OSError, ValueError, TypeError, yaml.YAMLError):
|
|
return {'status': 'UNAVAILABLE'}
|
|
observer = Observer(root, state_dir=state_dir, quota_source=quota_source)
|
|
observer.sample()
|
|
stop = threading.Event()
|
|
def sample():
|
|
while not stop.wait(interval):
|
|
observer.sample()
|
|
thread = threading.Thread(target=sample, name='monitor-observer', daemon=True)
|
|
thread.start()
|
|
server = make_server(observer, (host, port))
|
|
try:
|
|
server.serve_forever()
|
|
finally:
|
|
stop.set(); server.server_close(); thread.join(timeout=interval+3)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[1])
|
|
p.add_argument('--host', default='127.0.0.1'); p.add_argument('--port', type=int, default=8787)
|
|
a = p.parse_args(); serve(a.root, host=a.host, port=a.port)
|