ming01.wang 498187569c fix: close rollout finalization and review publication gaps
Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
2026-09-29 16:09:01 +08:00

349 lines
20 KiB
Python

"""Cached, read-only collection monitor. HTTP handlers never inspect or run jobs."""
from __future__ import annotations
import argparse
import collections
from contextlib import closing
import copy
import json
import hashlib
import re
import socket
import sqlite3
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, unquote, urlsplit
ACTIVE = {'RUNNING', 'VERIFYING', 'PAUSED_INFRA', 'CLEANING'}
AGENT_PHASES = {'WAITING_LLM', 'RUNNING_TOOL', 'COMPILING', 'RUNNING_BOARD'}
IDENTIFIER = re.compile(r'^[A-Za-z0-9_.-]+$')
def read_json(path: Path, default=None):
try:
return json.loads(path.read_text())
except (OSError, ValueError):
return default
def process_probe(identity: dict) -> dict:
"""PID reuse, another host and unreadable procfs are distinct from dead."""
try:
if identity.get('host') != socket.gethostname():
return {'status': 'UNKNOWN'}
if not identity.get('boot_id') or not isinstance(identity.get('proc_start_ticks'), int):
return {'status': 'UNKNOWN'}
boot = Path('/proc/sys/kernel/random/boot_id').read_text().strip()
if identity.get('boot_id') != boot:
return {'status': 'DEAD'}
proc = Path('/proc') / str(int(identity['pid']))
try:
stat = (proc / 'stat').read_text().rsplit(')', 1)[1].split()
except FileNotFoundError:
return {'status': 'DEAD'}
if int(stat[19]) != int(identity['proc_start_ticks']):
return {'status': 'REUSED'}
io = {}
try:
io = dict(line.split(': ', 1) for line in (proc / 'io').read_text().splitlines())
except OSError:
pass
return {'status': 'DEAD' if stat[0] == 'Z' else 'ALIVE', 'state': stat[0],
'cpu_ticks': int(stat[11]) + int(stat[12]),
'io_bytes': sum(int(io.get(k, 0)) for k in ('read_bytes', 'write_bytes')) if io else None}
except (OSError, ValueError, KeyError, IndexError, TypeError):
return {'status': 'UNKNOWN'}
def identity_key(record):
return tuple(record.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks'))
class Observer:
def __init__(self, root: Path, *, state_dir=None, process_probe=process_probe,
clock=time.time, stale_after=15, quota_source=None):
self.root = Path(root).resolve()
self.state = Path(state_dir) if state_dir else self.root / 'state'
self.probe, self.clock, self.stale_after = process_probe, clock, stale_after
self.quota_source = quota_source
self._lock = threading.Lock()
self._snapshots, self._rows, self._previous = {}, {}, {}
self._error = None
try:
self.local_boot = Path('/proc/sys/kernel/random/boot_id').read_text().strip()
except OSError:
self.local_boot = None
def _execution(self, row, probes, now):
rd = self.root / 'runs' / row['run_id'] / row['job_id'] / 'rollouts' / f"{row['kk']:02d}"
record = read_json(rd / 'execution.json')
rj = read_json(rd / 'rollout.json', {})
result = {k: row.get(k) for k in ('run_id', 'job_id', 'kk', 'state', 'started_at', 'finished_at', 'verdict_terminal')}
result.update(execution_state=rj.get('state'), execution_outcome=None, health='UNKNOWN',
processes=[], activity='UNKNOWN', cleanup_state='UNKNOWN')
if not isinstance(record, dict):
result['reason_code'] = 'EXECUTION_IDENTITY_NOT_RECORDED'
return result
for key in ('phases', 'execution_id', 'epoch', 'activity', 'execution_state', 'execution_outcome',
'cleanup_state', 'stop_reason', 'cancel_requested', 'config_version', 'updated_at',
'deadline_monotonic', 'activity_details', 'progress', 'health', 'cleanup_errors'):
result[key] = record.get(key)
identities = list(record.get('processes', []))
if record.get('owner'):
identities.append({**record['owner'], 'role': 'collector'})
for item in identities:
if not isinstance(item, dict):
continue
identity = item.get('identity', item)
key = identity_key(identity)
if key not in probes:
probes[key] = self.probe(identity)
observation = probes[key]
safe = {k: identity.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks')}
safe.update(role=item.get('role', 'unknown'), **observation)
previous = self._previous.get(key)
if previous and observation.get('status') == previous.get('status') == 'ALIVE':
for metric in ('cpu_ticks', 'io_bytes'):
if isinstance(observation.get(metric), (int, float)) and isinstance(previous.get(metric), (int, float)):
safe[metric + '_delta'] = max(0, observation[metric] - previous[metric])
result['processes'].append(safe)
unknown = not record.get('owner') or any(p['status'] == 'UNKNOWN' for p in result['processes'])
result['identity_status'] = 'UNKNOWN' if unknown else 'CONFIRMED'
result['sources'] = {name: self.freshness(at, now) for name, at in
[('execution', record.get('updated_at')), ('progress', (record.get('progress') or {}).get('sampled_at'))]}
active = record.get('execution_state') == 'ACTIVE'
expected_agent = active and record.get('activity') in AGENT_PHASES
agents = [p for p in result['processes'] if p['role'] == 'agent']
started = row.get('started_at') or now
if unknown:
result.update(health='UNKNOWN', reason_code='PROCESS_SOURCE_UNAVAILABLE')
elif expected_agent and now - started > 30 and not any(p['status'] == 'ALIVE' for p in agents):
result.update(health='STATE_MISMATCH', reason_code='EXPECTED_AGENT_MISSING')
elif record.get('cleanup_state') in ('UNKNOWN', 'QUARANTINED'):
result.update(health='QUARANTINED', reason_code=record.get('stop_reason') or 'CLEANUP_UNCONFIRMED')
elif active and result['sources']['progress']['status'] == 'STALE':
result.update(health='STALE', reason_code='PROGRESS_SNAPSHOT_STALE')
elif record.get('health') in ('SUSPECT', 'UNKNOWN'):
result.update(health=record['health'], reason_code=(record.get('progress') or {}).get('reason', 'PROGRESS_NOT_CONFIRMED'))
else:
result.update(health='WAITING' if str(record.get('activity', '')).startswith('WAITING') else 'OBSERVED',
reason_code='IDENTITY_CHECKED')
result['pending_registration'] = [p.parent.name for p in sorted((rd / 'handles').glob('*/status.json'))
if isinstance(status := read_json(p), dict) and status.get('registration') == 'PENDING']
if result['pending_registration'] and result['health'] not in ('QUARANTINED', 'STATE_MISMATCH', 'UNKNOWN'):
result.update(health='WAITING', reason_code='STAGE_REGISTRATION_PENDING')
# Exported and actual execution facts stay separate. No inference from exit 0.
verdict = read_json(rd / 'verdict.json', {})
result['hard_gates'] = {k: {'verdict': v.get('verdict'), 'reason': v.get('reason')}
for k, v in verdict.get('gates', {}).items() if isinstance(v, dict)}
result['claim_consistent'] = verdict.get('claim_consistent')
result['evidence'] = [name for name in ('rollout.json', 'execution.json', 'verdict.json') if (rd / name).is_file()]
return result
def freshness(self, at, now):
if not isinstance(at, (float, int)):
return {'status': 'UNKNOWN', 'sampled_at': None}
age = max(0, now - at)
return {'status': 'STALE' if age > self.stale_after else 'FRESH', 'sampled_at': at, 'age_s': age}
def resources(self, observed, now):
locks = self.state / 'locks'
leases = []
for suffix, state in (('lease', 'HELD'), ('quarantine', 'QUARANTINED')):
for path in sorted(locks.glob('*.' + suffix + '.json'))[:1000]:
row = read_json(path)
if not isinstance(row, dict):
leases.append({'resource': path.name, 'state': 'UNKNOWN'})
continue
leases.append({'resource': path.name.split('.' + suffix)[0], 'state': state,
**{k: row.get(k) for k in ('execution_id', 'epoch', 'pool', 'slot', 'phase', 'operation_id', 'acquired_at', 'reason')},
'holder': {k: row.get('holder', {}).get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks')}})
waits = []
for row in observed:
for operation, phase in (row.get('phases') or {}).items():
if phase.get('state') == 'QUEUED':
waits.append({'job_id': row['job_id'], 'kk': row['kk'], 'operation_id': operation,
**phase, 'wait_s': max(0, time.monotonic() - phase['queued_monotonic'])
if phase.get('boot_id') == self.local_boot and isinstance(phase.get('queued_monotonic'), (int, float)) else None})
return {'status': 'OBSERVED' if locks.is_dir() else 'UNKNOWN', 'sampled_at': now,
'leases': leases, 'waiting': waits, 'waiting_count': len(waits)}
def published(self, now):
pointer = read_json(self.root / 'datasets/CURRENT.json')
if not isinstance(pointer, dict):
return {'status': 'UNKNOWN', 'reason': 'NO_PUBLISHED_GENERATION'}
version = pointer.get('dataset_version', '')
if not re.fullmatch(r'sft-[0-9a-f]{24}', version):
return {'status': 'UNKNOWN', 'reason': 'INVALID_POINTER'}
folder = self.root / 'datasets' / version
try:
release_bytes = (folder / 'release.json').read_bytes()
release = json.loads(release_bytes)
data = (folder / 'acceptance.json').read_bytes()
manifest_bytes = (folder / 'dataset-manifest.json').read_bytes()
if (hashlib.sha256(release_bytes).hexdigest() != pointer.get('release_sha256')
or hashlib.sha256(data).hexdigest() != release.get('acceptance_sha256')
or hashlib.sha256(manifest_bytes).hexdigest() != release.get('manifest_sha256')):
raise ValueError('hash mismatch')
accepted, manifest = json.loads(data), json.loads(manifest_bytes)
revocation_path = self.root / 'datasets/revocations.json'
revoked = read_json(revocation_path) if revocation_path.exists() else {'versions': {}}
if not isinstance(revoked, dict) or not isinstance(revoked.get('versions'), dict):
raise ValueError('revocations unavailable')
return {'status': 'REVOKED' if version in revoked.get('versions', {}) else 'VERIFIED_MANIFEST',
'scope': 'published_generation', 'dataset_version': version, 'rules_version': manifest['rules_version'],
'accepted': accepted['accepted'], 'retained': accepted.get('retained'),
'quality': accepted.get('data_quality', {'status': 'UNKNOWN'}),
'source': 'immutable acceptance.json', 'sampled_at': now}
except (OSError, ValueError, KeyError, TypeError):
return {'status': 'UNKNOWN', 'reason': 'PUBLISHED_METADATA_UNVERIFIED', 'sampled_at': now}
def sample(self):
now = self.clock()
try:
with closing(sqlite3.connect(f'file:{self.state / "jobs.sqlite"}?mode=ro', uri=True, timeout=2)) as con:
con.row_factory = sqlite3.Row
runs = [r[0] for r in con.execute('SELECT run_id FROM runs ORDER BY created_at DESC')]
rows = [dict(r) for r in con.execute('SELECT * FROM rollouts ORDER BY run_id,job_id,kk')]
probes, details, snapshots = {}, {}, {}
quota = self.quota_source() if self.quota_source else {'status': 'UNKNOWN'}
for rid in runs:
if not IDENTIFIER.fullmatch(rid) or rid in ('.', '..'):
continue
selected = [r for r in rows if r['run_id'] == rid and IDENTIFIER.fullmatch(r['job_id']) and r['job_id'] not in ('.', '..')]
observed = [self._execution(row, probes, now) for row in selected]
details.update({(rid, r['job_id'], r['kk']): r for r in observed})
counts = {'db_active': sum(r['state'] in ACTIVE for r in selected),
'db_queued': sum(r['state'] == 'QUEUED' for r in selected), 'attempts': len(selected)}
identities = {name: set() for name in ('agent', 'stage', 'collector', 'supervisor', 'descendant', 'other')}
unknown = any(r.get('identity_status', 'UNKNOWN') == 'UNKNOWN' and r['state'] in ACTIVE for r in observed)
for row in observed:
for p in row['processes']:
if p['status'] == 'ALIVE':
role = p['role']
kind = ('stage' if role in ('stage', 'compiler', 'board') else
role if role in identities else 'other')
identities[kind].add(identity_key(p))
counts.update({kind + '_processes': None if unknown else len(ids) for kind, ids in identities.items()})
counts['verified_processes'] = {kind: len(ids) for kind, ids in identities.items()}
legacy = read_json(self.root / 'datasets' / f'{rid}-training' / 'manifest.json', {})
current = read_json(self.root / 'datasets' / 'CURRENT.json')
snapshots[rid] = {'run_id': rid, 'snapshot_id': f'{now:.6f}:{rid}', 'sampled_at': now,
'source_status': 'FRESH', 'counts': counts,
'db_states': dict(collections.Counter(r['state'] for r in selected)),
'data': {'mixed_records': legacy.get('stats', {}).get('written'),
'legacy_candidates': legacy.get('stats', {}).get('sft_eligible'),
'published_release': {k: current.get(k) for k in ('dataset_version', 'rules_version', 'revision')} if isinstance(current, dict) else None},
'quota': quota, 'resources': self.resources(observed, now),
'final_dataset': self.published(now),
'sources': {'database': {'status': 'FRESH', 'sampled_at': now}, 'quota': {'status': quota.get('status', 'UNKNOWN'), 'sampled_at': now}},
'alerts': [{'job_id': r['job_id'], 'kk': r['kk'], 'health': r['health'], 'reason': r.get('reason_code')}
for r in observed if r['health'] in ('UNKNOWN', 'STATE_MISMATCH', 'QUARANTINED', 'SUSPECT', 'STALE')]}
with self._lock:
self._snapshots, self._rows, self._previous, self._error = snapshots, details, probes, None
except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as exc:
with self._lock:
self._error = type(exc).__name__
def snapshot(self, run_id=None):
with self._lock:
if run_id is None:
run_id = next(iter(self._snapshots), None)
value = copy.deepcopy(self._snapshots.get(run_id))
error = self._error
if value is None:
return {'run_id': run_id, 'source_status': 'UNAVAILABLE', 'sampled_at': None, 'counts': {}, 'error': error or 'NO_SNAPSHOT'}
value['age_s'] = max(0, self.clock() - value['sampled_at'])
if error:
value.update(source_status='UNAVAILABLE', error=error)
elif value['age_s'] > self.stale_after:
value['source_status'] = 'STALE'
return value
def rows(self, run_id, state=None, offset=0, limit=100):
with self._lock:
selected = [copy.deepcopy(v) for key, v in self._rows.items() if key[0] == run_id and (not state or v['state'] == state)]
return {'items': selected[offset:offset+limit], 'total': len(selected), 'snapshot': self.snapshot(run_id)}
def detail(self, run_id, job_id, kk):
with self._lock:
value = copy.deepcopy(self._rows.get((run_id, job_id, kk)))
if value is None:
raise KeyError('ROLLOUT_NOT_FOUND')
return value
def make_server(observer, address=('127.0.0.1', 8787)):
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_GET(self):
parsed = urlsplit(self.path)
route = [unquote(s) for s in parsed.path.split('/') if s]
try:
if any(not IDENTIFIER.fullmatch(s) or s in ('.', '..') for s in route):
raise KeyError('INVALID_PATH')
if not route:
body = Path(__file__).with_name('monitor.html').read_bytes()
kind = 'text/html; charset=utf-8'
else:
query = parse_qs(parsed.query)
if route == ['api', 'snapshot']:
value = observer.snapshot()
elif len(route) == 4 and route[:2] == ['api', 'runs'] and route[3] == 'snapshot':
value = observer.snapshot(route[2])
elif len(route) == 4 and route[:2] == ['api', 'runs'] and route[3] == 'rollouts':
offset = max(0, int(query.get('cursor', ['0'])[0]))
limit = min(200, max(1, int(query.get('limit', ['100'])[0])))
value = observer.rows(route[2], query.get('state', [None])[0], offset, limit)
elif len(route) == 6 and route[:2] == ['api', 'runs'] and route[3] == 'rollouts':
value = observer.detail(route[2], route[4], int(route[5]))
else:
raise KeyError('NOT_FOUND')
body, kind = json.dumps(value, ensure_ascii=False).encode(), 'application/json; charset=utf-8'
self.send_response(200)
self.send_header('Content-Type', kind)
self.send_header('Content-Length', str(len(body)))
self.send_header('Cache-Control', 'no-store')
self.send_header('X-Content-Type-Options', 'nosniff')
self.end_headers(); self.wfile.write(body)
except (KeyError, ValueError):
self.send_error(404)
return ThreadingHTTPServer(address, Handler)
def serve(root, *, state_dir=None, host='127.0.0.1', port=8787, interval=5, quota_source=None):
if quota_source is None:
def quota_source():
import yaml
from .quota import quota_snapshot
try:
config = yaml.safe_load((Path(root) / 'config.yaml').read_text())
ledger = config.get('collection_v1', {}).get('quota_ledger')
return quota_snapshot(ledger) if ledger else {'status': 'UNKNOWN'}
except (OSError, ValueError, TypeError, yaml.YAMLError):
return {'status': 'UNAVAILABLE'}
observer = Observer(root, state_dir=state_dir, quota_source=quota_source)
observer.sample()
stop = threading.Event()
def sample():
while not stop.wait(interval):
observer.sample()
thread = threading.Thread(target=sample, name='monitor-observer', daemon=True)
thread.start()
server = make_server(observer, (host, port))
try:
server.serve_forever()
finally:
stop.set(); server.server_close(); thread.join(timeout=interval+3)
if __name__ == '__main__':
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[1])
p.add_argument('--host', default='127.0.0.1'); p.add_argument('--port', type=int, default=8787)
a = p.parse_args(); serve(a.root, host=a.host, port=a.port)