data-gen-bpu/scripts/test_lifecycle_joint.py
ming01.wang 2f54d407cb fix: recover from unknown request outcomes without stopping the collector
- quota: once an UNKNOWN attempt is a full token window past its local
  deadline, abandon it via the operator path (evidence kept, ended_at NULL,
  generation fenced); the group then gets one probe.
- quota: generations paused at admission by that group pause may resume
  when none of their own attempts is unended.
- pipeline: isolate an identifiable unreconciled rollout and keep
  collecting; unidentifiable reports still stop. Drive broker reconcile
  every loop so a fully paused group can recover.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 21:18:11 +08:00

247 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Actual lifecycle boundaries with owned processes, queues and read-only UI API."""
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
import urllib.request
from harness import collection_runtime, db, execution, monitor, paths, rollout, scheduler, watchdog
from scripts import test_execution, test_quota
ROOT = Path(__file__).resolve().parents[1]
@unittest.skipUnless(sys.platform == 'linux', 'Linux process identities and kernel containment')
class LifecycleJoint(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(prefix='bench-lifecycle-joint-')
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
self.rd = self.root/'runs/r/j/rollouts/00'; self.rd.mkdir(parents=True)
self.database = self.root/'state/jobs.sqlite'
self.connect, self.readonly = db.connect, db.readonly
with self.connect(self.database) as con:
db.create_run(con, 'r', {})
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
(self.rd/'rollout.json').write_text('{"state":"RUNNING","mode":"e2e","policy":"teacher"}')
self.config = {'pools': {'agent': {'workers': 1}, 'cpu': {'workers': 1}, 'board': {'workers': 1}}}
self.stack = ExitStack(); self.addCleanup(self.stack.close)
self.stack.enter_context(patch.object(paths, 'STATE', self.root/'state'))
self.stack.enter_context(patch.object(paths, 'RUNS', self.root/'runs'))
self.stack.enter_context(patch.object(paths, 'config', return_value=self.config))
self.stack.enter_context(patch.object(db, 'connect', lambda: self.connect(self.database)))
self.stack.enter_context(patch.object(db, 'readonly', lambda: self.readonly(self.database)))
self.stack.enter_context(patch.object(scheduler, 'board_slots', return_value={0: {'name': 'local-lock-only'}}))
def wait(self, predicate, timeout=5):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
result = predicate()
if result:
return result
time.sleep(.005)
self.fail('bounded fixture condition did not arrive')
@staticmethod
def receipt(case, **facts):
print('JOINT_RECEIPT '+json.dumps({'case': case, **facts}, sort_keys=True))
def test_cancel_same_execution_fences_quota_compile_and_board_queues(self):
owner = execution.Execution(self.rd)
policy = test_execution.policy(); policy['queue_deadline_s'] = 5
for phase in policy['phases'].values(): phase['queue_deadline_s'] = 5
owner.start(policy)
quota = test_quota.Quota(); quota.setUp(); self.addCleanup(quota.doCleanups)
entered, release = threading.Event(), threading.Event()
def held_provider(n, body):
entered.set()
if not release.wait(5): raise TimeoutError('fixture provider release missing')
return test_quota.ok(n, body)
provider = quota.provider(held_provider)
broker = quota.broker(provider, quota.policy(max_concurrency=1, wait_budget_s=5,
first_byte_timeout_s=5, request_total_s=6))
self.stack.enter_context(patch.object(collection_runtime, 'broker', return_value=broker))
hold_scope = broker.issue_scope('fixture-slot-holder', 'teacher', 'glm-test')
target = broker.issue_scope(owner.snapshot()['execution_id'], 'teacher', 'glm-test')
attempted = {name: threading.Event() for name in ('cpu', 'board')}
def queued_stage(pool, phase):
operation = 'queued-'+pool
owner.begin_phase(operation, phase)
def observed_wait():
attempted[pool].set()
return None
try:
with owner.scope(phase, operation), scheduler.acquire(pool, poll_s=.01, stop_check=observed_wait):
return watchdog.run([sys.executable, '-c', 'from pathlib import Path;Path("unexpected-spawn").touch()'],
cwd=self.rd, stdout=self.rd/(pool+'.out'), stderr=self.rd/(pool+'.err'), execution=owner,
role='stage', poll_s=.01).state
except execution.ExecutionStopped as error:
return str(error)
with ExitStack() as holds:
holds.enter_context(scheduler.acquire('cpu'))
holds.enter_context(scheduler.acquire('board'))
with ThreadPoolExecutor(max_workers=4) as workers:
active = workers.submit(quota.request, hold_scope, 'held')
pending = []
try:
self.assertTrue(entered.wait(3))
request = workers.submit(quota.request, target, 'queued')
pending = [workers.submit(queued_stage, pool, phase)
for pool, phase in [('cpu', '05-compile'), ('board', '07-board')]]
self.wait(lambda: all(event.is_set() for event in attempted.values()))
self.wait(lambda: any(g['state']=='QUEUED' for g in broker.execution_status(target.execution_id)['generations']))
before = len(provider.requests)
settled = rollout.settle_execution('r', 'j', 0, owner, 'joint-cancel', cancel=True)
self.assertEqual(settled['execution_outcome'], 'CANCELLED')
self.assertEqual([future.result(timeout=3) for future in pending], ['CANCEL_REQUESTED']*2)
self.assertEqual(request.result(timeout=3)[0], 409)
self.assertEqual(len(provider.requests), before)
self.assertEqual(owner.snapshot()['processes'], [])
self.assertFalse((self.rd/'unexpected-spawn').exists())
generation = broker.execution_status(target.execution_id)['generations'][0]
self.assertEqual((generation['state'], generation['attempts']), ('CANCELLED', 0))
finally:
release.set()
self.assertEqual(active.result(timeout=3)[0], 200)
self.assertEqual(scheduler.pool_usage(), {'agent': 0, 'cpu': 0, 'board': 0})
self.assertEqual(list((self.root/'state/locks').glob('*.lease.json')), [])
self.assertTrue(all(a['ended_at'] is not None for a in broker.snapshot()['attempts']))
self.assertEqual(owner.snapshot()['cleanup_state'], 'VERIFIED')
self.receipt('JOINT-005', queued_pools=['quota','cpu','board'], target_http_attempts=0,
target_spawns=0, upstream_requests=1, owned_slots_after=0, cleanup='VERIFIED')
def test_silent_work_spin_and_hang_report_real_progress_and_absolute_deadlines(self):
observer = monitor.Observer(self.root)
server = monitor.make_server(observer, ('127.0.0.1', 0))
thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start()
self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
facts = []
programs = {
'working': 'import os,time;f=open("work.bin","wb");end=time.monotonic()+.35\n'
'while time.monotonic()<end:\n f.write(b"x"*4096);f.flush();os.fsync(f.fileno());sum(i*i for i in range(20000))\n',
'spinning': 'while True: pass',
'hanging': 'import signal;signal.pause()',
}
for kind, program in programs.items():
with self.subTest(kind=kind):
rd = self.root/'runs/r'/kind/'rollouts/00'; rd.mkdir(parents=True)
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', kind, 0, 'e2e', 'teacher', state='RUNNING')
owner = execution.Execution(rd)
policy = test_execution.policy()
policy['phases']['05-compile'].update(execution_deadline_s=1.5 if kind=='working' else .35,
queue_deadline_s=1, suspect_after_s=.08)
owner.start(policy); owner.begin_phase('compile', '05-compile'); owner.phase_running('compile')
owner.activity('COMPILING', operation_id='compile')
observed = []
def sample():
observer.sample()
observed.append(observer.detail('r', kind, 0))
started = time.monotonic()
with owner.scope('05-compile', 'compile'):
result = watchdog.run([sys.executable, '-c', program], cwd=rd, stdout=rd/'out', stderr=rd/'err',
execution=owner, role='stage', poll_s=.02, stop_check=sample)
elapsed = time.monotonic()-started
state = owner.finish(result.exit_code, protocol_complete=result.state=='OK',
stop_reason=result.stalled_reason, cleanup_state=result.cleanup_state)
self.assertLess(elapsed, 3)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertEqual(execution.owned_processes(state), [])
self.assertEqual((rd/'out').read_bytes(), b'')
if kind=='working':
self.assertEqual(result.state, 'OK')
self.assertGreater((rd/'work.bin').stat().st_size, 0)
else:
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertIn('DEADLINE', result.stalled_reason)
progress = [row['progress'] for row in observed if (row.get('progress') or {}).get('signals')]
self.assertTrue(progress)
self.assertTrue(all(row['suspect_after_s']==.08 for row in progress))
self.assertTrue(all(row['signals']['business_log_bytes']==0 for row in progress))
if kind in ('working','spinning'):
self.assertGreater(max(row['signals']['cpu_ticks'] or 0 for row in progress),
min(row['signals']['cpu_ticks'] or 0 for row in progress))
else:
self.assertTrue(any(row['reason']=='ALL_SIGNALS_IDLE' for row in progress))
observer.sample()
url=f'http://127.0.0.1:{server.server_port}/api/runs/r/rollouts/{kind}/0'
with urllib.request.urlopen(url, timeout=3) as response: displayed=json.load(response)
phase=displayed['phases']['compile']
self.assertEqual(phase['phase'], '05-compile')
self.assertAlmostEqual(phase['execution_deadline_monotonic']-phase['execution_started_monotonic'],
policy['phases']['05-compile']['execution_deadline_s'])
self.assertEqual(displayed['progress']['suspect_after_s'], .08)
self.assertEqual(displayed['stop_reason'], state['stop_reason'])
self.assertTrue(displayed['progress']['reason'])
facts.append({'kind':kind,'outcome':state['execution_outcome'],'cleanup':result.cleanup_state,
'elapsed_s':round(elapsed,3),'phase':'05-compile','suspect_after_s':.08,
'display_reason':displayed['progress']['reason'],'log_bytes':0})
self.receipt('JOINT-006', processes=facts, scope='actual process telemetry and monitor HTTP detail; no compiler/board model workload')
def test_old_boot_identity_is_quarantined_before_real_collector_claims(self):
victim = subprocess.Popen([sys.executable, '-c', 'import time;time.sleep(15)'])
def stop():
if victim.poll() is None: victim.kill()
victim.wait(timeout=3)
self.addCleanup(stop)
owner = execution.Execution(self.rd); owner.start(test_execution.policy())
before = owner.snapshot()
wrong = execution.process_identity(victim.pid)
wrong.update(boot_id='old-boot-fixture', proc_start_ticks=wrong['proc_start_ticks']+1)
before.update(boot_id='old-boot-fixture', owner=wrong, processes=[dict(wrong, role='stage')])
execution.atomic_json(self.rd/'execution.json', before)
lockdir=self.root/'state/locks';lockdir.mkdir()
lease=lockdir/'board-0.lease.json'
execution.atomic_json(lease, {'execution_id':before['execution_id'],'execution_dir':str(self.rd),
'epoch':before['epoch'],'holder':wrong,'pool':'board','slot':0,'operation_id':'unknown-board'})
lease_bytes=lease.read_bytes()
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', 'next', 0, 'e2e', 'teacher')
program = '''
import json
from harness import collection_runtime,db,paths,pipeline
paths.config=lambda:{'pools':{'agent':{'workers':1}}}
collection_runtime.config=lambda:({}, {})
claims=[]
def forbidden(*args,**kwargs):
claims.append(True);raise AssertionError('new task admitted before reconciliation')
db.claim_rollout=forbidden
logs=[]
pipeline.collect_rollouts('r',log=logs.append)
assert any('已隔离:OWNERSHIP_UNKNOWN' in line for line in logs),logs
print(json.dumps({'new_claims':len(claims),'result':'QUARANTINED'}))
'''
env={k:v for k,v in os.environ.items() if not k.startswith('BENCH_')}
env.update(BENCH_ROOT=str(self.root), BENCH_STATE_DIR=str(self.root/'state'),
PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE='1')
child=subprocess.run([sys.executable,'-c',program],cwd=ROOT,env=env,capture_output=True,text=True,timeout=8)
self.assertEqual(child.returncode,0,child.stderr)
self.assertEqual(json.loads(child.stdout)['new_claims'],0)
after=owner.snapshot()
self.assertEqual(after['cleanup_state'],'QUARANTINED')
self.assertEqual(after['stop_reason'],'OWNERSHIP_UNKNOWN')
self.assertGreater(after['epoch'],before['epoch'])
self.assertEqual((after['execution_id'],after['deadline_monotonic']),
(before['execution_id'],before['deadline_monotonic']))
with self.assertRaisesRegex(execution.ExecutionStopped,'OWNER_FENCED'):
with owner.admission(): self.fail('old epoch reopened admission')
self.assertIsNone(victim.poll())
self.assertEqual(lease.read_bytes(),lease_bytes)
with self.readonly(self.database) as con:
self.assertEqual([(r['job_id'],r['state']) for r in con.execute('SELECT job_id,state FROM rollouts ORDER BY job_id')],
[('j','RUNNING'),('next','QUEUED')])
self.receipt('JOINT-010-boot', collector_processes=1,new_claims=0,cleanup='QUARANTINED',
unrelated_birth_identity_alive=True,board_lease_preserved=True,old_epoch_fenced=True)
if __name__ == '__main__':
unittest.main()