data-gen-bpu/scripts/test_collection_crash.py
ming01.wang 498187569c fix: close rollout finalization and review publication gaps
Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
2026-09-29 16:09:01 +08:00

231 lines
14 KiB
Python

"""Remaining collector/transport crash contracts using real SDK and local HTTP.
Only frozen job preparation and the physical-board verifier are fixtures. The
collector, Pi adapter, broker, execution ledger and monitor use production code.
"""
from contextlib import ExitStack, closing
from dataclasses import asdict
import json
import os
from pathlib import Path
import shutil
import sqlite3
import subprocess
import sys
import threading
import time
import unittest
from unittest.mock import patch
import yaml
from harness import collection_runtime as runtime, db, execution, monitor, paths, pipeline, quota, rollout
from scripts import test_collection_joint as joint_fixture, test_execution, test_pi_session, test_quota
ROOT = Path(__file__).resolve().parents[1]
@unittest.skipUnless(Path('/proc').is_dir() and test_pi_session.NODE.is_file(), 'Linux and installed Pi SDK required')
class CollectionCrash(unittest.TestCase):
def setup_collection(self, responder, agents=1, **limits):
self.joint = joint_fixture.JointSDKQuota()
self.addCleanup(self.joint.doCleanups)
self.joint.setup_shared(responder, **limits)
self.project = self.joint.template.root/'bench'
self.project.mkdir()
self.state = self.project/'state'; self.state.mkdir()
cfg = yaml.safe_load((ROOT/'config.yaml').read_text())
cfg['paths'].update(work_root=str(self.project/'work'), node_bin=str(test_pi_session.NODE))
cfg['pools']['agent']['workers'] = agents
cfg['rollouts'].update(jail=False, private_home=True, tools=['write'], hard_ceiling_s=25)
cfg['endpoints']['teacher'].update(base_url=self.joint.provider.url, model='fixture', api_key_env='FIXTURE_KEY')
lifecycle = test_execution.policy(); lifecycle['execution_deadline_s'] = 40
cfg['collection_v1'].update(quota_ledger=str(self.joint.broker.path),
quota_groups={'account':asdict(self.joint.broker.groups['account'])},
credential_groups={'teacher':'account'}, lifecycle=lifecycle)
config_path = self.project/'config.yaml'; config_path.write_text(yaml.safe_dump(cfg))
stack = ExitStack(); self.addCleanup(stack.close)
# Subprocesses and the read-only exporter must see the same fixture DB.
stack.enter_context(patch.dict(os.environ, {'BENCH_STATE_DIR': str(self.state)}))
original_connect, original_readonly = db.connect, db.readonly
for name,value in {'BENCH_ROOT':self.project,'PROJECT_ROOT':self.project.parent,'AGENT':ROOT/'agent',
'RUNS':self.project/'runs','STATE':self.state,'CONFIG_PATH':config_path}.items():
stack.enter_context(patch.object(paths,name,value))
stack.enter_context(patch.object(paths,'config',return_value=cfg))
stack.enter_context(patch.object(paths,'secret',return_value='upstream-fixture'))
stack.enter_context(patch.object(db,'connect',lambda **kwargs:original_connect(self.state/'jobs.sqlite',**kwargs)))
stack.enter_context(patch.object(db,'readonly',lambda:original_readonly(self.state/'jobs.sqlite')))
stack.enter_context(patch.object(runtime,'_broker',self.joint.broker))
stack.enter_context(patch.object(runtime,'_configuration',json.dumps(cfg['collection_v1'],sort_keys=True)))
stack.enter_context(patch.object(rollout,'prepare',side_effect=self.prepare))
stack.enter_context(patch.object(rollout,'verify_rollout',return_value={
'terminal':'UNVERIFIED','claim_consistent':False,'review':False}))
from export import to_agentrl
stack.enter_context(patch.object(to_agentrl,'BENCH',self.project))
self.prepared = []
with db.connect() as con:
db.create_run(con,'r',{})
for n in range(agents): db.rollout_upsert(con,'r',f'j{n}',0,'e2e','teacher')
def prepare(self, run_id, job_id, kk, mode, policy, **_):
rd=self.project/'runs'/run_id/job_id/'rollouts'/f'{kk:02d}'
workspace=rd/'workspace'; workspace.mkdir(parents=True)
(rd/'agent').mkdir(); (rd/'handles').mkdir()
shutil.copytree(self.joint.template.home,workspace/'.agent-home')
shutil.copytree(self.joint.template.root/'.pi',workspace/'.pi')
model_path=workspace/'.agent-home/.pi/agent/models.json'
models=json.loads(model_path.read_text())
models['providers']['dr-gateway']=models['providers'].pop('fixture-provider')
model_path.write_text(json.dumps(models))
settings_path=workspace/'.pi/settings.json'
settings=json.loads(settings_path.read_text());settings['defaultProvider']='dr-gateway'
settings_path.write_text(json.dumps(settings))
for name in ('system.md','append-system.md','prompt.md'):
(rd/'agent'/name).write_text(f'{job_id}: write once, then report done.')
metadata={'state':'RUNNING','runner':'pi','runner_version':'pi-sdk-v1','policy':policy,'mode':mode,
'jail_policy':'private-home','max_turns':60,'max_tokens':200000,'model':'fixture'}
(rd/'rollout.json').write_text(json.dumps(metadata))
job={'run_id':run_id,'job_id':job_id,'task':'cls','march':'nash-p','backend':'ptq'}
(rd.parent.parent/'job.json').write_text(json.dumps(job))
self.prepared.append((run_id,job_id,kk))
return rd,job,(rd/'agent/prompt.md').read_text()
def rows(self):
with db.readonly() as con:
return [dict(r) for r in con.execute('SELECT * FROM rollouts ORDER BY job_id,kk')]
@staticmethod
def receipt(case, **facts):
print('JOINT_RECEIPT '+json.dumps({'case':case,**facts},sort_keys=True))
def test_collector_multiple_sdk_429_does_not_allocate_empty_replacements(self):
self.setup_collection(lambda *_:(429,{'Retry-After':'.08'},b'{"error":{"type":"rate_limit"}}'),agents=2)
before=[(r['run_id'],r['job_id'],r['kk']) for r in self.rows()]
first=pipeline.collect_rollouts('r',parallel=2,limit=2,log=lambda _:None)
self.assertEqual(first,{'PAUSED_INFRA':2})
rows=self.rows()
self.assertEqual([(r['run_id'],r['job_id'],r['kk']) for r in rows],before)
self.assertTrue(all(r['state']=='PAUSED_INFRA' for r in rows))
frozen={}; agent_identities=set()
for _,job_id,kk in before:
rd=self.project/'runs/r'/job_id/'rollouts'/f'{kk:02d}'
state=execution.Execution(rd).snapshot()
self.assertEqual(state['execution_state'],'PAUSED_INFRA')
self.assertEqual(execution.owned_processes(state),[])
self.assertFalse((rd/'workspace/side-effect.txt').exists())
for process in state['processes']:
if process.get('role')=='agent':
self.assertEqual(execution.process_status(process),'DEAD')
agent_identities.add(execution.process_key(process))
frozen[job_id]=(state['execution_id'],state['deadline_monotonic'])
self.assertEqual(len(agent_identities),2)
requests=len(self.joint.provider.requests)
self.assertEqual(requests,6)
self.assertEqual(pipeline.collect_rollouts('r',parallel=2,limit=2,log=lambda _:None),{})
self.assertEqual([(r['run_id'],r['job_id'],r['kk']) for r in self.rows()],before)
self.assertEqual(len(self.prepared),2)
self.assertEqual(len(self.joint.provider.requests),requests)
for _,job_id,kk in before:
rd=self.project/'runs/r'/job_id/'rollouts'/f'{kk:02d}'
state=execution.Execution(rd).snapshot()
self.assertEqual((state['execution_id'],state['deadline_monotonic']),frozen[job_id])
self.assertEqual(len(list(rd.parent.iterdir())),1)
self.assertTrue(all(g['attempts']==3 for g in self.joint.broker.snapshot()['generations']))
self.receipt('JOINT-001-collector',sdk_processes=2,original_attempts=2,final_attempts=2,
upstream_requests=requests,restarted_collector_extra_requests=0,tool_executions=0)
def test_sent_request_broker_sigkill_restart_never_replays_unknown(self):
fixture=test_quota.Quota();fixture.setUp();self.addCleanup(fixture.doCleanups)
entered,release=threading.Event(),threading.Event()
def pending(n,body):
entered.set();release.wait(5)
return test_quota.ok(n,body)
provider=fixture.provider(pending)
policy=fixture.policy(request_total_s=.4,first_byte_timeout_s=1,wait_budget_s=.03,window_s=60)
program=r'''
import json,sys
from pathlib import Path
from harness.quota import Broker,QuotaPolicy,Upstream
from scripts.test_quota import Quota
ledger,url,raw=sys.argv[1:]
with Broker(Path(ledger),{'shared':QuotaPolicy(**json.loads(raw))},
{'teacher':Upstream('shared',url,'upstream-secret',('glm-test',))}) as b:
scope=b.issue_scope('crashed-execution','teacher','glm-test')
Quota().request(scope,'crashed-generation')
'''
child=subprocess.Popen([sys.executable,'-c',program,str(fixture.ledger),provider.url,json.dumps(asdict(policy))],
cwd=ROOT,env={**os.environ,'PYTHONPATH':str(ROOT),'PYTHONDONTWRITEBYTECODE':'1'},
stdout=subprocess.PIPE,stderr=subprocess.PIPE)
def cleanup():
release.set()
if child.poll() is None:child.kill()
child.communicate(timeout=3)
self.addCleanup(cleanup)
self.assertTrue(entered.wait(5))
deadline=time.monotonic()+1
while True:
with closing(sqlite3.connect(f'file:{fixture.ledger}?mode=ro',uri=True)) as con:
before=con.execute('SELECT state,sent,ended_at,charged_tokens FROM attempts').fetchone()
if before and before[0]=='SENT':break
self.assertLess(time.monotonic(),deadline);time.sleep(.005)
self.assertEqual(before[1:3],(1,None))
self.assertGreater(before[3],0)
child.kill();child.communicate(timeout=3);self.assertEqual(child.returncode,-9)
release.set();time.sleep(.45)
broker=fixture.broker(provider,policy)
replay=broker.issue_scope('crashed-execution','teacher','glm-test')
self.assertNotEqual(fixture.request(replay,'crashed-generation')[0],200)
other=broker.issue_scope('new-execution','teacher','glm-test')
self.assertNotEqual(fixture.request(other,'next-generation')[0],200)
snapshot=quota.quota_snapshot(fixture.ledger)
self.assertEqual(len(provider.requests),1)
self.assertEqual(len(snapshot['attempts']),1)
attempt=snapshot['attempts'][0]
self.assertEqual(attempt['state'],'UNKNOWN');self.assertIsNone(attempt['ended_at'])
self.assertEqual(attempt['charged_tokens'],before[3])
self.assertEqual(snapshot['groups'][0]['rpm_used'],1)
self.assertEqual(snapshot['groups'][0]['inflight'],1)
self.assertGreater(snapshot['groups'][0]['tpm_reserved'],0)
self.assertEqual(broker.execution_status('crashed-execution')['generations'][0]['attempts'],1)
self.receipt('JOINT-003/009-broker-crash',process_exit=-9,upstream_requests=1,
restart_extra_requests=0,attempts=1,state='UNKNOWN',reserved_tokens=before[3])
def test_busy_total_timeout_converges_sdk_execution_db_and_monitor(self):
def busy(_n,_body):
def stream(output):
output.write(joint_fixture.write_response()[2].split(b'\n\n')[0]+b'\n\n');output.flush()
for _ in range(80):
time.sleep(.015)
output.write(b'data: {"choices":[{"index":0,"delta":{"content":"tick"},"finish_reason":null}]}\n\n')
output.flush()
return 200,{'Content-Type':'text/event-stream'},stream
self.setup_collection(busy,stream_idle_timeout_s=.2,first_byte_timeout_s=.3,request_total_s=.35)
started=time.monotonic()
result=pipeline.collect_rollouts('r',limit=1,log=lambda _:None)
self.assertLess(time.monotonic()-started,12)
self.assertEqual(result,{'PAUSED_INFRA':1})
self.assertEqual([r['state'] for r in self.rows()],['PAUSED_INFRA'])
rd=self.project/'runs/r/j0/rollouts/00'
state=execution.Execution(rd).snapshot()
self.assertEqual(state['execution_state'],'PAUSED_INFRA')
self.assertEqual(execution.owned_processes(state),[])
self.assertFalse((rd/'workspace/side-effect.txt').exists())
attempts=self.joint.broker.snapshot()['attempts']
self.assertEqual(len(attempts),1)
self.assertEqual(attempts[0]['transport_reason'],'REQUEST_TOTAL_TIMEOUT')
self.assertEqual(len(self.joint.provider.requests),1)
observer=monitor.Observer(self.project,quota_source=lambda:quota.quota_snapshot(self.joint.broker.path))
observer.sample();snapshot=observer.snapshot('r');detail=observer.detail('r','j0',0)
self.assertEqual(snapshot['source_status'],'FRESH')
# The dashboard intentionally counts paused attempts as DB-active.
# Assert their actual state instead of treating db_active as RUNNING.
self.assertEqual(snapshot['counts']['agent_processes'],0)
self.assertEqual(snapshot['db_states'],{'PAUSED_INFRA':1})
self.assertEqual(snapshot['db_states'].get('RUNNING',0),0)
self.assertEqual((detail['state'],detail['execution_state']),('PAUSED_INFRA','PAUSED_INFRA'))
self.assertEqual(snapshot['quota']['attempts'][0]['transport_reason'],'REQUEST_TOTAL_TIMEOUT')
self.receipt('JOINT-004-total',upstream_requests=1,tool_executions=0,transport_reason='REQUEST_TOTAL_TIMEOUT',
db_state=detail['state'],execution_state=detail['execution_state'],monitor_agents=0,monitor_db_running=0,monitor_db_active=snapshot['counts']['db_active'])
if __name__=='__main__':unittest.main()