Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
231 lines
14 KiB
Python
231 lines
14 KiB
Python
"""Remaining collector/transport crash contracts using real SDK and local HTTP.
|
|
|
|
Only frozen job preparation and the physical-board verifier are fixtures. The
|
|
collector, Pi adapter, broker, execution ledger and monitor use production code.
|
|
"""
|
|
from contextlib import ExitStack, closing
|
|
from dataclasses import asdict
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import yaml
|
|
from harness import collection_runtime as runtime, db, execution, monitor, paths, pipeline, quota, rollout
|
|
from scripts import test_collection_joint as joint_fixture, test_execution, test_pi_session, test_quota
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
@unittest.skipUnless(Path('/proc').is_dir() and test_pi_session.NODE.is_file(), 'Linux and installed Pi SDK required')
|
|
class CollectionCrash(unittest.TestCase):
|
|
def setup_collection(self, responder, agents=1, **limits):
|
|
self.joint = joint_fixture.JointSDKQuota()
|
|
self.addCleanup(self.joint.doCleanups)
|
|
self.joint.setup_shared(responder, **limits)
|
|
self.project = self.joint.template.root/'bench'
|
|
self.project.mkdir()
|
|
self.state = self.project/'state'; self.state.mkdir()
|
|
cfg = yaml.safe_load((ROOT/'config.yaml').read_text())
|
|
cfg['paths'].update(work_root=str(self.project/'work'), node_bin=str(test_pi_session.NODE))
|
|
cfg['pools']['agent']['workers'] = agents
|
|
cfg['rollouts'].update(jail=False, private_home=True, tools=['write'], hard_ceiling_s=25)
|
|
cfg['endpoints']['teacher'].update(base_url=self.joint.provider.url, model='fixture', api_key_env='FIXTURE_KEY')
|
|
lifecycle = test_execution.policy(); lifecycle['execution_deadline_s'] = 40
|
|
cfg['collection_v1'].update(quota_ledger=str(self.joint.broker.path),
|
|
quota_groups={'account':asdict(self.joint.broker.groups['account'])},
|
|
credential_groups={'teacher':'account'}, lifecycle=lifecycle)
|
|
config_path = self.project/'config.yaml'; config_path.write_text(yaml.safe_dump(cfg))
|
|
stack = ExitStack(); self.addCleanup(stack.close)
|
|
# Subprocesses and the read-only exporter must see the same fixture DB.
|
|
stack.enter_context(patch.dict(os.environ, {'BENCH_STATE_DIR': str(self.state)}))
|
|
original_connect, original_readonly = db.connect, db.readonly
|
|
for name,value in {'BENCH_ROOT':self.project,'PROJECT_ROOT':self.project.parent,'AGENT':ROOT/'agent',
|
|
'RUNS':self.project/'runs','STATE':self.state,'CONFIG_PATH':config_path}.items():
|
|
stack.enter_context(patch.object(paths,name,value))
|
|
stack.enter_context(patch.object(paths,'config',return_value=cfg))
|
|
stack.enter_context(patch.object(paths,'secret',return_value='upstream-fixture'))
|
|
stack.enter_context(patch.object(db,'connect',lambda **kwargs:original_connect(self.state/'jobs.sqlite',**kwargs)))
|
|
stack.enter_context(patch.object(db,'readonly',lambda:original_readonly(self.state/'jobs.sqlite')))
|
|
stack.enter_context(patch.object(runtime,'_broker',self.joint.broker))
|
|
stack.enter_context(patch.object(runtime,'_configuration',json.dumps(cfg['collection_v1'],sort_keys=True)))
|
|
stack.enter_context(patch.object(rollout,'prepare',side_effect=self.prepare))
|
|
stack.enter_context(patch.object(rollout,'verify_rollout',return_value={
|
|
'terminal':'UNVERIFIED','claim_consistent':False,'review':False}))
|
|
from export import to_agentrl
|
|
stack.enter_context(patch.object(to_agentrl,'BENCH',self.project))
|
|
self.prepared = []
|
|
with db.connect() as con:
|
|
db.create_run(con,'r',{})
|
|
for n in range(agents): db.rollout_upsert(con,'r',f'j{n}',0,'e2e','teacher')
|
|
|
|
def prepare(self, run_id, job_id, kk, mode, policy, **_):
|
|
rd=self.project/'runs'/run_id/job_id/'rollouts'/f'{kk:02d}'
|
|
workspace=rd/'workspace'; workspace.mkdir(parents=True)
|
|
(rd/'agent').mkdir(); (rd/'handles').mkdir()
|
|
shutil.copytree(self.joint.template.home,workspace/'.agent-home')
|
|
shutil.copytree(self.joint.template.root/'.pi',workspace/'.pi')
|
|
model_path=workspace/'.agent-home/.pi/agent/models.json'
|
|
models=json.loads(model_path.read_text())
|
|
models['providers']['dr-gateway']=models['providers'].pop('fixture-provider')
|
|
model_path.write_text(json.dumps(models))
|
|
settings_path=workspace/'.pi/settings.json'
|
|
settings=json.loads(settings_path.read_text());settings['defaultProvider']='dr-gateway'
|
|
settings_path.write_text(json.dumps(settings))
|
|
for name in ('system.md','append-system.md','prompt.md'):
|
|
(rd/'agent'/name).write_text(f'{job_id}: write once, then report done.')
|
|
metadata={'state':'RUNNING','runner':'pi','runner_version':'pi-sdk-v1','policy':policy,'mode':mode,
|
|
'jail_policy':'private-home','max_turns':60,'max_tokens':200000,'model':'fixture'}
|
|
(rd/'rollout.json').write_text(json.dumps(metadata))
|
|
job={'run_id':run_id,'job_id':job_id,'task':'cls','march':'nash-p','backend':'ptq'}
|
|
(rd.parent.parent/'job.json').write_text(json.dumps(job))
|
|
self.prepared.append((run_id,job_id,kk))
|
|
return rd,job,(rd/'agent/prompt.md').read_text()
|
|
|
|
def rows(self):
|
|
with db.readonly() as con:
|
|
return [dict(r) for r in con.execute('SELECT * FROM rollouts ORDER BY job_id,kk')]
|
|
|
|
@staticmethod
|
|
def receipt(case, **facts):
|
|
print('JOINT_RECEIPT '+json.dumps({'case':case,**facts},sort_keys=True))
|
|
|
|
def test_collector_multiple_sdk_429_does_not_allocate_empty_replacements(self):
|
|
self.setup_collection(lambda *_:(429,{'Retry-After':'.08'},b'{"error":{"type":"rate_limit"}}'),agents=2)
|
|
before=[(r['run_id'],r['job_id'],r['kk']) for r in self.rows()]
|
|
first=pipeline.collect_rollouts('r',parallel=2,limit=2,log=lambda _:None)
|
|
self.assertEqual(first,{'PAUSED_INFRA':2})
|
|
rows=self.rows()
|
|
self.assertEqual([(r['run_id'],r['job_id'],r['kk']) for r in rows],before)
|
|
self.assertTrue(all(r['state']=='PAUSED_INFRA' for r in rows))
|
|
frozen={}; agent_identities=set()
|
|
for _,job_id,kk in before:
|
|
rd=self.project/'runs/r'/job_id/'rollouts'/f'{kk:02d}'
|
|
state=execution.Execution(rd).snapshot()
|
|
self.assertEqual(state['execution_state'],'PAUSED_INFRA')
|
|
self.assertEqual(execution.owned_processes(state),[])
|
|
self.assertFalse((rd/'workspace/side-effect.txt').exists())
|
|
for process in state['processes']:
|
|
if process.get('role')=='agent':
|
|
self.assertEqual(execution.process_status(process),'DEAD')
|
|
agent_identities.add(execution.process_key(process))
|
|
frozen[job_id]=(state['execution_id'],state['deadline_monotonic'])
|
|
self.assertEqual(len(agent_identities),2)
|
|
requests=len(self.joint.provider.requests)
|
|
self.assertEqual(requests,6)
|
|
self.assertEqual(pipeline.collect_rollouts('r',parallel=2,limit=2,log=lambda _:None),{})
|
|
self.assertEqual([(r['run_id'],r['job_id'],r['kk']) for r in self.rows()],before)
|
|
self.assertEqual(len(self.prepared),2)
|
|
self.assertEqual(len(self.joint.provider.requests),requests)
|
|
for _,job_id,kk in before:
|
|
rd=self.project/'runs/r'/job_id/'rollouts'/f'{kk:02d}'
|
|
state=execution.Execution(rd).snapshot()
|
|
self.assertEqual((state['execution_id'],state['deadline_monotonic']),frozen[job_id])
|
|
self.assertEqual(len(list(rd.parent.iterdir())),1)
|
|
self.assertTrue(all(g['attempts']==3 for g in self.joint.broker.snapshot()['generations']))
|
|
self.receipt('JOINT-001-collector',sdk_processes=2,original_attempts=2,final_attempts=2,
|
|
upstream_requests=requests,restarted_collector_extra_requests=0,tool_executions=0)
|
|
|
|
def test_sent_request_broker_sigkill_restart_never_replays_unknown(self):
|
|
fixture=test_quota.Quota();fixture.setUp();self.addCleanup(fixture.doCleanups)
|
|
entered,release=threading.Event(),threading.Event()
|
|
def pending(n,body):
|
|
entered.set();release.wait(5)
|
|
return test_quota.ok(n,body)
|
|
provider=fixture.provider(pending)
|
|
policy=fixture.policy(request_total_s=.4,first_byte_timeout_s=1,wait_budget_s=.03,window_s=60)
|
|
program=r'''
|
|
import json,sys
|
|
from pathlib import Path
|
|
from harness.quota import Broker,QuotaPolicy,Upstream
|
|
from scripts.test_quota import Quota
|
|
ledger,url,raw=sys.argv[1:]
|
|
with Broker(Path(ledger),{'shared':QuotaPolicy(**json.loads(raw))},
|
|
{'teacher':Upstream('shared',url,'upstream-secret',('glm-test',))}) as b:
|
|
scope=b.issue_scope('crashed-execution','teacher','glm-test')
|
|
Quota().request(scope,'crashed-generation')
|
|
'''
|
|
child=subprocess.Popen([sys.executable,'-c',program,str(fixture.ledger),provider.url,json.dumps(asdict(policy))],
|
|
cwd=ROOT,env={**os.environ,'PYTHONPATH':str(ROOT),'PYTHONDONTWRITEBYTECODE':'1'},
|
|
stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
|
def cleanup():
|
|
release.set()
|
|
if child.poll() is None:child.kill()
|
|
child.communicate(timeout=3)
|
|
self.addCleanup(cleanup)
|
|
self.assertTrue(entered.wait(5))
|
|
deadline=time.monotonic()+1
|
|
while True:
|
|
with closing(sqlite3.connect(f'file:{fixture.ledger}?mode=ro',uri=True)) as con:
|
|
before=con.execute('SELECT state,sent,ended_at,charged_tokens FROM attempts').fetchone()
|
|
if before and before[0]=='SENT':break
|
|
self.assertLess(time.monotonic(),deadline);time.sleep(.005)
|
|
self.assertEqual(before[1:3],(1,None))
|
|
self.assertGreater(before[3],0)
|
|
child.kill();child.communicate(timeout=3);self.assertEqual(child.returncode,-9)
|
|
release.set();time.sleep(.45)
|
|
broker=fixture.broker(provider,policy)
|
|
replay=broker.issue_scope('crashed-execution','teacher','glm-test')
|
|
self.assertNotEqual(fixture.request(replay,'crashed-generation')[0],200)
|
|
other=broker.issue_scope('new-execution','teacher','glm-test')
|
|
self.assertNotEqual(fixture.request(other,'next-generation')[0],200)
|
|
snapshot=quota.quota_snapshot(fixture.ledger)
|
|
self.assertEqual(len(provider.requests),1)
|
|
self.assertEqual(len(snapshot['attempts']),1)
|
|
attempt=snapshot['attempts'][0]
|
|
self.assertEqual(attempt['state'],'UNKNOWN');self.assertIsNone(attempt['ended_at'])
|
|
self.assertEqual(attempt['charged_tokens'],before[3])
|
|
self.assertEqual(snapshot['groups'][0]['rpm_used'],1)
|
|
self.assertEqual(snapshot['groups'][0]['inflight'],1)
|
|
self.assertGreater(snapshot['groups'][0]['tpm_reserved'],0)
|
|
self.assertEqual(broker.execution_status('crashed-execution')['generations'][0]['attempts'],1)
|
|
self.receipt('JOINT-003/009-broker-crash',process_exit=-9,upstream_requests=1,
|
|
restart_extra_requests=0,attempts=1,state='UNKNOWN',reserved_tokens=before[3])
|
|
|
|
def test_busy_total_timeout_converges_sdk_execution_db_and_monitor(self):
|
|
def busy(_n,_body):
|
|
def stream(output):
|
|
output.write(joint_fixture.write_response()[2].split(b'\n\n')[0]+b'\n\n');output.flush()
|
|
for _ in range(80):
|
|
time.sleep(.015)
|
|
output.write(b'data: {"choices":[{"index":0,"delta":{"content":"tick"},"finish_reason":null}]}\n\n')
|
|
output.flush()
|
|
return 200,{'Content-Type':'text/event-stream'},stream
|
|
self.setup_collection(busy,stream_idle_timeout_s=.2,first_byte_timeout_s=.3,request_total_s=.35)
|
|
started=time.monotonic()
|
|
result=pipeline.collect_rollouts('r',limit=1,log=lambda _:None)
|
|
self.assertLess(time.monotonic()-started,12)
|
|
self.assertEqual(result,{'PAUSED_INFRA':1})
|
|
self.assertEqual([r['state'] for r in self.rows()],['PAUSED_INFRA'])
|
|
rd=self.project/'runs/r/j0/rollouts/00'
|
|
state=execution.Execution(rd).snapshot()
|
|
self.assertEqual(state['execution_state'],'PAUSED_INFRA')
|
|
self.assertEqual(execution.owned_processes(state),[])
|
|
self.assertFalse((rd/'workspace/side-effect.txt').exists())
|
|
attempts=self.joint.broker.snapshot()['attempts']
|
|
self.assertEqual(len(attempts),1)
|
|
self.assertEqual(attempts[0]['transport_reason'],'REQUEST_TOTAL_TIMEOUT')
|
|
self.assertEqual(len(self.joint.provider.requests),1)
|
|
observer=monitor.Observer(self.project,quota_source=lambda:quota.quota_snapshot(self.joint.broker.path))
|
|
observer.sample();snapshot=observer.snapshot('r');detail=observer.detail('r','j0',0)
|
|
self.assertEqual(snapshot['source_status'],'FRESH')
|
|
# The dashboard intentionally counts paused attempts as DB-active.
|
|
# Assert their actual state instead of treating db_active as RUNNING.
|
|
self.assertEqual(snapshot['counts']['agent_processes'],0)
|
|
self.assertEqual(snapshot['db_states'],{'PAUSED_INFRA':1})
|
|
self.assertEqual(snapshot['db_states'].get('RUNNING',0),0)
|
|
self.assertEqual((detail['state'],detail['execution_state']),('PAUSED_INFRA','PAUSED_INFRA'))
|
|
self.assertEqual(snapshot['quota']['attempts'][0]['transport_reason'],'REQUEST_TOTAL_TIMEOUT')
|
|
self.receipt('JOINT-004-total',upstream_requests=1,tool_executions=0,transport_reason='REQUEST_TOTAL_TIMEOUT',
|
|
db_state=detail['state'],execution_state=detail['execution_state'],monitor_agents=0,monitor_db_running=0,monitor_db_active=snapshot['counts']['db_active'])
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|