data-gen-bpu/scripts/test_acceptance_edges.py
ming01.wang f611ef29ab fix: keep managed API requests within the broker lifecycle
Pi requests could hit Node fetch's independent five-minute header deadline,
and queued requests could outlive their local client or terminal rollout.
Set header/body deadlines only for the loopback broker origin, cancel scopes
when clients disconnect or executions terminate, and retain uncertain upstream
reservations instead of retrying after cancellation. Resumable pauses remain open.

Validation: real Pi SDK accepted a response delayed 310 seconds (314.139 s test);
quota, Pi session, acceptance, cancellation, and lifecycle regression coverage.
2026-09-22 20:28:42 +08:00

240 lines
14 KiB
Python

"""Three acceptance edges with private fixtures, real processes and no board calls."""
from contextlib import ExitStack
from dataclasses import asdict
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
import unittest
from unittest.mock import patch
import yaml
from export import test_publication, test_trajectory
from export.publication import PublicationError, Releases
from scripts import test_pi_session
ROOT = Path(__file__).resolve().parents[1]
RACER = r'''
import json,sys,time
from pathlib import Path
from export.publication import Releases,PublicationError
root,target,role,hold=sys.argv[1:]
root=Path(root)
def barrier():
(root/(role+'-holding')).touch()
deadline=time.monotonic()+5
while not (root/'release').exists():
if time.monotonic()>deadline:raise TimeoutError('fixture release missing')
time.sleep(.005)
store=Releases(root,fault=lambda phase:barrier() if hold=='1' and phase=='after_intent' else None)
if role=='revoke' and hold=='1':
verify=store._verify
def held_verify(version):
value=verify(version);barrier();return value
store._verify=held_verify
(root/(role+'-ready')).touch()
try:
if role=='revoke':store.revoke(target,'controlled-race');result='REVOKED'
else:result=store.switch(target,expected_revision=2,event_id='race',reason='controlled-race',rollback=True)['state']
except PublicationError as error:result=error.reason
print(json.dumps({'role':role,'result':result}))
'''
class AcceptanceEdges(unittest.TestCase):
@staticmethod
def receipt(case, **facts):
print('JOINT_RECEIPT ' + json.dumps({'case': case, **facts}, sort_keys=True))
def fixture(self, cls):
value = cls(); value.setUp()
self.addCleanup(value.tearDown)
return value
def test_entire_empty_error_batch_has_complete_disposition_and_cannot_publish(self):
fixture = self.fixture(test_publication.BuildTests)
before = {}
for index in range(3):
rd = fixture.fixture.add(index, terminal='FAIL')
empty = {'role': 'assistant', 'content': [], 'stopReason': 'error', 'errorMessage': 'fixture 429'}
events = [fixture.fixture.context_header()] + test_trajectory.segment([empty])
source = rd / 'agent/events.jsonl'
source.write_text('\n'.join(map(json.dumps, events)) + '\n')
before[source] = source.read_bytes()
manifest = fixture.materialize()
folder = fixture.root / 'datasets/candidate'
def rows(name):
return [json.loads(line) for line in (folder / (name + '.jsonl')).read_text().splitlines()]
for name in ('disposition', 'source-manifest', 'retained', 'validation', 'exclusions'):
values = rows(name)
self.assertEqual(len(values), 3, name)
self.assertEqual({(r['run_id'], r['job_id'], r['rollout']) for r in values},
{('r', 'j', i) for i in range(3)}, name)
self.assertTrue(all(r['disposition'] == 'no-usable' for r in rows('disposition')))
self.assertEqual(len(rows('excluded_observations')), 3)
self.assertEqual(rows('sft'), [])
self.assertEqual(rows('teacher'), [])
self.assertEqual(manifest['stats']['scanned'], 3)
self.assertEqual(manifest['stats']['excluded'], 3)
for row in rows('validation'):
checks = row['checks']
self.assertTrue(checks['OBSERVATION_ACCOUNTING']['pass'])
self.assertTrue(checks['ERROR_EXCLUSION']['pass'])
self.assertFalse(checks['USABLE_ASSISTANT']['pass'])
with self.assertRaisesRegex(PublicationError, 'NO_ACCEPTED_SAMPLES'):
fixture.build()
self.assertEqual({p: p.read_bytes() for p in before}, before)
self.assertFalse((fixture.root / 'releases/CURRENT.json').exists())
self.receipt('DATA-012', input_attempts=3, unique_dispositions=3,
excluded_empty_errors=3, accepted=0, publication='NO_ACCEPTED_SAMPLES', source_unchanged=True)
@unittest.skipUnless(sys.platform == 'linux', 'Linux cross-process publication locking')
def test_revoke_and_rollback_processes_serialize_in_both_lock_orders(self):
outcomes = []
for first in ('revoke', 'rollback'):
with self.subTest(first=first):
fixture = self.fixture(test_publication.PublicationTests)
old, current = fixture.seal(), fixture.seal('rules-b')
fixture.switch(old); fixture.switch(current, 1, 'second')
children = []
def launch(role, hold):
proc = subprocess.Popen([sys.executable, '-c', RACER, str(fixture.root), old, role, str(int(hold))],
cwd=ROOT, env={**os.environ, 'PYTHONPATH': str(ROOT), 'PYTHONDONTWRITEBYTECODE': '1'},
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
children.append(proc)
return proc
def wait(path):
deadline = time.monotonic() + 5
while not path.exists() and time.monotonic() < deadline:
time.sleep(.005)
self.assertTrue(path.exists(), path.name)
try:
launch(first, True)
wait(fixture.root / (first + '-holding'))
second = 'rollback' if first == 'revoke' else 'revoke'
following = launch(second, False)
wait(fixture.root / (second + '-ready'))
self.assertIsNone(following.poll()) # The first process still owns the real file lock.
(fixture.root / 'release').touch()
result = {}
for proc in children:
out, err = proc.communicate(timeout=6)
self.assertEqual(proc.returncode, 0, err)
row = json.loads(out); result[row['role']] = row['result']
self.assertEqual(result['revoke'], 'REVOKED')
self.assertEqual(result['rollback'], 'VERSION_REVOKED' if first == 'revoke' else 'COMMIT')
if first == 'rollback':
with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'):
fixture.store.current()
else:
self.assertEqual(fixture.store.current()['pointer']['dataset_version'], current)
with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'):
fixture.store.switch(old, expected_revision=2, event_id='after-revoke',
reason='must refuse', rollback=True)
outcomes.append({'first_lock_owner': first, **result})
finally:
(fixture.root / 'release').touch()
for proc in children:
if proc.poll() is None:
proc.kill()
proc.communicate(timeout=3)
self.receipt('DATA-013/VERSION-001', real_processes=4, orders=outcomes, revoked_target_served=False)
@unittest.skipUnless(sys.platform == 'linux' and test_pi_session.NODE.is_file(), 'Linux and installed Pi SDK')
def test_provider_only_error_wrapped_exit_zero_is_not_completed_or_sft(self):
from harness import collection_runtime as runtime, db, paths, rollout
from harness.runners import pi
from harness.verify import verify_rollout
fixture = self.fixture(test_publication.BuildTests)
sdk = test_pi_session.PiSDKTests(); sdk.setUp(); self.addCleanup(sdk.doCleanups)
sdk.fail_second, sdk.fail_on = True, {1, 2, 3, 4}
rd = fixture.fixture.add(terminal='FAIL', execution='RUNNING')
fixture.fixture.con.execute("UPDATE rollouts SET mode='e2e', policy='teacher', state='RUNNING'")
fixture.fixture.con.commit()
job = json.loads((rd.parent.parent / 'job.json').read_text())
measured_before = verify_rollout(rd, write=False)['gates']
gate_bytes = {p: p.read_bytes() for p in (rd / 'stages').rglob('meta.json')}
workspace = rd / 'workspace'; workspace.mkdir()
shutil.move(str(sdk.root / '.pi'), workspace / '.pi')
shutil.move(str(sdk.home), workspace / '.agent-home')
models_path = workspace / '.agent-home/.pi/agent/models.json'
models = json.loads(models_path.read_text())
models['providers']['dr-gateway'] = models['providers'].pop('fixture-provider')
models_path.write_text(json.dumps(models))
settings_path = workspace / '.pi/settings.json'
settings = json.loads(settings_path.read_text()); settings['defaultProvider'] = 'dr-gateway'
settings_path.write_text(json.dumps(settings))
for name in ('system.md', 'append-system.md', 'prompt.md'):
(rd / 'agent' / name).write_text('Attempt the task once.')
(rd / 'agent/events.jsonl').write_bytes(b'')
metadata = json.loads((rd / 'rollout.json').read_text())
metadata.pop('execution_id') # The actual controller allocates this execution.
metadata.update(runner='pi', jail_policy='private-home', max_turns=60, max_tokens=200000)
(rd / 'rollout.json').write_text(json.dumps(metadata))
config = yaml.safe_load((ROOT / 'config.yaml').read_text())
config['paths'].update(work_root=str(fixture.root/'work'), node_bin=str(test_pi_session.NODE))
config['rollouts'].update(jail=False, private_home=True, tools=['write'], hard_ceiling_s=15)
config['endpoints']['teacher'].update(base_url='http://127.0.0.1:%d/v1' % sdk.server.server_port,
model='fixture', api_key_env='FIXTURE_KEY')
config['collection_v1'].update(quota_ledger=str(fixture.root/'runtime-quota.sqlite'),
quota_groups={'account': asdict(sdk.broker.groups['account'])}, credential_groups={'teacher': 'account'})
config_path = fixture.root/'config.yaml'; config_path.write_text(yaml.safe_dump(config))
original_connect, original_readonly, command = db.connect, db.readonly, pi.command
def wrapped_command(*args):
script = ('import subprocess,sys;from pathlib import Path;'
'result=subprocess.run(sys.argv[2:]);Path(sys.argv[1]).write_text(str(result.returncode));sys.exit(0)')
return [sys.executable, '-c', script, str(workspace/'sdk-exit'), *command(*args)]
with ExitStack() as stack:
for name, value in {'BENCH_ROOT': fixture.root, 'PROJECT_ROOT': fixture.root.parent, 'AGENT': ROOT/'agent',
'RUNS': fixture.root/'runs', 'STATE': fixture.root/'state', 'CONFIG_PATH': config_path}.items():
stack.enter_context(patch.object(paths, name, value))
stack.enter_context(patch.object(paths, 'config', return_value=config))
stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture'))
stack.enter_context(patch.object(db, 'connect', lambda: original_connect(fixture.root/'state/jobs.sqlite')))
stack.enter_context(patch.object(db, 'readonly', lambda: original_readonly(fixture.root/'state/jobs.sqlite')))
stack.enter_context(patch.object(runtime, '_broker', None))
stack.enter_context(patch.object(runtime, '_configuration', None))
stack.enter_context(patch.object(rollout, 'prepare', return_value=(rd, job, 'Attempt the task once.')))
stack.enter_context(patch.object(pi, 'command', side_effect=wrapped_command))
try:
rollout._execute_rollout('r', 'j', 0, 'e2e', 'teacher', False)
state = json.loads((rd/'execution.json').read_text())
self.assertEqual((workspace/'sdk-exit').read_text(), '76')
self.assertEqual(state['exit_code'], 0)
self.assertEqual(state['execution_state'], 'TERMINAL')
self.assertNotEqual(state['execution_outcome'], 'COMPLETED')
self.assertFalse(state['protocol_complete'])
self.assertEqual(state['cleanup_state'], 'VERIFIED')
self.assertEqual(json.loads((rd/'rollout.json').read_text())['state'], 'ERROR')
self.assertEqual(verify_rollout(rd, write=False)['gates'], measured_before)
self.assertEqual({p: p.read_bytes() for p in gate_bytes}, gate_bytes)
self.assertFalse((workspace/'side-effect.txt').exists())
errors = [json.loads(line) for line in (rd/'agent/events.jsonl').read_text().splitlines()
if json.loads(line).get('type') == 'message_end']
assistants = [r['message'] for r in errors if r['message']['role'] == 'assistant']
self.assertTrue(assistants)
self.assertTrue(all(m.get('stopReason') == 'error' and not m['content'] for m in assistants))
quota = runtime.broker().execution_status(state['execution_id'])
# This execution is terminal, unlike the resumable pause path.
# Its request must never become eligible after a slot is freed.
self.assertTrue(quota['cancelled'])
self.assertEqual(quota['generations'][-1]['state'], 'CANCELLED')
self.assertTrue(quota['generations'][-1]['reason'])
self.assertEqual(fixture.materialize()['stats']['sft_eligible'], 0)
self.assertEqual((fixture.root/'datasets/candidate/sft.jsonl').read_bytes(), b'')
self.receipt('JOINT-008', sdk_exit=76, wrapper_exit=0, outcome=state['execution_outcome'],
protocol_complete=False, cleanup='VERIFIED', gates_unchanged=True, accepted_sft=0,
upstream_requests=len(sdk.bodies), quota_reason=quota['generations'][-1]['reason'])
finally:
if runtime._broker:
runtime._broker.close()
if __name__ == '__main__':
unittest.main()