Pi requests could hit Node fetch's independent five-minute header deadline, and queued requests could outlive their local client or terminal rollout. Set header/body deadlines only for the loopback broker origin, cancel scopes when clients disconnect or executions terminate, and retain uncertain upstream reservations instead of retrying after cancellation. Resumable pauses remain open. Validation: real Pi SDK accepted a response delayed 310 seconds (314.139 s test); quota, Pi session, acceptance, cancellation, and lifecycle regression coverage.
240 lines
14 KiB
Python
240 lines
14 KiB
Python
"""Three acceptance edges with private fixtures, real processes and no board calls."""
|
|
from contextlib import ExitStack
|
|
from dataclasses import asdict
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import yaml
|
|
|
|
from export import test_publication, test_trajectory
|
|
from export.publication import PublicationError, Releases
|
|
from scripts import test_pi_session
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
RACER = r'''
|
|
import json,sys,time
|
|
from pathlib import Path
|
|
from export.publication import Releases,PublicationError
|
|
root,target,role,hold=sys.argv[1:]
|
|
root=Path(root)
|
|
def barrier():
|
|
(root/(role+'-holding')).touch()
|
|
deadline=time.monotonic()+5
|
|
while not (root/'release').exists():
|
|
if time.monotonic()>deadline:raise TimeoutError('fixture release missing')
|
|
time.sleep(.005)
|
|
store=Releases(root,fault=lambda phase:barrier() if hold=='1' and phase=='after_intent' else None)
|
|
if role=='revoke' and hold=='1':
|
|
verify=store._verify
|
|
def held_verify(version):
|
|
value=verify(version);barrier();return value
|
|
store._verify=held_verify
|
|
(root/(role+'-ready')).touch()
|
|
try:
|
|
if role=='revoke':store.revoke(target,'controlled-race');result='REVOKED'
|
|
else:result=store.switch(target,expected_revision=2,event_id='race',reason='controlled-race',rollback=True)['state']
|
|
except PublicationError as error:result=error.reason
|
|
print(json.dumps({'role':role,'result':result}))
|
|
'''
|
|
|
|
|
|
class AcceptanceEdges(unittest.TestCase):
|
|
@staticmethod
|
|
def receipt(case, **facts):
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': case, **facts}, sort_keys=True))
|
|
|
|
def fixture(self, cls):
|
|
value = cls(); value.setUp()
|
|
self.addCleanup(value.tearDown)
|
|
return value
|
|
|
|
def test_entire_empty_error_batch_has_complete_disposition_and_cannot_publish(self):
|
|
fixture = self.fixture(test_publication.BuildTests)
|
|
before = {}
|
|
for index in range(3):
|
|
rd = fixture.fixture.add(index, terminal='FAIL')
|
|
empty = {'role': 'assistant', 'content': [], 'stopReason': 'error', 'errorMessage': 'fixture 429'}
|
|
events = [fixture.fixture.context_header()] + test_trajectory.segment([empty])
|
|
source = rd / 'agent/events.jsonl'
|
|
source.write_text('\n'.join(map(json.dumps, events)) + '\n')
|
|
before[source] = source.read_bytes()
|
|
manifest = fixture.materialize()
|
|
folder = fixture.root / 'datasets/candidate'
|
|
def rows(name):
|
|
return [json.loads(line) for line in (folder / (name + '.jsonl')).read_text().splitlines()]
|
|
for name in ('disposition', 'source-manifest', 'retained', 'validation', 'exclusions'):
|
|
values = rows(name)
|
|
self.assertEqual(len(values), 3, name)
|
|
self.assertEqual({(r['run_id'], r['job_id'], r['rollout']) for r in values},
|
|
{('r', 'j', i) for i in range(3)}, name)
|
|
self.assertTrue(all(r['disposition'] == 'no-usable' for r in rows('disposition')))
|
|
self.assertEqual(len(rows('excluded_observations')), 3)
|
|
self.assertEqual(rows('sft'), [])
|
|
self.assertEqual(rows('teacher'), [])
|
|
self.assertEqual(manifest['stats']['scanned'], 3)
|
|
self.assertEqual(manifest['stats']['excluded'], 3)
|
|
for row in rows('validation'):
|
|
checks = row['checks']
|
|
self.assertTrue(checks['OBSERVATION_ACCOUNTING']['pass'])
|
|
self.assertTrue(checks['ERROR_EXCLUSION']['pass'])
|
|
self.assertFalse(checks['USABLE_ASSISTANT']['pass'])
|
|
with self.assertRaisesRegex(PublicationError, 'NO_ACCEPTED_SAMPLES'):
|
|
fixture.build()
|
|
self.assertEqual({p: p.read_bytes() for p in before}, before)
|
|
self.assertFalse((fixture.root / 'releases/CURRENT.json').exists())
|
|
self.receipt('DATA-012', input_attempts=3, unique_dispositions=3,
|
|
excluded_empty_errors=3, accepted=0, publication='NO_ACCEPTED_SAMPLES', source_unchanged=True)
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux cross-process publication locking')
|
|
def test_revoke_and_rollback_processes_serialize_in_both_lock_orders(self):
|
|
outcomes = []
|
|
for first in ('revoke', 'rollback'):
|
|
with self.subTest(first=first):
|
|
fixture = self.fixture(test_publication.PublicationTests)
|
|
old, current = fixture.seal(), fixture.seal('rules-b')
|
|
fixture.switch(old); fixture.switch(current, 1, 'second')
|
|
children = []
|
|
def launch(role, hold):
|
|
proc = subprocess.Popen([sys.executable, '-c', RACER, str(fixture.root), old, role, str(int(hold))],
|
|
cwd=ROOT, env={**os.environ, 'PYTHONPATH': str(ROOT), 'PYTHONDONTWRITEBYTECODE': '1'},
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
children.append(proc)
|
|
return proc
|
|
def wait(path):
|
|
deadline = time.monotonic() + 5
|
|
while not path.exists() and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
self.assertTrue(path.exists(), path.name)
|
|
try:
|
|
launch(first, True)
|
|
wait(fixture.root / (first + '-holding'))
|
|
second = 'rollback' if first == 'revoke' else 'revoke'
|
|
following = launch(second, False)
|
|
wait(fixture.root / (second + '-ready'))
|
|
self.assertIsNone(following.poll()) # The first process still owns the real file lock.
|
|
(fixture.root / 'release').touch()
|
|
result = {}
|
|
for proc in children:
|
|
out, err = proc.communicate(timeout=6)
|
|
self.assertEqual(proc.returncode, 0, err)
|
|
row = json.loads(out); result[row['role']] = row['result']
|
|
self.assertEqual(result['revoke'], 'REVOKED')
|
|
self.assertEqual(result['rollback'], 'VERSION_REVOKED' if first == 'revoke' else 'COMMIT')
|
|
if first == 'rollback':
|
|
with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'):
|
|
fixture.store.current()
|
|
else:
|
|
self.assertEqual(fixture.store.current()['pointer']['dataset_version'], current)
|
|
with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'):
|
|
fixture.store.switch(old, expected_revision=2, event_id='after-revoke',
|
|
reason='must refuse', rollback=True)
|
|
outcomes.append({'first_lock_owner': first, **result})
|
|
finally:
|
|
(fixture.root / 'release').touch()
|
|
for proc in children:
|
|
if proc.poll() is None:
|
|
proc.kill()
|
|
proc.communicate(timeout=3)
|
|
self.receipt('DATA-013/VERSION-001', real_processes=4, orders=outcomes, revoked_target_served=False)
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux' and test_pi_session.NODE.is_file(), 'Linux and installed Pi SDK')
|
|
def test_provider_only_error_wrapped_exit_zero_is_not_completed_or_sft(self):
|
|
from harness import collection_runtime as runtime, db, paths, rollout
|
|
from harness.runners import pi
|
|
from harness.verify import verify_rollout
|
|
fixture = self.fixture(test_publication.BuildTests)
|
|
sdk = test_pi_session.PiSDKTests(); sdk.setUp(); self.addCleanup(sdk.doCleanups)
|
|
sdk.fail_second, sdk.fail_on = True, {1, 2, 3, 4}
|
|
rd = fixture.fixture.add(terminal='FAIL', execution='RUNNING')
|
|
fixture.fixture.con.execute("UPDATE rollouts SET mode='e2e', policy='teacher', state='RUNNING'")
|
|
fixture.fixture.con.commit()
|
|
job = json.loads((rd.parent.parent / 'job.json').read_text())
|
|
measured_before = verify_rollout(rd, write=False)['gates']
|
|
gate_bytes = {p: p.read_bytes() for p in (rd / 'stages').rglob('meta.json')}
|
|
workspace = rd / 'workspace'; workspace.mkdir()
|
|
shutil.move(str(sdk.root / '.pi'), workspace / '.pi')
|
|
shutil.move(str(sdk.home), workspace / '.agent-home')
|
|
models_path = workspace / '.agent-home/.pi/agent/models.json'
|
|
models = json.loads(models_path.read_text())
|
|
models['providers']['dr-gateway'] = models['providers'].pop('fixture-provider')
|
|
models_path.write_text(json.dumps(models))
|
|
settings_path = workspace / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_text()); settings['defaultProvider'] = 'dr-gateway'
|
|
settings_path.write_text(json.dumps(settings))
|
|
for name in ('system.md', 'append-system.md', 'prompt.md'):
|
|
(rd / 'agent' / name).write_text('Attempt the task once.')
|
|
(rd / 'agent/events.jsonl').write_bytes(b'')
|
|
metadata = json.loads((rd / 'rollout.json').read_text())
|
|
metadata.pop('execution_id') # The actual controller allocates this execution.
|
|
metadata.update(runner='pi', jail_policy='private-home', max_turns=60, max_tokens=200000)
|
|
(rd / 'rollout.json').write_text(json.dumps(metadata))
|
|
config = yaml.safe_load((ROOT / 'config.yaml').read_text())
|
|
config['paths'].update(work_root=str(fixture.root/'work'), node_bin=str(test_pi_session.NODE))
|
|
config['rollouts'].update(jail=False, private_home=True, tools=['write'], hard_ceiling_s=15)
|
|
config['endpoints']['teacher'].update(base_url='http://127.0.0.1:%d/v1' % sdk.server.server_port,
|
|
model='fixture', api_key_env='FIXTURE_KEY')
|
|
config['collection_v1'].update(quota_ledger=str(fixture.root/'runtime-quota.sqlite'),
|
|
quota_groups={'account': asdict(sdk.broker.groups['account'])}, credential_groups={'teacher': 'account'})
|
|
config_path = fixture.root/'config.yaml'; config_path.write_text(yaml.safe_dump(config))
|
|
original_connect, original_readonly, command = db.connect, db.readonly, pi.command
|
|
def wrapped_command(*args):
|
|
script = ('import subprocess,sys;from pathlib import Path;'
|
|
'result=subprocess.run(sys.argv[2:]);Path(sys.argv[1]).write_text(str(result.returncode));sys.exit(0)')
|
|
return [sys.executable, '-c', script, str(workspace/'sdk-exit'), *command(*args)]
|
|
with ExitStack() as stack:
|
|
for name, value in {'BENCH_ROOT': fixture.root, 'PROJECT_ROOT': fixture.root.parent, 'AGENT': ROOT/'agent',
|
|
'RUNS': fixture.root/'runs', 'STATE': fixture.root/'state', 'CONFIG_PATH': config_path}.items():
|
|
stack.enter_context(patch.object(paths, name, value))
|
|
stack.enter_context(patch.object(paths, 'config', return_value=config))
|
|
stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture'))
|
|
stack.enter_context(patch.object(db, 'connect', lambda: original_connect(fixture.root/'state/jobs.sqlite')))
|
|
stack.enter_context(patch.object(db, 'readonly', lambda: original_readonly(fixture.root/'state/jobs.sqlite')))
|
|
stack.enter_context(patch.object(runtime, '_broker', None))
|
|
stack.enter_context(patch.object(runtime, '_configuration', None))
|
|
stack.enter_context(patch.object(rollout, 'prepare', return_value=(rd, job, 'Attempt the task once.')))
|
|
stack.enter_context(patch.object(pi, 'command', side_effect=wrapped_command))
|
|
try:
|
|
rollout._execute_rollout('r', 'j', 0, 'e2e', 'teacher', False)
|
|
state = json.loads((rd/'execution.json').read_text())
|
|
self.assertEqual((workspace/'sdk-exit').read_text(), '76')
|
|
self.assertEqual(state['exit_code'], 0)
|
|
self.assertEqual(state['execution_state'], 'TERMINAL')
|
|
self.assertNotEqual(state['execution_outcome'], 'COMPLETED')
|
|
self.assertFalse(state['protocol_complete'])
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(json.loads((rd/'rollout.json').read_text())['state'], 'ERROR')
|
|
self.assertEqual(verify_rollout(rd, write=False)['gates'], measured_before)
|
|
self.assertEqual({p: p.read_bytes() for p in gate_bytes}, gate_bytes)
|
|
self.assertFalse((workspace/'side-effect.txt').exists())
|
|
errors = [json.loads(line) for line in (rd/'agent/events.jsonl').read_text().splitlines()
|
|
if json.loads(line).get('type') == 'message_end']
|
|
assistants = [r['message'] for r in errors if r['message']['role'] == 'assistant']
|
|
self.assertTrue(assistants)
|
|
self.assertTrue(all(m.get('stopReason') == 'error' and not m['content'] for m in assistants))
|
|
quota = runtime.broker().execution_status(state['execution_id'])
|
|
# This execution is terminal, unlike the resumable pause path.
|
|
# Its request must never become eligible after a slot is freed.
|
|
self.assertTrue(quota['cancelled'])
|
|
self.assertEqual(quota['generations'][-1]['state'], 'CANCELLED')
|
|
self.assertTrue(quota['generations'][-1]['reason'])
|
|
self.assertEqual(fixture.materialize()['stats']['sft_eligible'], 0)
|
|
self.assertEqual((fixture.root/'datasets/candidate/sft.jsonl').read_bytes(), b'')
|
|
self.receipt('JOINT-008', sdk_exit=76, wrapper_exit=0, outcome=state['execution_outcome'],
|
|
protocol_complete=False, cleanup='VERIFIED', gates_unchanged=True, accepted_sft=0,
|
|
upstream_requests=len(sdk.bodies), quota_reason=quota['generations'][-1]['reason'])
|
|
finally:
|
|
if runtime._broker:
|
|
runtime._broker.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|