"""Three acceptance edges with private fixtures, real processes and no board calls.""" from contextlib import ExitStack from dataclasses import asdict import json import os from pathlib import Path import shutil import subprocess import sys import time import unittest from unittest.mock import patch import yaml from export import test_publication, test_trajectory from export.publication import PublicationError, Releases from scripts import test_pi_session ROOT = Path(__file__).resolve().parents[1] RACER = r''' import json,sys,time from pathlib import Path from export.publication import Releases,PublicationError root,target,role,hold=sys.argv[1:] root=Path(root) def barrier(): (root/(role+'-holding')).touch() deadline=time.monotonic()+5 while not (root/'release').exists(): if time.monotonic()>deadline:raise TimeoutError('fixture release missing') time.sleep(.005) store=Releases(root,fault=lambda phase:barrier() if hold=='1' and phase=='after_intent' else None) if role=='revoke' and hold=='1': verify=store._verify def held_verify(version): value=verify(version);barrier();return value store._verify=held_verify (root/(role+'-ready')).touch() try: if role=='revoke':store.revoke(target,'controlled-race');result='REVOKED' else:result=store.switch(target,expected_revision=2,event_id='race',reason='controlled-race',rollback=True)['state'] except PublicationError as error:result=error.reason print(json.dumps({'role':role,'result':result})) ''' class AcceptanceEdges(unittest.TestCase): @staticmethod def receipt(case, **facts): print('JOINT_RECEIPT ' + json.dumps({'case': case, **facts}, sort_keys=True)) def fixture(self, cls): value = cls(); value.setUp() self.addCleanup(value.tearDown) return value def test_entire_empty_error_batch_has_complete_disposition_and_cannot_publish(self): fixture = self.fixture(test_publication.BuildTests) before = {} for index in range(3): rd = fixture.fixture.add(index, terminal='FAIL') empty = {'role': 'assistant', 'content': [], 'stopReason': 'error', 'errorMessage': 'fixture 429'} events = [fixture.fixture.context_header()] + test_trajectory.segment([empty]) source = rd / 'agent/events.jsonl' source.write_text('\n'.join(map(json.dumps, events)) + '\n') before[source] = source.read_bytes() manifest = fixture.materialize() folder = fixture.root / 'datasets/candidate' def rows(name): return [json.loads(line) for line in (folder / (name + '.jsonl')).read_text().splitlines()] for name in ('disposition', 'source-manifest', 'retained', 'validation', 'exclusions'): values = rows(name) self.assertEqual(len(values), 3, name) self.assertEqual({(r['run_id'], r['job_id'], r['rollout']) for r in values}, {('r', 'j', i) for i in range(3)}, name) self.assertTrue(all(r['disposition'] == 'no-usable' for r in rows('disposition'))) self.assertEqual(len(rows('excluded_observations')), 3) self.assertEqual(rows('sft'), []) self.assertEqual(rows('teacher'), []) self.assertEqual(manifest['stats']['scanned'], 3) self.assertEqual(manifest['stats']['excluded'], 3) for row in rows('validation'): checks = row['checks'] self.assertTrue(checks['OBSERVATION_ACCOUNTING']['pass']) self.assertTrue(checks['ERROR_EXCLUSION']['pass']) self.assertFalse(checks['USABLE_ASSISTANT']['pass']) with self.assertRaisesRegex(PublicationError, 'NO_ACCEPTED_SAMPLES'): fixture.build() self.assertEqual({p: p.read_bytes() for p in before}, before) self.assertFalse((fixture.root / 'releases/CURRENT.json').exists()) self.receipt('DATA-012', input_attempts=3, unique_dispositions=3, excluded_empty_errors=3, accepted=0, publication='NO_ACCEPTED_SAMPLES', source_unchanged=True) @unittest.skipUnless(sys.platform == 'linux', 'Linux cross-process publication locking') def test_revoke_and_rollback_processes_serialize_in_both_lock_orders(self): outcomes = [] for first in ('revoke', 'rollback'): with self.subTest(first=first): fixture = self.fixture(test_publication.PublicationTests) old, current = fixture.seal(), fixture.seal('rules-b') fixture.switch(old); fixture.switch(current, 1, 'second') children = [] def launch(role, hold): proc = subprocess.Popen([sys.executable, '-c', RACER, str(fixture.root), old, role, str(int(hold))], cwd=ROOT, env={**os.environ, 'PYTHONPATH': str(ROOT), 'PYTHONDONTWRITEBYTECODE': '1'}, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) children.append(proc) return proc def wait(path): deadline = time.monotonic() + 5 while not path.exists() and time.monotonic() < deadline: time.sleep(.005) self.assertTrue(path.exists(), path.name) try: launch(first, True) wait(fixture.root / (first + '-holding')) second = 'rollback' if first == 'revoke' else 'revoke' following = launch(second, False) wait(fixture.root / (second + '-ready')) self.assertIsNone(following.poll()) # The first process still owns the real file lock. (fixture.root / 'release').touch() result = {} for proc in children: out, err = proc.communicate(timeout=6) self.assertEqual(proc.returncode, 0, err) row = json.loads(out); result[row['role']] = row['result'] self.assertEqual(result['revoke'], 'REVOKED') self.assertEqual(result['rollback'], 'VERSION_REVOKED' if first == 'revoke' else 'COMMIT') if first == 'rollback': with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'): fixture.store.current() else: self.assertEqual(fixture.store.current()['pointer']['dataset_version'], current) with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'): fixture.store.switch(old, expected_revision=2, event_id='after-revoke', reason='must refuse', rollback=True) outcomes.append({'first_lock_owner': first, **result}) finally: (fixture.root / 'release').touch() for proc in children: if proc.poll() is None: proc.kill() proc.communicate(timeout=3) self.receipt('DATA-013/VERSION-001', real_processes=4, orders=outcomes, revoked_target_served=False) @unittest.skipUnless(sys.platform == 'linux' and test_pi_session.NODE.is_file(), 'Linux and installed Pi SDK') def test_provider_only_error_wrapped_exit_zero_is_not_completed_or_sft(self): from harness import collection_runtime as runtime, db, paths, rollout from harness.runners import pi from harness.verify import verify_rollout fixture = self.fixture(test_publication.BuildTests) sdk = test_pi_session.PiSDKTests(); sdk.setUp(); self.addCleanup(sdk.doCleanups) sdk.fail_second, sdk.fail_on = True, {1, 2, 3, 4} rd = fixture.fixture.add(terminal='FAIL', execution='RUNNING') fixture.fixture.con.execute("UPDATE rollouts SET mode='e2e', policy='teacher', state='RUNNING'") fixture.fixture.con.commit() job = json.loads((rd.parent.parent / 'job.json').read_text()) measured_before = verify_rollout(rd, write=False)['gates'] gate_bytes = {p: p.read_bytes() for p in (rd / 'stages').rglob('meta.json')} workspace = rd / 'workspace'; workspace.mkdir() shutil.move(str(sdk.root / '.pi'), workspace / '.pi') shutil.move(str(sdk.home), workspace / '.agent-home') models_path = workspace / '.agent-home/.pi/agent/models.json' models = json.loads(models_path.read_text()) models['providers']['dr-gateway'] = models['providers'].pop('fixture-provider') models_path.write_text(json.dumps(models)) settings_path = workspace / '.pi/settings.json' settings = json.loads(settings_path.read_text()); settings['defaultProvider'] = 'dr-gateway' settings_path.write_text(json.dumps(settings)) for name in ('system.md', 'append-system.md', 'prompt.md'): (rd / 'agent' / name).write_text('Attempt the task once.') (rd / 'agent/events.jsonl').write_bytes(b'') metadata = json.loads((rd / 'rollout.json').read_text()) metadata.pop('execution_id') # The actual controller allocates this execution. metadata.update(runner='pi', jail_policy='private-home', max_turns=60, max_tokens=200000) (rd / 'rollout.json').write_text(json.dumps(metadata)) config = yaml.safe_load((ROOT / 'config.yaml').read_text()) config['paths'].update(work_root=str(fixture.root/'work'), node_bin=str(test_pi_session.NODE)) config['rollouts'].update(jail=False, private_home=True, tools=['write'], hard_ceiling_s=15) config['endpoints']['teacher'].update(base_url='http://127.0.0.1:%d/v1' % sdk.server.server_port, model='fixture', api_key_env='FIXTURE_KEY') config['collection_v1'].update(quota_ledger=str(fixture.root/'runtime-quota.sqlite'), quota_groups={'account': asdict(sdk.broker.groups['account'])}, credential_groups={'teacher': 'account'}) config_path = fixture.root/'config.yaml'; config_path.write_text(yaml.safe_dump(config)) original_connect, original_readonly, command = db.connect, db.readonly, pi.command def wrapped_command(*args): script = ('import subprocess,sys;from pathlib import Path;' 'result=subprocess.run(sys.argv[2:]);Path(sys.argv[1]).write_text(str(result.returncode));sys.exit(0)') return [sys.executable, '-c', script, str(workspace/'sdk-exit'), *command(*args)] with ExitStack() as stack: for name, value in {'BENCH_ROOT': fixture.root, 'PROJECT_ROOT': fixture.root.parent, 'AGENT': ROOT/'agent', 'RUNS': fixture.root/'runs', 'STATE': fixture.root/'state', 'CONFIG_PATH': config_path}.items(): stack.enter_context(patch.object(paths, name, value)) stack.enter_context(patch.object(paths, 'config', return_value=config)) stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture')) stack.enter_context(patch.object(db, 'connect', lambda: original_connect(fixture.root/'state/jobs.sqlite'))) stack.enter_context(patch.object(db, 'readonly', lambda: original_readonly(fixture.root/'state/jobs.sqlite'))) stack.enter_context(patch.object(runtime, '_broker', None)) stack.enter_context(patch.object(runtime, '_configuration', None)) stack.enter_context(patch.object(rollout, 'prepare', return_value=(rd, job, 'Attempt the task once.'))) stack.enter_context(patch.object(pi, 'command', side_effect=wrapped_command)) try: rollout._execute_rollout('r', 'j', 0, 'e2e', 'teacher', False) state = json.loads((rd/'execution.json').read_text()) self.assertEqual((workspace/'sdk-exit').read_text(), '76') self.assertEqual(state['exit_code'], 0) self.assertEqual(state['execution_state'], 'TERMINAL') self.assertNotEqual(state['execution_outcome'], 'COMPLETED') self.assertFalse(state['protocol_complete']) self.assertEqual(state['cleanup_state'], 'VERIFIED') self.assertEqual(json.loads((rd/'rollout.json').read_text())['state'], 'ERROR') self.assertEqual(verify_rollout(rd, write=False)['gates'], measured_before) self.assertEqual({p: p.read_bytes() for p in gate_bytes}, gate_bytes) self.assertFalse((workspace/'side-effect.txt').exists()) errors = [json.loads(line) for line in (rd/'agent/events.jsonl').read_text().splitlines() if json.loads(line).get('type') == 'message_end'] assistants = [r['message'] for r in errors if r['message']['role'] == 'assistant'] self.assertTrue(assistants) self.assertTrue(all(m.get('stopReason') == 'error' and not m['content'] for m in assistants)) quota = runtime.broker().execution_status(state['execution_id']) # This execution is terminal, unlike the resumable pause path. # Its request must never become eligible after a slot is freed. self.assertTrue(quota['cancelled']) self.assertEqual(quota['generations'][-1]['state'], 'CANCELLED') self.assertTrue(quota['generations'][-1]['reason']) self.assertEqual(fixture.materialize()['stats']['sft_eligible'], 0) self.assertEqual((fixture.root/'datasets/candidate/sft.jsonl').read_bytes(), b'') self.receipt('JOINT-008', sdk_exit=76, wrapper_exit=0, outcome=state['execution_outcome'], protocol_complete=False, cleanup='VERIFIED', gates_unchanged=True, accepted_sft=0, upstream_requests=len(sdk.bodies), quota_reason=quota['generations'][-1]['reason']) finally: if runtime._broker: runtime._broker.close() if __name__ == '__main__': unittest.main()