"""Real drain/reconcile/sync protocol with an owned process and compatible config. Only the task payload is a local wait-and-write fixture. The collector, CLI, process containment, execution/DB ledgers, broker and source sync are real. """ from contextlib import closing from dataclasses import asdict import base64 import copy import hashlib import json import os from pathlib import Path import shutil import sqlite3 import subprocess import sys import tempfile import time import unittest import uuid import yaml from scripts import release_files, sync_patch, test_execution, test_pi_session, test_quota ROOT = Path(__file__).resolve().parents[1] COLLECTOR = r''' import json,sys from pathlib import Path from harness import cli, collection_runtime, db, execution, paths, rollout, watchdog # Replace the costly task only; retain actual collector admission, preflight, # drain handling, DB commits, export entry and all lifecycle/process code. def local_task(run_id, job_id, kk, mode, policy, **kwargs): rd = paths.RUNS/run_id/job_id/'rollouts'/f'{kk:02d}' rd.mkdir(parents=True, exist_ok=True) owner = execution.Execution(rd) owner.start(collection_runtime.config()[0]['lifecycle']) command = """ import os,sys,time from pathlib import Path rd, root, run_id = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3] (root/('entered-'+run_id)).touch() deadline = time.monotonic()+10 while not (root/('release-'+run_id)).exists(): if time.monotonic()>deadline: raise TimeoutError('fixture release missing') time.sleep(.01) with (rd/'side-effect.txt').open('a') as stream: stream.write('once\\n'); stream.flush(); os.fsync(stream.fileno()) """ result = watchdog.run([sys.executable,'-c',command,str(rd),str(paths.BENCH_ROOT),run_id], cwd=rd, stdout=rd/'stdout.log', stderr=rd/'stderr.log', execution=owner, poll_s=.02) state = owner.finish(result.exit_code, protocol_complete=result.state=='OK', cleanup_state=result.cleanup_state) assert state['execution_outcome']=='COMPLETED', state with db.connect() as con: db.rollout_set(con,run_id,job_id,kk,state='UNVERIFIED',finished_at=state['finished_at'], verdict={'terminal':'UNVERIFIED','fixture':'no-board'}) return {'terminal':'UNVERIFIED','claim_consistent':False} rollout.run_rollout = local_task sys.exit(cli.main(['collect','--run',sys.argv[1]])) ''' @unittest.skipUnless(Path('/proc').is_dir() and test_pi_session.NODE.is_file(), 'Linux and installed Pi preflight required') class RuntimeRollback(unittest.TestCase): def setUp(self): temp = tempfile.TemporaryDirectory(prefix='bench-runtime-rollback-') self.addCleanup(temp.cleanup) self.root = Path(temp.name) for source in release_files.source_files(ROOT): target = self.root / source.relative_to(ROOT) target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, target) self.quota = test_quota.Quota(); self.quota.setUp() self.addCleanup(self.quota.doCleanups) self.provider = self.quota.provider(lambda n, body: test_quota.ok(n, body) if n == 1 else (429, {'Retry-After': '60'}, b'{"error":{"type":"rate_limit"}}')) self.quota.ledger = self.root / 'state/quota.sqlite' self.policy = self.quota.policy(wait_budget_s=.03, window_s=60) self.broker = self.quota.broker(self.provider, self.policy) self.scope = self.broker.issue_scope('before-rollback', 'teacher', 'glm-test') self.assertEqual(self.quota.request(self.scope, 'completed')[0], 200) self.assertNotEqual(self.quota.request(self.scope, 'paused')[0], 200) self.broker.close() lifecycle = test_execution.policy(); lifecycle.update(version='runtime-v2', execution_deadline_s=20) self.config = {'paths': {'harness_python': sys.executable, 'node_bin': str(test_pi_session.NODE), 'work_root': str(self.root/'work')}, 'pools': {'agent': {'workers': 1}}, 'rollouts': {'private_home': True, 'tools': ['write']}, 'endpoints': {'teacher': {'base_url': self.provider.url, 'model': 'glm-test', 'api_key_env': 'ROLLBACK_FIXTURE_KEY'}}, 'collection_v1': {'version': 'collection-v2', 'quota_ledger': str(self.quota.ledger), 'quota_groups': {'shared': asdict(self.policy)}, 'credential_groups': {'teacher': 'shared'}, 'lifecycle': lifecycle}} (self.root/'config.yaml').write_text(yaml.safe_dump(self.config)) self.env = {k: v for k, v in os.environ.items() if not k.startswith('BENCH_')} self.env.update(BENCH_ROOT=str(self.root), BENCH_CONFIG=str(self.root/'config.yaml'), BENCH_STATE_DIR=str(self.root/'state'), BENCH_PYTHON=sys.executable, PYTHONPATH=str(self.root), PYTHONDONTWRITEBYTECODE='1', ROLLBACK_FIXTURE_KEY='upstream-secret') self.seed('r', ['first', 'second']) def python(self, program, *args): result = subprocess.run([sys.executable, '-c', program, *args], cwd=self.root, env=self.env, capture_output=True, text=True, timeout=15) self.assertEqual(result.returncode, 0, result.stderr) return result.stdout def seed(self, run_id, jobs): self.python(""" import json,sys from harness import db,paths rid,jobs=sys.argv[1],json.loads(sys.argv[2]) with db.connect() as con: db.create_run(con,rid,{}) for jid in jobs: db.rollout_upsert(con,rid,jid,0,'e2e','teacher') jd=paths.RUNS/rid/jid;jd.mkdir(parents=True) (jd/'job.json').write_text(json.dumps({'run_id':rid,'job_id':jid})) """, run_id, json.dumps(jobs)) def cli(self, *args): result = subprocess.run([str(self.root/'bin/bench'), *args], cwd=self.root, env=self.env, capture_output=True, text=True, timeout=15) self.assertEqual(result.returncode, 0, result.stderr) return result.stdout def collect(self, run_id): proc = subprocess.Popen([sys.executable, '-c', COLLECTOR, run_id], cwd=self.root, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) def cleanup(): # Release only this fixture; let Scope finish and reap before deleting. (self.root/('release-'+run_id)).touch() try: proc.communicate(timeout=15) except subprocess.TimeoutExpired: proc.kill(); proc.communicate(timeout=3) self.addCleanup(cleanup) return proc def rows(self, run='r'): with closing(sqlite3.connect('file:'+str(self.root/'state/jobs.sqlite')+'?mode=ro', uri=True)) as con: return con.execute('SELECT job_id,kk,state,started_at FROM rollouts WHERE run_id=? ORDER BY job_id', (run,)).fetchall() @staticmethod def dump(path): with closing(sqlite3.connect('file:'+str(path)+'?mode=ro', uri=True)) as con: return '\n'.join(con.iterdump()) def test_drain_complete_reconcile_then_compatible_config_rollback(self): collector = self.collect('r') deadline = time.monotonic()+10 while not (self.root/'entered-r').exists() and collector.poll() is None and time.monotonic()