data-gen-bpu/scripts/test_runtime_rollback.py

224 lines
12 KiB
Python

"""Real drain/reconcile/sync protocol with an owned process and compatible config.
Only the task payload is a local wait-and-write fixture. The collector, CLI,
process containment, execution/DB ledgers, broker and source sync are real.
"""
from contextlib import closing
from dataclasses import asdict
import base64
import copy
import hashlib
import json
import os
from pathlib import Path
import shutil
import sqlite3
import subprocess
import sys
import tempfile
import time
import unittest
import uuid
import yaml
from scripts import release_files, sync_patch, test_execution, test_pi_session, test_quota
ROOT = Path(__file__).resolve().parents[1]
COLLECTOR = r'''
import json,sys
from pathlib import Path
from harness import cli, collection_runtime, db, execution, paths, rollout, watchdog
# Replace the costly task only; retain actual collector admission, preflight,
# drain handling, DB commits, export entry and all lifecycle/process code.
def local_task(run_id, job_id, kk, mode, policy, **kwargs):
rd = paths.RUNS/run_id/job_id/'rollouts'/f'{kk:02d}'
rd.mkdir(parents=True, exist_ok=True)
owner = execution.Execution(rd)
owner.start(collection_runtime.config()[0]['lifecycle'])
command = """
import os,sys,time
from pathlib import Path
rd, root, run_id = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3]
(root/('entered-'+run_id)).touch()
deadline = time.monotonic()+10
while not (root/('release-'+run_id)).exists():
if time.monotonic()>deadline: raise TimeoutError('fixture release missing')
time.sleep(.01)
with (rd/'side-effect.txt').open('a') as stream:
stream.write('once\\n'); stream.flush(); os.fsync(stream.fileno())
"""
result = watchdog.run([sys.executable,'-c',command,str(rd),str(paths.BENCH_ROOT),run_id],
cwd=rd, stdout=rd/'stdout.log', stderr=rd/'stderr.log', execution=owner, poll_s=.02)
state = owner.finish(result.exit_code, protocol_complete=result.state=='OK', cleanup_state=result.cleanup_state)
assert state['execution_outcome']=='COMPLETED', state
with db.connect() as con:
db.rollout_set(con,run_id,job_id,kk,state='UNVERIFIED',finished_at=state['finished_at'],
verdict={'terminal':'UNVERIFIED','fixture':'no-board'})
return {'terminal':'UNVERIFIED','claim_consistent':False}
rollout.run_rollout = local_task
sys.exit(cli.main(['collect','--run',sys.argv[1]]))
'''
@unittest.skipUnless(Path('/proc').is_dir() and test_pi_session.NODE.is_file(), 'Linux and installed Pi preflight required')
class RuntimeRollback(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(prefix='bench-runtime-rollback-')
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
for source in release_files.source_files(ROOT):
target = self.root / source.relative_to(ROOT)
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, target)
self.quota = test_quota.Quota(); self.quota.setUp()
self.addCleanup(self.quota.doCleanups)
self.provider = self.quota.provider(lambda n, body: test_quota.ok(n, body) if n == 1 else
(429, {'Retry-After': '60'}, b'{"error":{"type":"rate_limit"}}'))
self.quota.ledger = self.root / 'state/quota.sqlite'
self.policy = self.quota.policy(wait_budget_s=.03, window_s=60)
self.broker = self.quota.broker(self.provider, self.policy)
self.scope = self.broker.issue_scope('before-rollback', 'teacher', 'glm-test')
self.assertEqual(self.quota.request(self.scope, 'completed')[0], 200)
self.assertNotEqual(self.quota.request(self.scope, 'paused')[0], 200)
self.broker.close()
lifecycle = test_execution.policy(); lifecycle.update(version='runtime-v2', execution_deadline_s=20)
self.config = {'paths': {'harness_python': sys.executable, 'node_bin': str(test_pi_session.NODE),
'work_root': str(self.root/'work')},
'pools': {'agent': {'workers': 1}}, 'rollouts': {'private_home': True, 'tools': ['write']},
'endpoints': {'teacher': {'base_url': self.provider.url, 'model': 'glm-test', 'api_key_env': 'ROLLBACK_FIXTURE_KEY'}},
'collection_v1': {'version': 'collection-v2', 'quota_ledger': str(self.quota.ledger),
'quota_groups': {'shared': asdict(self.policy)}, 'credential_groups': {'teacher': 'shared'},
'lifecycle': lifecycle}}
(self.root/'config.yaml').write_text(yaml.safe_dump(self.config))
self.env = {k: v for k, v in os.environ.items() if not k.startswith('BENCH_')}
self.env.update(BENCH_ROOT=str(self.root), BENCH_CONFIG=str(self.root/'config.yaml'),
BENCH_STATE_DIR=str(self.root/'state'), BENCH_PYTHON=sys.executable,
PYTHONPATH=str(self.root), PYTHONDONTWRITEBYTECODE='1', ROLLBACK_FIXTURE_KEY='upstream-secret')
self.seed('r', ['first', 'second'])
def python(self, program, *args):
result = subprocess.run([sys.executable, '-c', program, *args], cwd=self.root, env=self.env,
capture_output=True, text=True, timeout=15)
self.assertEqual(result.returncode, 0, result.stderr)
return result.stdout
def seed(self, run_id, jobs):
self.python("""
import json,sys
from harness import db,paths
rid,jobs=sys.argv[1],json.loads(sys.argv[2])
with db.connect() as con:
db.create_run(con,rid,{})
for jid in jobs:
db.rollout_upsert(con,rid,jid,0,'e2e','teacher')
jd=paths.RUNS/rid/jid;jd.mkdir(parents=True)
(jd/'job.json').write_text(json.dumps({'run_id':rid,'job_id':jid}))
""", run_id, json.dumps(jobs))
def cli(self, *args):
result = subprocess.run([str(self.root/'bin/bench'), *args], cwd=self.root, env=self.env,
capture_output=True, text=True, timeout=15)
self.assertEqual(result.returncode, 0, result.stderr)
return result.stdout
def collect(self, run_id):
proc = subprocess.Popen([sys.executable, '-c', COLLECTOR, run_id], cwd=self.root, env=self.env,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
def cleanup():
# Release only this fixture; let Scope finish and reap before deleting.
(self.root/('release-'+run_id)).touch()
try: proc.communicate(timeout=15)
except subprocess.TimeoutExpired:
proc.kill(); proc.communicate(timeout=3)
self.addCleanup(cleanup)
return proc
def rows(self, run='r'):
with closing(sqlite3.connect('file:'+str(self.root/'state/jobs.sqlite')+'?mode=ro', uri=True)) as con:
return con.execute('SELECT job_id,kk,state,started_at FROM rollouts WHERE run_id=? ORDER BY job_id', (run,)).fetchall()
@staticmethod
def dump(path):
with closing(sqlite3.connect('file:'+str(path)+'?mode=ro', uri=True)) as con:
return '\n'.join(con.iterdump())
def test_drain_complete_reconcile_then_compatible_config_rollback(self):
collector = self.collect('r')
deadline = time.monotonic()+10
while not (self.root/'entered-r').exists() and collector.poll() is None and time.monotonic()<deadline:
time.sleep(.01)
self.assertTrue((self.root/'entered-r').exists(), collector.communicate(timeout=2) if collector.poll() is not None else 'entry deadline')
rd = self.root/'runs/r/first/rollouts/00'
self.assertEqual([r[2] for r in self.rows()], ['RUNNING','QUEUED'])
self.cli('drain','--run','r','--all')
self.assertTrue((self.root/'state/drain-collect-r').is_file())
self.assertIsNone(json.loads((rd/'execution.json').read_text())['cancel_requested'])
self.assertIsNone(collector.poll())
(self.root/'release-r').touch()
out, err = collector.communicate(timeout=15)
self.assertEqual(collector.returncode, 0, (out,err))
self.assertEqual([r[2] for r in self.rows()], ['UNVERIFIED','QUEUED'])
self.assertTrue(json.loads(self.cli('execution','reconcile','--run','r'))['ready'])
state = json.loads((rd/'execution.json').read_text())
self.assertEqual((state['execution_outcome'],state['cleanup_state'],state['config_version']),
('COMPLETED','VERIFIED','runtime-v2'))
receipts = [json.loads(p.read_text()) for p in (rd/'process-receipts').glob('*.json')]
self.assertTrue(receipts)
self.assertTrue(all(r['all_children_reaped'] and r['cleanup_state']=='VERIFIED'
and r['binding']['proof']=='waitpid:ECHILD' for r in receipts))
self.assertEqual((rd/'side-effect.txt').read_text(),'once\n')
frozen = {p: p.read_bytes() for p in (self.root/'runs/r').rglob('*') if p.is_file()}
db_before, quota_before = self.dump(self.root/'state/jobs.sqlite'), self.dump(self.quota.ledger)
before = sync_patch.snapshot(self.root, for_sync=True)
previous = copy.deepcopy(self.config)
previous['collection_v1']['version'] = 'collection-v1-compatible'
previous['collection_v1']['lifecycle'].update(version='runtime-v1-compatible', execution_deadline_s=15)
raw = yaml.safe_dump(previous).encode()
payload = {'operation':'apply', 'root':str(self.root), 'expected':before, 'id':uuid.uuid4().hex,
'files':[{'path':'config.yaml','data':base64.b64encode(raw).decode(),
'sha256':hashlib.sha256(raw).hexdigest(),'mode':0o644}]}
result = subprocess.run([sys.executable,'-c',sync_patch.REMOTE_PROGRAM],input=json.dumps(payload),
capture_output=True,text=True,env=self.env,timeout=15)
self.assertEqual(result.returncode,0,result.stderr)
receipt = json.loads(result.stdout)
self.assertEqual(receipt['changed'],['config.yaml']);self.assertTrue(receipt['complete'])
self.assertEqual(yaml.safe_load((Path(receipt['backup'])/'config.yaml').read_text()),self.config)
after = sync_patch.snapshot(self.root, for_sync=True)
self.assertEqual({k:v for k,v in before['files'].items() if k!='config.yaml'},
{k:v for k,v in after['files'].items() if k!='config.yaml'})
self.assertEqual({p:p.read_bytes() for p in frozen},frozen)
self.assertEqual(self.dump(self.root/'state/jobs.sqlite'),db_before)
self.assertEqual(self.dump(self.quota.ledger),quota_before)
self.assertTrue(json.loads(self.cli('execution','reconcile','--run','r'))['ready'])
# Restarting the drained old run cannot silently claim its remaining row.
restarted=self.collect('r');out,err=restarted.communicate(timeout=15)
self.assertEqual(restarted.returncode,0,(out,err))
self.assertEqual(self.dump(self.root/'state/jobs.sqlite'),db_before)
self.assertEqual({p:p.read_bytes() for p in frozen},frozen)
# A newly admitted execution uses the compatible old policy; the previous
# execution's frozen policy and request budget do not rewind.
self.seed('next',['new']);(self.root/'release-next').touch()
fresh=self.collect('next');out,err=fresh.communicate(timeout=15)
self.assertEqual(fresh.returncode,0,(out,err))
next_state=json.loads((self.root/'runs/next/new/rollouts/00/execution.json').read_text())
self.assertEqual(next_state['config_version'],'runtime-v1-compatible')
self.assertEqual((self.root/'runs/next/new/rollouts/00/side-effect.txt').read_text(),'once\n')
self.assertNotEqual(next_state['execution_id'],state['execution_id'])
self.assertEqual({p:p.read_bytes() for p in frozen},frozen)
self.assertEqual(self.dump(self.quota.ledger),quota_before)
reopened=self.quota.broker(self.provider,self.policy)
scope=reopened.issue_scope('before-rollback','teacher','glm-test')
self.assertEqual(self.quota.request(scope,'completed')[0],409)
self.assertEqual(len(self.provider.requests),2)
self.assertEqual([r[2] for r in self.rows()],['UNVERIFIED','QUEUED'])
print('JOINT_RECEIPT '+json.dumps({'case':'JOINT-013-runtime-config','drain_cancelled_active':False,
'original_attempts':2,'completed_side_effects':1,'queued_after_restart':1,
'rollback_changed_files':receipt['changed'],'old_execution_version':state['config_version'],
'new_execution_version':next_state['config_version'],'original_source_bytes_unchanged':True,
'quota_sql_dump_unchanged':True,'completed_generation_replay_http':409,'provider_requests':2,
'scope':'compatible config rollback; production business source and exporter unchanged'}))
if __name__ == '__main__':
unittest.main()