224 lines
12 KiB
Python
224 lines
12 KiB
Python
"""Real drain/reconcile/sync protocol with an owned process and compatible config.
|
|
|
|
Only the task payload is a local wait-and-write fixture. The collector, CLI,
|
|
process containment, execution/DB ledgers, broker and source sync are real.
|
|
"""
|
|
from contextlib import closing
|
|
from dataclasses import asdict
|
|
import base64
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
import uuid
|
|
|
|
import yaml
|
|
from scripts import release_files, sync_patch, test_execution, test_pi_session, test_quota
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COLLECTOR = r'''
|
|
import json,sys
|
|
from pathlib import Path
|
|
from harness import cli, collection_runtime, db, execution, paths, rollout, watchdog
|
|
# Replace the costly task only; retain actual collector admission, preflight,
|
|
# drain handling, DB commits, export entry and all lifecycle/process code.
|
|
def local_task(run_id, job_id, kk, mode, policy, **kwargs):
|
|
rd = paths.RUNS/run_id/job_id/'rollouts'/f'{kk:02d}'
|
|
rd.mkdir(parents=True, exist_ok=True)
|
|
owner = execution.Execution(rd)
|
|
owner.start(collection_runtime.config()[0]['lifecycle'])
|
|
command = """
|
|
import os,sys,time
|
|
from pathlib import Path
|
|
rd, root, run_id = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3]
|
|
(root/('entered-'+run_id)).touch()
|
|
deadline = time.monotonic()+10
|
|
while not (root/('release-'+run_id)).exists():
|
|
if time.monotonic()>deadline: raise TimeoutError('fixture release missing')
|
|
time.sleep(.01)
|
|
with (rd/'side-effect.txt').open('a') as stream:
|
|
stream.write('once\\n'); stream.flush(); os.fsync(stream.fileno())
|
|
"""
|
|
result = watchdog.run([sys.executable,'-c',command,str(rd),str(paths.BENCH_ROOT),run_id],
|
|
cwd=rd, stdout=rd/'stdout.log', stderr=rd/'stderr.log', execution=owner, poll_s=.02)
|
|
state = owner.finish(result.exit_code, protocol_complete=result.state=='OK', cleanup_state=result.cleanup_state)
|
|
assert state['execution_outcome']=='COMPLETED', state
|
|
with db.connect() as con:
|
|
db.rollout_set(con,run_id,job_id,kk,state='UNVERIFIED',finished_at=state['finished_at'],
|
|
verdict={'terminal':'UNVERIFIED','fixture':'no-board'})
|
|
return {'terminal':'UNVERIFIED','claim_consistent':False}
|
|
rollout.run_rollout = local_task
|
|
sys.exit(cli.main(['collect','--run',sys.argv[1]]))
|
|
'''
|
|
|
|
|
|
@unittest.skipUnless(Path('/proc').is_dir() and test_pi_session.NODE.is_file(), 'Linux and installed Pi preflight required')
|
|
class RuntimeRollback(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory(prefix='bench-runtime-rollback-')
|
|
self.addCleanup(temp.cleanup)
|
|
self.root = Path(temp.name)
|
|
for source in release_files.source_files(ROOT):
|
|
target = self.root / source.relative_to(ROOT)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(source, target)
|
|
self.quota = test_quota.Quota(); self.quota.setUp()
|
|
self.addCleanup(self.quota.doCleanups)
|
|
self.provider = self.quota.provider(lambda n, body: test_quota.ok(n, body) if n == 1 else
|
|
(429, {'Retry-After': '60'}, b'{"error":{"type":"rate_limit"}}'))
|
|
self.quota.ledger = self.root / 'state/quota.sqlite'
|
|
self.policy = self.quota.policy(wait_budget_s=.03, window_s=60)
|
|
self.broker = self.quota.broker(self.provider, self.policy)
|
|
self.scope = self.broker.issue_scope('before-rollback', 'teacher', 'glm-test')
|
|
self.assertEqual(self.quota.request(self.scope, 'completed')[0], 200)
|
|
self.assertNotEqual(self.quota.request(self.scope, 'paused')[0], 200)
|
|
self.broker.close()
|
|
lifecycle = test_execution.policy(); lifecycle.update(version='runtime-v2', execution_deadline_s=20)
|
|
self.config = {'paths': {'harness_python': sys.executable, 'node_bin': str(test_pi_session.NODE),
|
|
'work_root': str(self.root/'work')},
|
|
'pools': {'agent': {'workers': 1}}, 'rollouts': {'private_home': True, 'tools': ['write']},
|
|
'endpoints': {'teacher': {'base_url': self.provider.url, 'model': 'glm-test', 'api_key_env': 'ROLLBACK_FIXTURE_KEY'}},
|
|
'collection_v1': {'version': 'collection-v2', 'quota_ledger': str(self.quota.ledger),
|
|
'quota_groups': {'shared': asdict(self.policy)}, 'credential_groups': {'teacher': 'shared'},
|
|
'lifecycle': lifecycle}}
|
|
(self.root/'config.yaml').write_text(yaml.safe_dump(self.config))
|
|
self.env = {k: v for k, v in os.environ.items() if not k.startswith('BENCH_')}
|
|
self.env.update(BENCH_ROOT=str(self.root), BENCH_CONFIG=str(self.root/'config.yaml'),
|
|
BENCH_STATE_DIR=str(self.root/'state'), BENCH_PYTHON=sys.executable,
|
|
PYTHONPATH=str(self.root), PYTHONDONTWRITEBYTECODE='1', ROLLBACK_FIXTURE_KEY='upstream-secret')
|
|
self.seed('r', ['first', 'second'])
|
|
|
|
def python(self, program, *args):
|
|
result = subprocess.run([sys.executable, '-c', program, *args], cwd=self.root, env=self.env,
|
|
capture_output=True, text=True, timeout=15)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
return result.stdout
|
|
|
|
def seed(self, run_id, jobs):
|
|
self.python("""
|
|
import json,sys
|
|
from harness import db,paths
|
|
rid,jobs=sys.argv[1],json.loads(sys.argv[2])
|
|
with db.connect() as con:
|
|
db.create_run(con,rid,{})
|
|
for jid in jobs:
|
|
db.rollout_upsert(con,rid,jid,0,'e2e','teacher')
|
|
jd=paths.RUNS/rid/jid;jd.mkdir(parents=True)
|
|
(jd/'job.json').write_text(json.dumps({'run_id':rid,'job_id':jid}))
|
|
""", run_id, json.dumps(jobs))
|
|
|
|
def cli(self, *args):
|
|
result = subprocess.run([str(self.root/'bin/bench'), *args], cwd=self.root, env=self.env,
|
|
capture_output=True, text=True, timeout=15)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
return result.stdout
|
|
|
|
def collect(self, run_id):
|
|
proc = subprocess.Popen([sys.executable, '-c', COLLECTOR, run_id], cwd=self.root, env=self.env,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
def cleanup():
|
|
# Release only this fixture; let Scope finish and reap before deleting.
|
|
(self.root/('release-'+run_id)).touch()
|
|
try: proc.communicate(timeout=15)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill(); proc.communicate(timeout=3)
|
|
self.addCleanup(cleanup)
|
|
return proc
|
|
|
|
def rows(self, run='r'):
|
|
with closing(sqlite3.connect('file:'+str(self.root/'state/jobs.sqlite')+'?mode=ro', uri=True)) as con:
|
|
return con.execute('SELECT job_id,kk,state,started_at FROM rollouts WHERE run_id=? ORDER BY job_id', (run,)).fetchall()
|
|
|
|
@staticmethod
|
|
def dump(path):
|
|
with closing(sqlite3.connect('file:'+str(path)+'?mode=ro', uri=True)) as con:
|
|
return '\n'.join(con.iterdump())
|
|
|
|
def test_drain_complete_reconcile_then_compatible_config_rollback(self):
|
|
collector = self.collect('r')
|
|
deadline = time.monotonic()+10
|
|
while not (self.root/'entered-r').exists() and collector.poll() is None and time.monotonic()<deadline:
|
|
time.sleep(.01)
|
|
self.assertTrue((self.root/'entered-r').exists(), collector.communicate(timeout=2) if collector.poll() is not None else 'entry deadline')
|
|
rd = self.root/'runs/r/first/rollouts/00'
|
|
self.assertEqual([r[2] for r in self.rows()], ['RUNNING','QUEUED'])
|
|
self.cli('drain','--run','r','--all')
|
|
self.assertTrue((self.root/'state/drain-collect-r').is_file())
|
|
self.assertIsNone(json.loads((rd/'execution.json').read_text())['cancel_requested'])
|
|
self.assertIsNone(collector.poll())
|
|
(self.root/'release-r').touch()
|
|
out, err = collector.communicate(timeout=15)
|
|
self.assertEqual(collector.returncode, 0, (out,err))
|
|
self.assertEqual([r[2] for r in self.rows()], ['UNVERIFIED','QUEUED'])
|
|
self.assertTrue(json.loads(self.cli('execution','reconcile','--run','r'))['ready'])
|
|
state = json.loads((rd/'execution.json').read_text())
|
|
self.assertEqual((state['execution_outcome'],state['cleanup_state'],state['config_version']),
|
|
('COMPLETED','VERIFIED','runtime-v2'))
|
|
receipts = [json.loads(p.read_text()) for p in (rd/'process-receipts').glob('*.json')]
|
|
self.assertTrue(receipts)
|
|
self.assertTrue(all(r['all_children_reaped'] and r['cleanup_state']=='VERIFIED'
|
|
and r['binding']['proof']=='waitpid:ECHILD' for r in receipts))
|
|
self.assertEqual((rd/'side-effect.txt').read_text(),'once\n')
|
|
frozen = {p: p.read_bytes() for p in (self.root/'runs/r').rglob('*') if p.is_file()}
|
|
db_before, quota_before = self.dump(self.root/'state/jobs.sqlite'), self.dump(self.quota.ledger)
|
|
before = sync_patch.snapshot(self.root, for_sync=True)
|
|
previous = copy.deepcopy(self.config)
|
|
previous['collection_v1']['version'] = 'collection-v1-compatible'
|
|
previous['collection_v1']['lifecycle'].update(version='runtime-v1-compatible', execution_deadline_s=15)
|
|
raw = yaml.safe_dump(previous).encode()
|
|
payload = {'operation':'apply', 'root':str(self.root), 'expected':before, 'id':uuid.uuid4().hex,
|
|
'files':[{'path':'config.yaml','data':base64.b64encode(raw).decode(),
|
|
'sha256':hashlib.sha256(raw).hexdigest(),'mode':0o644}]}
|
|
result = subprocess.run([sys.executable,'-c',sync_patch.REMOTE_PROGRAM],input=json.dumps(payload),
|
|
capture_output=True,text=True,env=self.env,timeout=15)
|
|
self.assertEqual(result.returncode,0,result.stderr)
|
|
receipt = json.loads(result.stdout)
|
|
self.assertEqual(receipt['changed'],['config.yaml']);self.assertTrue(receipt['complete'])
|
|
self.assertEqual(yaml.safe_load((Path(receipt['backup'])/'config.yaml').read_text()),self.config)
|
|
after = sync_patch.snapshot(self.root, for_sync=True)
|
|
self.assertEqual({k:v for k,v in before['files'].items() if k!='config.yaml'},
|
|
{k:v for k,v in after['files'].items() if k!='config.yaml'})
|
|
self.assertEqual({p:p.read_bytes() for p in frozen},frozen)
|
|
self.assertEqual(self.dump(self.root/'state/jobs.sqlite'),db_before)
|
|
self.assertEqual(self.dump(self.quota.ledger),quota_before)
|
|
self.assertTrue(json.loads(self.cli('execution','reconcile','--run','r'))['ready'])
|
|
# Restarting the drained old run cannot silently claim its remaining row.
|
|
restarted=self.collect('r');out,err=restarted.communicate(timeout=15)
|
|
self.assertEqual(restarted.returncode,0,(out,err))
|
|
self.assertEqual(self.dump(self.root/'state/jobs.sqlite'),db_before)
|
|
self.assertEqual({p:p.read_bytes() for p in frozen},frozen)
|
|
# A newly admitted execution uses the compatible old policy; the previous
|
|
# execution's frozen policy and request budget do not rewind.
|
|
self.seed('next',['new']);(self.root/'release-next').touch()
|
|
fresh=self.collect('next');out,err=fresh.communicate(timeout=15)
|
|
self.assertEqual(fresh.returncode,0,(out,err))
|
|
next_state=json.loads((self.root/'runs/next/new/rollouts/00/execution.json').read_text())
|
|
self.assertEqual(next_state['config_version'],'runtime-v1-compatible')
|
|
self.assertEqual((self.root/'runs/next/new/rollouts/00/side-effect.txt').read_text(),'once\n')
|
|
self.assertNotEqual(next_state['execution_id'],state['execution_id'])
|
|
self.assertEqual({p:p.read_bytes() for p in frozen},frozen)
|
|
self.assertEqual(self.dump(self.quota.ledger),quota_before)
|
|
reopened=self.quota.broker(self.provider,self.policy)
|
|
scope=reopened.issue_scope('before-rollback','teacher','glm-test')
|
|
self.assertEqual(self.quota.request(scope,'completed')[0],409)
|
|
self.assertEqual(len(self.provider.requests),2)
|
|
self.assertEqual([r[2] for r in self.rows()],['UNVERIFIED','QUEUED'])
|
|
print('JOINT_RECEIPT '+json.dumps({'case':'JOINT-013-runtime-config','drain_cancelled_active':False,
|
|
'original_attempts':2,'completed_side_effects':1,'queued_after_restart':1,
|
|
'rollback_changed_files':receipt['changed'],'old_execution_version':state['config_version'],
|
|
'new_execution_version':next_state['config_version'],'original_source_bytes_unchanged':True,
|
|
'quota_sql_dump_unchanged':True,'completed_generation_replay_http':409,'provider_requests':2,
|
|
'scope':'compatible config rollback; production business source and exporter unchanged'}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|