data-gen-bpu/export/test_publication_runtime.py

102 lines
4.7 KiB
Python

"""Publication crash/rollback while durable execution and quota state coexist."""
from contextlib import closing
import hashlib
import json
import os
from pathlib import Path
import sqlite3
import subprocess
import sys
import unittest
from export.publication import Releases
from export import test_publication
from harness.execution import Execution
from scripts.test_execution import policy
from scripts import test_quota
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux execution identities required')
class PublicationRuntime(unittest.TestCase):
def setUp(self):
self.data = test_publication.BuildTests()
self.data.setUp()
self.addCleanup(self.data.tearDown)
self.quota = test_quota.Quota()
self.quota.setUp()
self.addCleanup(self.quota.doCleanups)
def ledger(self):
with closing(sqlite3.connect('file:' + str(self.quota.ledger) + '?mode=ro', uri=True)) as con:
dump = '\n'.join(con.iterdump())
return hashlib.sha256(dump.encode()).hexdigest()
def test_crash_and_rollback_never_rewind_execution_or_http_budget(self):
runtime = Path(self.quota.tmp.name) / 'runtime'
execution = Execution(runtime)
execution.start(policy())
provider = self.quota.provider(lambda n, body: test_quota.ok(n, body) if n == 1 else
(429, {'Retry-After': '60'}, b'{"error":{"type":"rate_limit"}}'))
broker = self.quota.broker(provider, self.quota.policy(wait_budget_s=.03, window_s=60))
scope = broker.issue_scope(execution.snapshot()['execution_id'], 'teacher', 'glm-test')
self.assertEqual(self.quota.request(scope, 'completed')[0], 200)
self.assertNotEqual(self.quota.request(scope, 'paused')[0], 200)
self.assertEqual(len(provider.requests), 2)
self.assertEqual(broker.execution_status(scope.execution_id)['generations'][-1]['state'], 'PAUSED')
checkpoint = runtime / 'checkpoint.jsonl'
checkpoint.write_text('{"fixture":"durable completed tool"}\n')
execution.pause('quota', {'complete': True, 'path': str(checkpoint),
'sha256': hashlib.sha256(checkpoint.read_bytes()).hexdigest()})
frozen = {p: p.read_bytes() for p in (runtime / 'execution.json', checkpoint)}
quota_before = self.ledger()
self.data.fixture.add()
self.data.materialize()
first = self.data.build()['dataset_version']
store = Releases(self.data.root / 'releases')
store.switch(first, expected_revision=0, event_id='first', reason='fixture')
self.data.fixture.add(1)
from export.to_agentrl import materialize
materialize(['r'], 'candidate', refresh=True)
second = self.data.build()['dataset_version']
self.assertNotEqual(first, second)
# A real child exits between filesystem operations; exceptions alone
# would exercise finally blocks that are absent in an actual crash.
program = """
import os,sys
from pathlib import Path
from export.publication import Releases
def fault(point):
if point == sys.argv[4]: os._exit(67)
Releases(Path(sys.argv[1]), fault=fault).switch(sys.argv[2],
expected_revision=1, event_id=sys.argv[3], reason='crash fixture')
"""
for phase, expected in (('after_intent', 'ABORT'), ('before_replace', 'ABORT'), ('after_replace', 'COMMIT')):
with self.subTest(phase=phase):
event = 'crash-' + phase
child = subprocess.run([sys.executable, '-c', program, str(store.root), second, event, phase],
cwd=Path(__file__).resolve().parents[1], env={**os.environ, 'PYTHONDONTWRITEBYTECODE': '1'},
capture_output=True, text=True, timeout=10)
self.assertEqual(child.returncode, 67, child.stderr)
self.assertEqual(store.status(event)['state'], expected)
self.assertEqual(self.ledger(), quota_before)
self.assertEqual({p: p.read_bytes() for p in frozen}, frozen)
self.assertEqual(len(provider.requests), 2)
store.switch(first, expected_revision=2, event_id='rollback', reason='fixture rollback', rollback=True)
current = store.current()
self.assertEqual(current['pointer']['dataset_version'], first)
self.assertEqual(current['acceptance']['accepted'], 1)
self.assertEqual(current['manifest']['rules_version'], 'sft-preserve-v1')
self.assertEqual(self.ledger(), quota_before)
self.assertEqual({p: p.read_bytes() for p in frozen}, frozen)
self.assertEqual(self.quota.request(scope, 'completed')[0], 409)
self.assertEqual(len(provider.requests), 2)
self.assertEqual(list(store.root.glob('.CURRENT-*')), [])
if __name__ == '__main__':
unittest.main()