102 lines
4.7 KiB
Python
102 lines
4.7 KiB
Python
"""Publication crash/rollback while durable execution and quota state coexist."""
|
|
from contextlib import closing
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
|
|
from export.publication import Releases
|
|
from export import test_publication
|
|
from harness.execution import Execution
|
|
from scripts.test_execution import policy
|
|
from scripts import test_quota
|
|
|
|
|
|
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux execution identities required')
|
|
class PublicationRuntime(unittest.TestCase):
|
|
def setUp(self):
|
|
self.data = test_publication.BuildTests()
|
|
self.data.setUp()
|
|
self.addCleanup(self.data.tearDown)
|
|
self.quota = test_quota.Quota()
|
|
self.quota.setUp()
|
|
self.addCleanup(self.quota.doCleanups)
|
|
|
|
def ledger(self):
|
|
with closing(sqlite3.connect('file:' + str(self.quota.ledger) + '?mode=ro', uri=True)) as con:
|
|
dump = '\n'.join(con.iterdump())
|
|
return hashlib.sha256(dump.encode()).hexdigest()
|
|
|
|
def test_crash_and_rollback_never_rewind_execution_or_http_budget(self):
|
|
runtime = Path(self.quota.tmp.name) / 'runtime'
|
|
execution = Execution(runtime)
|
|
execution.start(policy())
|
|
provider = self.quota.provider(lambda n, body: test_quota.ok(n, body) if n == 1 else
|
|
(429, {'Retry-After': '60'}, b'{"error":{"type":"rate_limit"}}'))
|
|
broker = self.quota.broker(provider, self.quota.policy(wait_budget_s=.03, window_s=60))
|
|
scope = broker.issue_scope(execution.snapshot()['execution_id'], 'teacher', 'glm-test')
|
|
self.assertEqual(self.quota.request(scope, 'completed')[0], 200)
|
|
self.assertNotEqual(self.quota.request(scope, 'paused')[0], 200)
|
|
self.assertEqual(len(provider.requests), 2)
|
|
self.assertEqual(broker.execution_status(scope.execution_id)['generations'][-1]['state'], 'PAUSED')
|
|
|
|
checkpoint = runtime / 'checkpoint.jsonl'
|
|
checkpoint.write_text('{"fixture":"durable completed tool"}\n')
|
|
execution.pause('quota', {'complete': True, 'path': str(checkpoint),
|
|
'sha256': hashlib.sha256(checkpoint.read_bytes()).hexdigest()})
|
|
frozen = {p: p.read_bytes() for p in (runtime / 'execution.json', checkpoint)}
|
|
quota_before = self.ledger()
|
|
|
|
self.data.fixture.add()
|
|
self.data.materialize()
|
|
first = self.data.build()['dataset_version']
|
|
store = Releases(self.data.root / 'releases')
|
|
store.switch(first, expected_revision=0, event_id='first', reason='fixture')
|
|
self.data.fixture.add(1)
|
|
from export.to_agentrl import materialize
|
|
materialize(['r'], 'candidate', refresh=True)
|
|
second = self.data.build()['dataset_version']
|
|
self.assertNotEqual(first, second)
|
|
|
|
# A real child exits between filesystem operations; exceptions alone
|
|
# would exercise finally blocks that are absent in an actual crash.
|
|
program = """
|
|
import os,sys
|
|
from pathlib import Path
|
|
from export.publication import Releases
|
|
def fault(point):
|
|
if point == sys.argv[4]: os._exit(67)
|
|
Releases(Path(sys.argv[1]), fault=fault).switch(sys.argv[2],
|
|
expected_revision=1, event_id=sys.argv[3], reason='crash fixture')
|
|
"""
|
|
for phase, expected in (('after_intent', 'ABORT'), ('before_replace', 'ABORT'), ('after_replace', 'COMMIT')):
|
|
with self.subTest(phase=phase):
|
|
event = 'crash-' + phase
|
|
child = subprocess.run([sys.executable, '-c', program, str(store.root), second, event, phase],
|
|
cwd=Path(__file__).resolve().parents[1], env={**os.environ, 'PYTHONDONTWRITEBYTECODE': '1'},
|
|
capture_output=True, text=True, timeout=10)
|
|
self.assertEqual(child.returncode, 67, child.stderr)
|
|
self.assertEqual(store.status(event)['state'], expected)
|
|
self.assertEqual(self.ledger(), quota_before)
|
|
self.assertEqual({p: p.read_bytes() for p in frozen}, frozen)
|
|
self.assertEqual(len(provider.requests), 2)
|
|
|
|
store.switch(first, expected_revision=2, event_id='rollback', reason='fixture rollback', rollback=True)
|
|
current = store.current()
|
|
self.assertEqual(current['pointer']['dataset_version'], first)
|
|
self.assertEqual(current['acceptance']['accepted'], 1)
|
|
self.assertEqual(current['manifest']['rules_version'], 'sft-preserve-v1')
|
|
self.assertEqual(self.ledger(), quota_before)
|
|
self.assertEqual({p: p.read_bytes() for p in frozen}, frozen)
|
|
self.assertEqual(self.quota.request(scope, 'completed')[0], 409)
|
|
self.assertEqual(len(provider.requests), 2)
|
|
self.assertEqual(list(store.root.glob('.CURRENT-*')), [])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|