data-gen-bpu/export/test_publication.py

298 lines
15 KiB
Python

"""Filesystem publication faults; source/gate validation lives in trajectory tests."""
import contextlib
import json
import os
import sqlite3
from pathlib import Path
import tempfile
import threading
import unittest
from unittest.mock import patch
from export.publication import PublicationError, Releases, build
class PublicationTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.root = Path(self.temp.name) / 'datasets'
self.store = Releases(self.root)
def tearDown(self):
for folder, _, _ in os.walk(self.temp.name):
Path(folder).chmod(0o755)
self.temp.cleanup()
def seal(self, rules='rules-a', suffix='', store=None, accepted=1):
return (store or self.store).seal(
{name + '.jsonl': (b'{"id":"sample"}\n' if name == 'sft' else b'')
for name in ('sft', 'retained', 'lineage', 'annotations')},
rules={'rules_version': rules}, source_manifest={'fixture': suffix},
validation={'accepted': accepted, 'checks': {'source_replay': True, 'disposition': True}},
code_sha256={'parser': 'a' * 64})['dataset_version']
def switch(self, version, revision=0, event='event-1', store=None, rollback=False):
return (store or self.store).switch(version, expected_revision=revision,
event_id=event, reason='test switch', rollback=rollback)
def test_immutable_version_and_rules_rollback(self):
a, b = self.seal(), self.seal('rules-b')
self.assertNotEqual(a, b)
self.assertEqual(a, self.seal())
self.switch(a)
self.switch(b, 1, 'event-2')
self.switch(a, 2, 'event-3', rollback=True)
current = self.store.current()
self.assertEqual(current['manifest']['rules_version'], 'rules-a')
self.assertEqual(current['pointer']['revision'], 3)
self.assertEqual((self.root / a / 'sft.jsonl').stat().st_mode & 0o222, 0)
def test_zero_accepted_refuses_build(self):
with self.assertRaisesRegex(PublicationError, 'NO_ACCEPTED_SAMPLES'):
self.seal(accepted=0)
self.assertIsNone(self.store.current())
def test_torn_final_audit_recovers_same_commit_but_middle_corruption_fails(self):
a = self.seal()
def fault(phase):
if phase == 'after_replace': raise OSError('crash')
with self.assertRaises(PublicationError):
self.switch(a, store=Releases(self.root, fault=fault))
audit = self.root / 'publication-events.jsonl'
with audit.open('ab') as stream: stream.write(b'{"event":"COM')
self.assertEqual(self.store.status('event-1')['state'], 'COMMIT')
self.assertEqual(self.store.status('event-1')['state'], 'COMMIT')
self.assertEqual(len(list(self.root.glob('.audit-tail-*'))), 1)
audit.write_bytes(audit.read_bytes() + b'{"event":"COM\n')
with self.assertRaisesRegex(PublicationError, 'AUDIT_INCOMPLETE'):
self.store.status('event-1')
def test_cas_two_publishers_only_one_wins(self):
a, b = self.seal(), self.seal('rules-b')
result = []
barrier = threading.Barrier(2)
def publish(version, event):
barrier.wait()
try:
result.append(self.switch(version, event=event)['state'])
except PublicationError as error:
result.append(error.reason)
threads = [threading.Thread(target=publish, args=(a, 'a')), threading.Thread(target=publish, args=(b, 'b'))]
for thread in threads: thread.start()
for thread in threads: thread.join(5)
self.assertEqual(sorted(result), ['COMMIT', 'PUBLISH_CONFLICT'])
self.assertEqual(self.store.current()['pointer']['revision'], 1)
def test_each_build_write_fault_leaves_pointer_unchanged_and_no_staging(self):
old = self.seal(); self.switch(old)
for phase in ('write:sft.jsonl', 'write:rules.json', 'write:dataset-manifest.json',
'write:acceptance.json', 'write:release.json', 'ready'):
def fault(actual, expected=phase):
if actual == expected: raise OSError('injected')
with self.assertRaises(OSError):
self.seal('new-' + phase.replace(':', '-'), store=Releases(self.root, fault=fault))
self.assertEqual(self.store.current()['pointer']['dataset_version'], old)
self.assertEqual(list(self.root.glob('.building-*')), [])
def test_precommit_failures_keep_old_pointer(self):
old, new = self.seal(), self.seal('rules-b'); self.switch(old)
for phase in ('after_intent', 'before_replace'):
def fault(actual, expected=phase):
if actual == expected: raise OSError('injected')
with self.assertRaises(OSError):
self.switch(new, 1, phase, store=Releases(self.root, fault=fault))
self.assertEqual(self.store.status(phase)['state'], 'ABORT')
self.assertEqual(self.store.current()['pointer']['dataset_version'], old)
def test_postcommit_unknown_same_event_converges_and_blocks_new_event(self):
old, new = self.seal(), self.seal('rules-b'); self.switch(old)
def fault(phase):
if phase == 'after_replace': raise OSError('response lost')
with self.assertRaisesRegex(PublicationError, 'COMMIT_UNKNOWN'):
self.switch(new, 1, 'unknown', store=Releases(self.root, fault=fault))
with self.assertRaisesRegex(PublicationError, 'COMMIT_UNKNOWN'):
self.switch(old, 2, 'must-wait')
self.assertEqual(self.store.status('unknown')['state'], 'COMMIT')
self.assertEqual(self.switch(new, 1, 'unknown')['state'], 'COMMIT')
self.assertEqual(self.store.current()['pointer']['revision'], 2)
def test_response_lost_after_audit_commit_is_idempotent(self):
version = self.seal()
def fault(phase):
if phase == 'after_commit': raise OSError('response lost')
with self.assertRaisesRegex(PublicationError, 'COMMIT_UNKNOWN'):
self.switch(version, store=Releases(self.root, fault=fault))
self.assertEqual(self.switch(version)['pointer']['revision'], 1)
def test_revoked_current_not_served_and_cannot_rollback(self):
a, b = self.seal(), self.seal('rules-b'); self.switch(a)
self.store.revoke(a, 'known missing prefix')
with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'):
self.store.current()
self.switch(b, 1, 'event-2')
with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'):
self.switch(a, 2, 'event-3', rollback=True)
self.assertEqual(self.store.current()['pointer']['dataset_version'], b)
def test_unpublished_rollback_and_payload_tamper_fail(self):
a, b = self.seal(), self.seal('rules-b'); self.switch(a)
with self.assertRaisesRegex(PublicationError, 'NO_ROLLBACK_TARGET'):
self.switch(b, 1, 'event-2', rollback=True)
path = self.root / a / 'sft.jsonl'
path.chmod(0o644); path.write_bytes(b'altered')
with self.assertRaisesRegex(PublicationError, 'ARTIFACT_HASH_MISMATCH'):
self.store.current()
def test_event_id_cannot_be_reused_for_other_request(self):
a, b = self.seal(), self.seal('rules-b'); self.switch(a)
with self.assertRaisesRegex(PublicationError, 'EVENT_ID_CONFLICT'):
self.switch(b)
class BuildTests(unittest.TestCase):
def setUp(self):
from export.test_materialization import MaterializationTests
self.fixture = MaterializationTests()
self.fixture.setUp()
self.root = self.fixture.root
self.rules = self.root / 'rules.json'
self.rules.write_bytes((Path(__file__).resolve().parents[1] / 'docs/plans/sft-collection-v1/rules/sft-preserve-v1.json').read_bytes())
def tearDown(self):
for folder, _, _ in os.walk(self.root):
Path(folder).chmod(0o755)
self.fixture.tearDown()
def materialize(self):
from export import to_agentrl
return to_agentrl.materialize(['r'], 'candidate', purpose='all', min_agent_version=0)
def build(self):
return build(self.root / 'datasets/candidate', self.root / 'releases', self.rules, self.root)
def test_real_export_replayed_before_seal(self):
self.fixture.add()
self.fixture.add(1, terminal='FAIL')
self.materialize()
release = self.build()
store = Releases(self.root / 'releases')
store.switch(release['dataset_version'], expected_revision=0, event_id='first', reason='fixture')
acceptance = store.current()['acceptance']
self.assertEqual((acceptance['accepted'], acceptance['retained'], acceptance['inputs']), (1, 1, 2))
def test_saved_pass_cannot_hide_modified_payload(self):
import hashlib
self.fixture.add(); self.materialize()
folder = self.root / 'datasets/candidate'
record = json.loads((folder / 'sft.jsonl').read_text())
record['messages'][-1]['content'] = 'invented success'
data = (json.dumps(record) + '\n').encode()
(folder / 'sft.jsonl').write_bytes(data)
manifest = json.loads((folder / 'manifest.json').read_text())
manifest['files']['sft.jsonl'].update(bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
(folder / 'manifest.json').write_text(json.dumps(manifest))
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
self.build()
def test_hard_gate_change_after_export_blocks_build(self):
rd = self.fixture.add(); self.materialize()
(rd / 'artifacts/j.hbm').write_bytes(b'changed HBM')
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
self.build()
def test_annotation_added_after_materialization_blocks_build(self):
from export.to_agentrl import validate_exported_record
rd = self.fixture.add()
self.materialize()
record = json.loads((self.root / 'datasets/candidate/sft.jsonl').read_text())
before = {path: path.read_bytes() for path in rd.rglob('*') if path.is_file()}
for note, reason in (
('rollout#0 contaminated: operator_abandoned_unknown', 'contaminated'),
('rollout#0 stalled_killed by operator', 'stalled_killed')):
with self.subTest(reason=reason):
self.fixture.con.execute('DELETE FROM annotations')
self.fixture.con.execute('INSERT INTO annotations(run_id,job_id,note) VALUES(?,?,?)', ('r', 'j', note))
self.fixture.con.commit()
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
self.build()
check = validate_exported_record(self.root, record)['checks']['DB_SOURCE_ANNOTATIONS']
self.assertEqual(check, {'pass': False, 'reasons': [reason]})
self.assertEqual(before, {path: path.read_bytes() for path in before})
self.assertFalse((self.root / 'releases').exists())
def test_unrelated_annotations_do_not_block_build(self):
self.fixture.add(); self.materialize()
self.fixture.con.executemany('INSERT INTO annotations(run_id,job_id,note) VALUES(?,?,?)', [
('other-run', 'j', 'rollout#0 contaminated: unrelated run'),
('r', 'other-job', 'rollout#0 contaminated: unrelated job'),
('r', 'j', 'rollout#1 contaminated: unrelated attempt'),
('r', 'j', 'rollout#0 ordinary operator observation'),
])
self.fixture.con.commit()
self.assertIn('dataset_version', self.build())
def test_run_wide_annotation_blocks_candidate(self):
self.fixture.add(); self.materialize()
self.fixture.con.execute('INSERT INTO annotations(run_id,job_id,note) VALUES(?,?,?)',
('r', None, 'contaminated: whole run'))
self.fixture.con.commit()
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
self.build()
def test_annotation_db_error_fails_closed(self):
from export.to_agentrl import validate_exported_record
self.fixture.add(); self.materialize()
record = json.loads((self.root / 'datasets/candidate/sft.jsonl').read_text())
self.fixture.con.execute('DROP TABLE annotations'); self.fixture.con.commit()
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
self.build()
check = validate_exported_record(self.root, record)['checks']['DB_SOURCE_ANNOTATIONS']
self.assertEqual(check, {'pass': False, 'reasons': ['DB_ANNOTATIONS_UNAVAILABLE']})
def test_build_uses_explicit_source_database_not_process_environment(self):
self.fixture.add(); self.materialize()
elsewhere = self.root / 'other-state'
elsewhere.mkdir()
with contextlib.closing(sqlite3.connect(elsewhere / 'jobs.sqlite')) as con:
con.executescript('CREATE TABLE rollouts(run_id,job_id,kk,state); '
"INSERT INTO rollouts VALUES('r','j',0,'PAUSED_INFRA'); "
'CREATE TABLE annotations(run_id,job_id,note); '
"INSERT INTO annotations VALUES('r','j','contaminated: wrong database');")
con.commit()
with patch.dict(os.environ, {'BENCH_STATE_DIR': str(elsewhere)}):
self.assertIn('dataset_version', self.build())
self.fixture.con.execute('INSERT INTO annotations(run_id,job_id,note) VALUES(?,?,?)',
('r', 'j', 'rollout#0 contaminated: actual source'))
self.fixture.con.commit()
with contextlib.closing(sqlite3.connect(elsewhere / 'jobs.sqlite')) as con:
con.execute('DELETE FROM annotations')
con.execute("UPDATE rollouts SET state='PASS'")
con.commit()
with patch.dict(os.environ, {'BENCH_STATE_DIR': str(elsewhere)}):
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
self.build()
def test_unexecuted_rules_and_changed_semantics_refused(self):
self.fixture.add(); self.materialize()
rules = json.loads(self.rules.read_text())
for changed in ({**rules, 'rules_version': 'UNEXECUTED'},
{**rules, 'transform': {**rules['transform'], 'new_truncation': True}}):
self.rules.write_text(json.dumps(changed))
with self.assertRaisesRegex(PublicationError, 'UNIMPLEMENTED_RULES'):
self.build()
def test_retained_frozen_hard_gate_sources_must_still_match(self):
self.fixture.add(); rd = self.fixture.add(1, terminal='FAIL'); self.materialize()
(rd / 'claim.json').write_text('{"changed":true}')
with self.assertRaisesRegex(PublicationError, 'SOURCE_CHANGED'):
self.build()
def test_all_failed_batch_cannot_publish(self):
self.fixture.add(terminal='FAIL'); self.materialize()
with self.assertRaisesRegex(PublicationError, 'NO_ACCEPTED_SAMPLES'):
self.build()
if __name__ == '__main__':
unittest.main()