298 lines
15 KiB
Python
298 lines
15 KiB
Python
"""Filesystem publication faults; source/gate validation lives in trajectory tests."""
|
|
import contextlib
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from export.publication import PublicationError, Releases, build
|
|
|
|
|
|
class PublicationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temp.name) / 'datasets'
|
|
self.store = Releases(self.root)
|
|
|
|
def tearDown(self):
|
|
for folder, _, _ in os.walk(self.temp.name):
|
|
Path(folder).chmod(0o755)
|
|
self.temp.cleanup()
|
|
|
|
def seal(self, rules='rules-a', suffix='', store=None, accepted=1):
|
|
return (store or self.store).seal(
|
|
{name + '.jsonl': (b'{"id":"sample"}\n' if name == 'sft' else b'')
|
|
for name in ('sft', 'retained', 'lineage', 'annotations')},
|
|
rules={'rules_version': rules}, source_manifest={'fixture': suffix},
|
|
validation={'accepted': accepted, 'checks': {'source_replay': True, 'disposition': True}},
|
|
code_sha256={'parser': 'a' * 64})['dataset_version']
|
|
|
|
def switch(self, version, revision=0, event='event-1', store=None, rollback=False):
|
|
return (store or self.store).switch(version, expected_revision=revision,
|
|
event_id=event, reason='test switch', rollback=rollback)
|
|
|
|
def test_immutable_version_and_rules_rollback(self):
|
|
a, b = self.seal(), self.seal('rules-b')
|
|
self.assertNotEqual(a, b)
|
|
self.assertEqual(a, self.seal())
|
|
self.switch(a)
|
|
self.switch(b, 1, 'event-2')
|
|
self.switch(a, 2, 'event-3', rollback=True)
|
|
current = self.store.current()
|
|
self.assertEqual(current['manifest']['rules_version'], 'rules-a')
|
|
self.assertEqual(current['pointer']['revision'], 3)
|
|
self.assertEqual((self.root / a / 'sft.jsonl').stat().st_mode & 0o222, 0)
|
|
|
|
def test_zero_accepted_refuses_build(self):
|
|
with self.assertRaisesRegex(PublicationError, 'NO_ACCEPTED_SAMPLES'):
|
|
self.seal(accepted=0)
|
|
self.assertIsNone(self.store.current())
|
|
|
|
def test_torn_final_audit_recovers_same_commit_but_middle_corruption_fails(self):
|
|
a = self.seal()
|
|
def fault(phase):
|
|
if phase == 'after_replace': raise OSError('crash')
|
|
with self.assertRaises(PublicationError):
|
|
self.switch(a, store=Releases(self.root, fault=fault))
|
|
audit = self.root / 'publication-events.jsonl'
|
|
with audit.open('ab') as stream: stream.write(b'{"event":"COM')
|
|
self.assertEqual(self.store.status('event-1')['state'], 'COMMIT')
|
|
self.assertEqual(self.store.status('event-1')['state'], 'COMMIT')
|
|
self.assertEqual(len(list(self.root.glob('.audit-tail-*'))), 1)
|
|
audit.write_bytes(audit.read_bytes() + b'{"event":"COM\n')
|
|
with self.assertRaisesRegex(PublicationError, 'AUDIT_INCOMPLETE'):
|
|
self.store.status('event-1')
|
|
|
|
def test_cas_two_publishers_only_one_wins(self):
|
|
a, b = self.seal(), self.seal('rules-b')
|
|
result = []
|
|
barrier = threading.Barrier(2)
|
|
def publish(version, event):
|
|
barrier.wait()
|
|
try:
|
|
result.append(self.switch(version, event=event)['state'])
|
|
except PublicationError as error:
|
|
result.append(error.reason)
|
|
threads = [threading.Thread(target=publish, args=(a, 'a')), threading.Thread(target=publish, args=(b, 'b'))]
|
|
for thread in threads: thread.start()
|
|
for thread in threads: thread.join(5)
|
|
self.assertEqual(sorted(result), ['COMMIT', 'PUBLISH_CONFLICT'])
|
|
self.assertEqual(self.store.current()['pointer']['revision'], 1)
|
|
|
|
def test_each_build_write_fault_leaves_pointer_unchanged_and_no_staging(self):
|
|
old = self.seal(); self.switch(old)
|
|
for phase in ('write:sft.jsonl', 'write:rules.json', 'write:dataset-manifest.json',
|
|
'write:acceptance.json', 'write:release.json', 'ready'):
|
|
def fault(actual, expected=phase):
|
|
if actual == expected: raise OSError('injected')
|
|
with self.assertRaises(OSError):
|
|
self.seal('new-' + phase.replace(':', '-'), store=Releases(self.root, fault=fault))
|
|
self.assertEqual(self.store.current()['pointer']['dataset_version'], old)
|
|
self.assertEqual(list(self.root.glob('.building-*')), [])
|
|
|
|
def test_precommit_failures_keep_old_pointer(self):
|
|
old, new = self.seal(), self.seal('rules-b'); self.switch(old)
|
|
for phase in ('after_intent', 'before_replace'):
|
|
def fault(actual, expected=phase):
|
|
if actual == expected: raise OSError('injected')
|
|
with self.assertRaises(OSError):
|
|
self.switch(new, 1, phase, store=Releases(self.root, fault=fault))
|
|
self.assertEqual(self.store.status(phase)['state'], 'ABORT')
|
|
self.assertEqual(self.store.current()['pointer']['dataset_version'], old)
|
|
|
|
def test_postcommit_unknown_same_event_converges_and_blocks_new_event(self):
|
|
old, new = self.seal(), self.seal('rules-b'); self.switch(old)
|
|
def fault(phase):
|
|
if phase == 'after_replace': raise OSError('response lost')
|
|
with self.assertRaisesRegex(PublicationError, 'COMMIT_UNKNOWN'):
|
|
self.switch(new, 1, 'unknown', store=Releases(self.root, fault=fault))
|
|
with self.assertRaisesRegex(PublicationError, 'COMMIT_UNKNOWN'):
|
|
self.switch(old, 2, 'must-wait')
|
|
self.assertEqual(self.store.status('unknown')['state'], 'COMMIT')
|
|
self.assertEqual(self.switch(new, 1, 'unknown')['state'], 'COMMIT')
|
|
self.assertEqual(self.store.current()['pointer']['revision'], 2)
|
|
|
|
def test_response_lost_after_audit_commit_is_idempotent(self):
|
|
version = self.seal()
|
|
def fault(phase):
|
|
if phase == 'after_commit': raise OSError('response lost')
|
|
with self.assertRaisesRegex(PublicationError, 'COMMIT_UNKNOWN'):
|
|
self.switch(version, store=Releases(self.root, fault=fault))
|
|
self.assertEqual(self.switch(version)['pointer']['revision'], 1)
|
|
|
|
def test_revoked_current_not_served_and_cannot_rollback(self):
|
|
a, b = self.seal(), self.seal('rules-b'); self.switch(a)
|
|
self.store.revoke(a, 'known missing prefix')
|
|
with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'):
|
|
self.store.current()
|
|
self.switch(b, 1, 'event-2')
|
|
with self.assertRaisesRegex(PublicationError, 'VERSION_REVOKED'):
|
|
self.switch(a, 2, 'event-3', rollback=True)
|
|
self.assertEqual(self.store.current()['pointer']['dataset_version'], b)
|
|
|
|
def test_unpublished_rollback_and_payload_tamper_fail(self):
|
|
a, b = self.seal(), self.seal('rules-b'); self.switch(a)
|
|
with self.assertRaisesRegex(PublicationError, 'NO_ROLLBACK_TARGET'):
|
|
self.switch(b, 1, 'event-2', rollback=True)
|
|
path = self.root / a / 'sft.jsonl'
|
|
path.chmod(0o644); path.write_bytes(b'altered')
|
|
with self.assertRaisesRegex(PublicationError, 'ARTIFACT_HASH_MISMATCH'):
|
|
self.store.current()
|
|
|
|
def test_event_id_cannot_be_reused_for_other_request(self):
|
|
a, b = self.seal(), self.seal('rules-b'); self.switch(a)
|
|
with self.assertRaisesRegex(PublicationError, 'EVENT_ID_CONFLICT'):
|
|
self.switch(b)
|
|
|
|
|
|
class BuildTests(unittest.TestCase):
|
|
def setUp(self):
|
|
from export.test_materialization import MaterializationTests
|
|
self.fixture = MaterializationTests()
|
|
self.fixture.setUp()
|
|
self.root = self.fixture.root
|
|
self.rules = self.root / 'rules.json'
|
|
self.rules.write_bytes((Path(__file__).resolve().parents[1] / 'docs/plans/sft-collection-v1/rules/sft-preserve-v1.json').read_bytes())
|
|
|
|
def tearDown(self):
|
|
for folder, _, _ in os.walk(self.root):
|
|
Path(folder).chmod(0o755)
|
|
self.fixture.tearDown()
|
|
|
|
def materialize(self):
|
|
from export import to_agentrl
|
|
return to_agentrl.materialize(['r'], 'candidate', purpose='all', min_agent_version=0)
|
|
|
|
def build(self):
|
|
return build(self.root / 'datasets/candidate', self.root / 'releases', self.rules, self.root)
|
|
|
|
def test_real_export_replayed_before_seal(self):
|
|
self.fixture.add()
|
|
self.fixture.add(1, terminal='FAIL')
|
|
self.materialize()
|
|
release = self.build()
|
|
store = Releases(self.root / 'releases')
|
|
store.switch(release['dataset_version'], expected_revision=0, event_id='first', reason='fixture')
|
|
acceptance = store.current()['acceptance']
|
|
self.assertEqual((acceptance['accepted'], acceptance['retained'], acceptance['inputs']), (1, 1, 2))
|
|
|
|
def test_saved_pass_cannot_hide_modified_payload(self):
|
|
import hashlib
|
|
self.fixture.add(); self.materialize()
|
|
folder = self.root / 'datasets/candidate'
|
|
record = json.loads((folder / 'sft.jsonl').read_text())
|
|
record['messages'][-1]['content'] = 'invented success'
|
|
data = (json.dumps(record) + '\n').encode()
|
|
(folder / 'sft.jsonl').write_bytes(data)
|
|
manifest = json.loads((folder / 'manifest.json').read_text())
|
|
manifest['files']['sft.jsonl'].update(bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
|
|
(folder / 'manifest.json').write_text(json.dumps(manifest))
|
|
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
|
|
self.build()
|
|
|
|
def test_hard_gate_change_after_export_blocks_build(self):
|
|
rd = self.fixture.add(); self.materialize()
|
|
(rd / 'artifacts/j.hbm').write_bytes(b'changed HBM')
|
|
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
|
|
self.build()
|
|
|
|
def test_annotation_added_after_materialization_blocks_build(self):
|
|
from export.to_agentrl import validate_exported_record
|
|
rd = self.fixture.add()
|
|
self.materialize()
|
|
record = json.loads((self.root / 'datasets/candidate/sft.jsonl').read_text())
|
|
before = {path: path.read_bytes() for path in rd.rglob('*') if path.is_file()}
|
|
for note, reason in (
|
|
('rollout#0 contaminated: operator_abandoned_unknown', 'contaminated'),
|
|
('rollout#0 stalled_killed by operator', 'stalled_killed')):
|
|
with self.subTest(reason=reason):
|
|
self.fixture.con.execute('DELETE FROM annotations')
|
|
self.fixture.con.execute('INSERT INTO annotations(run_id,job_id,note) VALUES(?,?,?)', ('r', 'j', note))
|
|
self.fixture.con.commit()
|
|
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
|
|
self.build()
|
|
check = validate_exported_record(self.root, record)['checks']['DB_SOURCE_ANNOTATIONS']
|
|
self.assertEqual(check, {'pass': False, 'reasons': [reason]})
|
|
self.assertEqual(before, {path: path.read_bytes() for path in before})
|
|
self.assertFalse((self.root / 'releases').exists())
|
|
|
|
def test_unrelated_annotations_do_not_block_build(self):
|
|
self.fixture.add(); self.materialize()
|
|
self.fixture.con.executemany('INSERT INTO annotations(run_id,job_id,note) VALUES(?,?,?)', [
|
|
('other-run', 'j', 'rollout#0 contaminated: unrelated run'),
|
|
('r', 'other-job', 'rollout#0 contaminated: unrelated job'),
|
|
('r', 'j', 'rollout#1 contaminated: unrelated attempt'),
|
|
('r', 'j', 'rollout#0 ordinary operator observation'),
|
|
])
|
|
self.fixture.con.commit()
|
|
self.assertIn('dataset_version', self.build())
|
|
|
|
def test_run_wide_annotation_blocks_candidate(self):
|
|
self.fixture.add(); self.materialize()
|
|
self.fixture.con.execute('INSERT INTO annotations(run_id,job_id,note) VALUES(?,?,?)',
|
|
('r', None, 'contaminated: whole run'))
|
|
self.fixture.con.commit()
|
|
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
|
|
self.build()
|
|
|
|
def test_annotation_db_error_fails_closed(self):
|
|
from export.to_agentrl import validate_exported_record
|
|
self.fixture.add(); self.materialize()
|
|
record = json.loads((self.root / 'datasets/candidate/sft.jsonl').read_text())
|
|
self.fixture.con.execute('DROP TABLE annotations'); self.fixture.con.commit()
|
|
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
|
|
self.build()
|
|
check = validate_exported_record(self.root, record)['checks']['DB_SOURCE_ANNOTATIONS']
|
|
self.assertEqual(check, {'pass': False, 'reasons': ['DB_ANNOTATIONS_UNAVAILABLE']})
|
|
|
|
def test_build_uses_explicit_source_database_not_process_environment(self):
|
|
self.fixture.add(); self.materialize()
|
|
elsewhere = self.root / 'other-state'
|
|
elsewhere.mkdir()
|
|
with contextlib.closing(sqlite3.connect(elsewhere / 'jobs.sqlite')) as con:
|
|
con.executescript('CREATE TABLE rollouts(run_id,job_id,kk,state); '
|
|
"INSERT INTO rollouts VALUES('r','j',0,'PAUSED_INFRA'); "
|
|
'CREATE TABLE annotations(run_id,job_id,note); '
|
|
"INSERT INTO annotations VALUES('r','j','contaminated: wrong database');")
|
|
con.commit()
|
|
with patch.dict(os.environ, {'BENCH_STATE_DIR': str(elsewhere)}):
|
|
self.assertIn('dataset_version', self.build())
|
|
self.fixture.con.execute('INSERT INTO annotations(run_id,job_id,note) VALUES(?,?,?)',
|
|
('r', 'j', 'rollout#0 contaminated: actual source'))
|
|
self.fixture.con.commit()
|
|
with contextlib.closing(sqlite3.connect(elsewhere / 'jobs.sqlite')) as con:
|
|
con.execute('DELETE FROM annotations')
|
|
con.execute("UPDATE rollouts SET state='PASS'")
|
|
con.commit()
|
|
with patch.dict(os.environ, {'BENCH_STATE_DIR': str(elsewhere)}):
|
|
with self.assertRaisesRegex(PublicationError, 'SOURCE_REPLAY_FAILED'):
|
|
self.build()
|
|
|
|
def test_unexecuted_rules_and_changed_semantics_refused(self):
|
|
self.fixture.add(); self.materialize()
|
|
rules = json.loads(self.rules.read_text())
|
|
for changed in ({**rules, 'rules_version': 'UNEXECUTED'},
|
|
{**rules, 'transform': {**rules['transform'], 'new_truncation': True}}):
|
|
self.rules.write_text(json.dumps(changed))
|
|
with self.assertRaisesRegex(PublicationError, 'UNIMPLEMENTED_RULES'):
|
|
self.build()
|
|
|
|
def test_retained_frozen_hard_gate_sources_must_still_match(self):
|
|
self.fixture.add(); rd = self.fixture.add(1, terminal='FAIL'); self.materialize()
|
|
(rd / 'claim.json').write_text('{"changed":true}')
|
|
with self.assertRaisesRegex(PublicationError, 'SOURCE_CHANGED'):
|
|
self.build()
|
|
|
|
def test_all_failed_batch_cannot_publish(self):
|
|
self.fixture.add(terminal='FAIL'); self.materialize()
|
|
with self.assertRaisesRegex(PublicationError, 'NO_ACCEPTED_SAMPLES'):
|
|
self.build()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|