data-gen-bpu/export/publication.py

499 lines
25 KiB
Python

"""Immutable dataset releases and one crash-recoverable, revocable pointer.
Publication never changes rollout evidence, execution state or quota ledgers.
All mutations share one advisory lock; readers pin one verified release.
"""
from __future__ import annotations
from contextlib import contextmanager
import fcntl
import hashlib
import json
import os
from pathlib import Path
import re
import tempfile
import time
class PublicationError(Exception):
def __init__(self, reason, code=2):
self.reason, self.code = reason, code
super().__init__(reason)
def _bytes(value):
return (json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(',', ':')) + '\n').encode()
def _sha(data):
return hashlib.sha256(data).hexdigest()
def _name(value):
if not isinstance(value, str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,159}', value):
raise PublicationError('INVALID_ID')
return value
def _read(path, default=None):
if path.is_symlink():
raise PublicationError('SYMLINK_REJECTED')
try:
return json.loads(path.read_bytes())
except FileNotFoundError:
return default
except (ValueError, OSError) as error:
raise PublicationError('INVALID_JSON') from error
def _sync_dir(path):
fd = os.open(path, os.O_RDONLY)
try:
os.fsync(fd)
finally:
os.close(fd)
def _write(path, data):
with path.open('xb') as stream:
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
def _replace_json(path, value):
with tempfile.NamedTemporaryFile(prefix='.' + path.name + '-', dir=path.parent, delete=False) as stream:
temp = Path(stream.name)
stream.write(_bytes(value)); stream.flush(); os.fsync(stream.fileno())
try:
os.replace(temp, path)
_sync_dir(path.parent)
finally:
temp.unlink(missing_ok=True)
class Releases:
def __init__(self, root, *, fault=None):
self.root = Path(root)
if self.root.is_symlink():
raise PublicationError('SYMLINK_REJECTED')
self.fault = fault or (lambda phase: None)
@contextmanager
def _lock(self):
self.root.mkdir(parents=True, exist_ok=True)
path = self.root / '.publication.lock'
if path.is_symlink():
raise PublicationError('SYMLINK_REJECTED')
with path.open('a') as stream:
fcntl.flock(stream, fcntl.LOCK_EX)
yield
def _events(self):
path = self.root / 'publication-events.jsonl'
if not path.exists():
return []
if path.is_symlink():
raise PublicationError('SYMLINK_REJECTED')
data = path.read_bytes()
result, offset = [], 0
for line in data.splitlines(keepends=True):
try:
event = json.loads(line)
if not isinstance(event, dict) or not isinstance(event.get('event'), str):
raise ValueError('invalid event')
result.append(event)
except ValueError as error:
# Called only under the publication lock. Repair only an
# unterminated final write; interior corruption is never hidden.
if line.endswith(b'\n') or offset + len(line) != len(data):
raise PublicationError('AUDIT_INCOMPLETE', 5) from error
digest = _sha(line)
backup = self.root / ('.audit-tail-' + digest)
if not backup.exists():
_write(backup, line)
_sync_dir(self.root)
with path.open('r+b') as stream:
stream.truncate(offset); stream.flush(); os.fsync(stream.fileno())
event = {'event': 'AUDIT_TAIL_RECOVERED', 'sha256': digest,
'bytes': len(line), 'offset': offset}
self._log(event)
result.append(event)
else:
if not line.endswith(b'\n'):
with path.open('ab') as stream:
stream.write(b'\n'); stream.flush(); os.fsync(stream.fileno())
offset += len(line)
return result
def _log(self, event):
path = self.root / 'publication-events.jsonl'
if path.is_symlink():
raise PublicationError('SYMLINK_REJECTED')
with path.open('ab') as stream:
stream.write(_bytes({**event, 'at': time.time()}))
stream.flush(); os.fsync(stream.fileno())
_sync_dir(self.root)
def _revocations(self):
return _read(self.root / 'revocations.json', {'epoch': 0, 'versions': {}})
def _verify(self, version):
folder = self.root / _name(version)
if folder.is_symlink() or not folder.is_dir():
raise PublicationError('RELEASE_MISSING')
release = _read(folder / 'release.json')
if not release or release.get('dataset_version') != version:
raise PublicationError('INVALID_RELEASE')
for name, key in [('dataset-manifest.json', 'manifest_sha256'), ('acceptance.json', 'acceptance_sha256')]:
path = folder / name
if path.is_symlink() or not path.is_file() or _sha(path.read_bytes()) != release.get(key):
raise PublicationError('RELEASE_HASH_MISMATCH')
manifest = _read(folder / 'dataset-manifest.json')
acceptance = _read(folder / 'acceptance.json')
if (manifest.get('dataset_version') != version
or acceptance.get('manifest_sha256') != release['manifest_sha256']
or acceptance.get('integrity_floor') != 1
or acceptance.get('accepted', 0) < 1
or not acceptance.get('checks')
or any(value is not True for value in acceptance['checks'].values())):
raise PublicationError('ACCEPTANCE_FAILED')
for name, info in manifest.get('files', {}).items():
path = folder / _name(name)
if path.is_symlink() or not path.is_file():
raise PublicationError('ARTIFACT_MISSING')
content = path.read_bytes()
if _sha(content) != info.get('sha256') or len(content) != info.get('bytes'):
raise PublicationError('ARTIFACT_HASH_MISMATCH')
required = {'rules.json', 'source-manifest.json', 'sft.jsonl', 'retained.jsonl', 'lineage.jsonl', 'annotations.jsonl'}
if not required <= manifest.get('files', {}).keys():
raise PublicationError('ARTIFACT_MISSING')
rules = _read(folder / 'rules.json')
if (manifest.get('rules_sha256') != _sha((folder / 'rules.json').read_bytes())
or manifest.get('rules_version') != rules.get('rules_version')
or not manifest.get('code_sha256')):
raise PublicationError('VERSION_BINDING_MISSING')
return release, manifest, acceptance
def seal(self, artifacts, *, rules, source_manifest, validation, code_sha256):
"""Freeze a completed independently validated build; no pointer switch.
The builder supplies validation from a fresh source replay, rather than
trusting each exported record's saved PASS. Public CLI never accepts a
caller-authored validation JSON in place of running that validator.
"""
if not validation.get('checks') or any(v is not True for v in validation['checks'].values()):
raise PublicationError('ACCEPTANCE_FAILED')
if validation.get('accepted', 0) < 1:
raise PublicationError('NO_ACCEPTED_SAMPLES')
if not rules.get('rules_version') or not code_sha256:
raise PublicationError('VERSION_BINDING_MISSING')
files = {name: bytes(data) for name, data in artifacts.items()}
files.update({'rules.json': _bytes(rules), 'source-manifest.json': _bytes(source_manifest)})
required = {'sft.jsonl', 'retained.jsonl', 'lineage.jsonl', 'annotations.jsonl'}
if not required <= files.keys() or any(_name(name) != name for name in files):
raise PublicationError('ARTIFACT_MISSING')
metadata = {name: {'sha256': _sha(data), 'bytes': len(data)} for name, data in files.items()}
basis = {'schema': 'bench.dataset-release/1', 'rules_version': rules['rules_version'],
'rules_sha256': _sha(files['rules.json']), 'source_sha256': _sha(files['source-manifest.json']),
'code_sha256': code_sha256, 'files': metadata}
version = 'sft-' + _sha(_bytes(basis))[:24]
manifest = {**basis, 'dataset_version': version}
acceptance = {**validation, 'integrity_floor': 1, 'manifest_sha256': _sha(_bytes(manifest))}
release = {'schema': 'bench.release/1', 'dataset_version': version,
'manifest_sha256': _sha(_bytes(manifest)), 'acceptance_sha256': _sha(_bytes(acceptance))}
files.update({'dataset-manifest.json': _bytes(manifest), 'acceptance.json': _bytes(acceptance), 'release.json': _bytes(release)})
with self._lock():
target = self.root / version
if target.exists():
old, _, _ = self._verify(version)
if old != release:
raise PublicationError('VERSION_COLLISION', 4)
return release
with tempfile.TemporaryDirectory(prefix='.building-', dir=self.root) as temp:
work = Path(temp) / 'release'
work.mkdir()
for name, data in files.items():
_write(work / name, data)
self.fault('write:' + name)
_sync_dir(work)
self.fault('ready')
for path in work.iterdir():
path.chmod(0o444)
os.rename(work, target)
target.chmod(0o555)
_sync_dir(target)
_sync_dir(self.root)
self._verify(version)
self._log({'event': 'READY', 'dataset_version': version, 'rules_version': rules['rules_version']})
return release
def _status(self, event_id, events=None):
events = self._events() if events is None else events
own = [event for event in events if event.get('event_id') == event_id]
if not own:
return {'event_id': event_id, 'state': 'NOT_FOUND'}
intent = next((event for event in own if event['event'] == 'INTENT'), None)
if not intent:
raise PublicationError('AUDIT_INCOMPLETE', 5)
temporary = intent.get('temporary')
if temporary:
expected = '.CURRENT-' + _sha(event_id.encode())
if temporary != expected:
raise PublicationError('INVALID_TEMPORARY')
path = self.root / temporary
if path.is_file() and not path.is_symlink() and _bytes(intent['pointer']).startswith(path.read_bytes()):
path.unlink()
_sync_dir(self.root)
terminal = next((event for event in reversed(own) if event['event'] in {'COMMIT', 'ABORT'}), None)
if terminal:
return {'event_id': event_id, 'state': terminal['event'], 'pointer': intent['pointer'] if terminal['event'] == 'COMMIT' else None}
pointer = _read(self.root / 'CURRENT.json')
if pointer == intent['pointer']:
self._verify(pointer['dataset_version'])
_sync_dir(self.root)
self._log({'event': 'COMMIT', 'event_id': event_id, 'reconciled': True})
return {'event_id': event_id, 'state': 'COMMIT', 'pointer': pointer}
if pointer != intent['previous']:
raise PublicationError('COMMIT_UNKNOWN', 5)
self._log({'event': 'ABORT', 'event_id': event_id, 'reason': 'POINTER_UNCHANGED'})
return {'event_id': event_id, 'state': 'ABORT', 'pointer': None}
def status(self, event_id):
with self._lock():
return self._status(_name(event_id))
def switch(self, version, *, expected_revision, event_id, reason, rollback=False):
_name(version); _name(event_id)
if not isinstance(expected_revision, int) or expected_revision < 0 or not reason.strip():
raise PublicationError('INVALID_SWITCH_REQUEST')
with self._lock():
events = self._events()
existing = [event for event in events if event.get('event_id') == event_id]
if existing:
intent = next((event for event in existing if event['event'] == 'INTENT'), {})
if (intent.get('pointer', {}).get('dataset_version') != version
or intent.get('expected_revision') != expected_revision
or intent.get('rollback') != rollback or intent.get('reason') != reason):
raise PublicationError('EVENT_ID_CONFLICT', 4)
return self._status(event_id, events)
pending = {e['event_id'] for e in events if e['event'] == 'INTENT'} - {
e['event_id'] for e in events if e['event'] in {'COMMIT', 'ABORT'}}
if pending:
raise PublicationError('COMMIT_UNKNOWN', 5)
release, manifest, _ = self._verify(version)
revoked = self._revocations()
if version in revoked['versions']:
raise PublicationError('VERSION_REVOKED')
if rollback:
committed = {e['event_id'] for e in events if e['event'] == 'COMMIT'}
if not any(e['event'] == 'INTENT' and e['event_id'] in committed
and e['pointer']['dataset_version'] == version for e in events):
raise PublicationError('NO_ROLLBACK_TARGET')
previous = _read(self.root / 'CURRENT.json')
if (previous or {}).get('revision', 0) != expected_revision:
raise PublicationError('PUBLISH_CONFLICT', 4)
pointer = {'schema': 'bench.dataset-pointer/1', 'revision': expected_revision + 1,
'event_id': event_id, 'dataset_version': version,
'rules_version': manifest['rules_version'], 'revocation_epoch': revoked['epoch'],
'release_sha256': _sha(_bytes(release))}
temporary = '.CURRENT-' + _sha(event_id.encode())
self._log({'event': 'INTENT', 'event_id': event_id, 'previous': previous, 'pointer': pointer, 'temporary': temporary,
'expected_revision': expected_revision, 'rollback': rollback, 'reason': reason})
switched = False
temp = None
try:
self.fault('after_intent')
temp = self.root / temporary
_write(temp, _bytes(pointer))
self.fault('before_replace')
os.replace(temp, self.root / 'CURRENT.json')
switched = True
self.fault('after_replace')
_sync_dir(self.root)
self.fault('after_directory_sync')
self._log({'event': 'COMMIT', 'event_id': event_id})
self.fault('after_commit')
return {'event_id': event_id, 'state': 'COMMIT', 'pointer': pointer}
except Exception as error:
if switched:
raise PublicationError('COMMIT_UNKNOWN', 5) from error
self._log({'event': 'ABORT', 'event_id': event_id, 'reason': type(error).__name__})
raise
finally:
if temp is not None:
temp.unlink(missing_ok=True)
def revoke(self, version, reason):
if not reason.strip():
raise PublicationError('REVOCATION_REASON_REQUIRED')
with self._lock():
self._verify(version)
data = self._revocations()
data['epoch'] += 1
data['versions'][version] = {'reason': reason, 'at': time.time()}
_replace_json(self.root / 'revocations.json', data)
self._log({'event': 'REVOKE', 'dataset_version': version, 'epoch': data['epoch'], 'reason': reason})
return data
def current(self):
# Lock only protects the short pointer/revocation read. The selected
# directory is immutable; callers never re-read CURRENT per artifact.
with self._lock():
pointer = _read(self.root / 'CURRENT.json')
if pointer is None:
return None
if pointer['dataset_version'] in self._revocations()['versions']:
raise PublicationError('VERSION_REVOKED')
release, manifest, acceptance = self._verify(pointer['dataset_version'])
if _sha(_bytes(release)) != pointer['release_sha256']:
raise PublicationError('POINTER_HASH_MISMATCH')
return {'pointer': pointer, 'manifest': manifest, 'acceptance': acceptance}
def build(materialized, datasets, rules_path, source_root):
"""Recheck a coherent materialization against its original evidence, then seal.
Only this path is exposed for building through the CLI. Saved PASS fields
are never accepted as a substitute for replay and hard-gate verification.
"""
from collections import defaultdict, Counter
from .to_agentrl import validate_exported_record, _gate_sources
from .trajectory import RULES_VERSION, RULES_SPEC_SHA256
rules = _read(Path(rules_path))
semantic = {k: v for k, v in (rules or {}).items() if k not in ('status', 'purpose')}
if rules is None or rules.get('rules_version') != RULES_VERSION or _sha(_bytes(semantic)) != RULES_SPEC_SHA256:
raise PublicationError('UNIMPLEMENTED_RULES')
folder, source_root = Path(materialized), Path(source_root).resolve()
if folder.is_symlink():
raise PublicationError('SYMLINK_REJECTED')
with (folder / '.export.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_SH)
manifest = _read(folder / 'manifest.json')
if not manifest:
raise PublicationError('MATERIALIZATION_MISSING')
contents = {}
for name, info in manifest.get('files', {}).items():
path = folder / _name(name)
if path.is_symlink() or not path.is_file():
raise PublicationError('ARTIFACT_MISSING')
data = path.read_bytes()
if len(data) != info['bytes'] or _sha(data) != info['sha256']:
raise PublicationError('SOURCE_CHANGED', 3)
contents[name] = data
def rows(name):
try:
return [json.loads(line) for line in contents[name].splitlines()]
except (KeyError, ValueError) as error:
raise PublicationError('INVALID_MATERIALIZATION') from error
def identity(row):
return (row['run_id'], row['job_id'], row['rollout'])
sources, dispositions = rows('source-manifest.jsonl'), rows('disposition.jsonl')
source_ids, disposition_ids = [identity(r) for r in sources], [identity(r) for r in dispositions]
accepted, retained = rows('sft.jsonl'), rows('retained.jsonl')
accepted_ids, retained_ids = [identity(r) for r in accepted], [identity(r) for r in retained]
expected_accepted = [identity(r) for r in dispositions if r['disposition'] == 'accepted']
expected_retained = [identity(r) for r in dispositions if r['disposition'] in {'retained', 'no-usable'}]
if (len(source_ids) != len(set(source_ids)) or len(disposition_ids) != len(set(disposition_ids))
or set(source_ids) != set(disposition_ids)
or len(source_ids) != manifest['stats']['scanned']
or sorted(accepted_ids) != sorted(expected_accepted)
or sorted(retained_ids) != sorted(expected_retained)
or len(expected_accepted) + len(expected_retained) != len(dispositions)):
raise PublicationError('DISPOSITION_MISMATCH')
if not accepted:
raise PublicationError('NO_ACCEPTED_SAMPLES')
grouped = {name: defaultdict(list) for name in ('lineage', 'excluded_observations')}
for name, index in grouped.items():
for row in rows(name + '.jsonl'):
index[identity(row)].append({k: v for k, v in row.items() if k not in ('run_id', 'job_id', 'rollout')})
checks = []
for record in accepted:
if record.get('trajectory', {}).get('rules_version') != RULES_VERSION:
raise PublicationError('RULE_BINDING_MISMATCH')
key = identity(record)
result = validate_exported_record(source_root, record, grouped['lineage'][key], grouped['excluded_observations'][key])
if not result.get('pass'):
raise PublicationError('SOURCE_REPLAY_FAILED')
checks.append({'run_id': key[0], 'job_id': key[1], 'rollout': key[2], 'checks': result['checks']})
# Recheck every frozen source, including retained inputs. Otherwise a
# changing failed attempt could silently enter a supposedly frozen batch.
for row in sources:
relative = Path(row['source'])
if relative.is_absolute() or '..' in relative.parts:
raise PublicationError('INVALID_SOURCE_PATH')
root = source_root / relative
if not isinstance(row.get('hard_gates'), dict) or row['hard_gates'] != _gate_sources(root):
raise PublicationError('SOURCE_CHANGED', 3)
for name, info in row.get('files', {}).items():
path = root / name
if not path.resolve().is_relative_to(source_root):
raise PublicationError('INVALID_SOURCE_PATH')
data = path.read_bytes()
if len(data) != info['bytes'] or _sha(data) != info['sha256']:
raise PublicationError('SOURCE_CHANGED', 3)
code_root = Path(__file__).resolve().parent.parent
code_files = ['export/publication.py', 'export/to_agentrl.py', 'export/trajectory.py', 'export/redact.py',
'harness/verify.py', 'harness/gates.py', 'export/content_review.py']
code_hashes = {name: _sha((code_root / name).read_bytes()) for name in code_files if (code_root / name).is_file()}
artifacts = {name: contents[name] for name in ('sft.jsonl', 'retained.jsonl', 'lineage.jsonl',
'annotations.jsonl', 'excluded_observations.jsonl', 'disposition.jsonl')}
coverage = [item['checks']['VALID_MESSAGE_COVERAGE'] for item in checks]
quality = {'valid_messages': sum(row['numerator'] for row in coverage),
'source_messages': sum(row['denominator'] for row in coverage),
'coverage_percent': 100, 'closure_passed': len(checks), 'closure_failed': 0,
'empty_error_fragments_excluded': len(rows('excluded_observations.jsonl')),
'annotations': dict(Counter(row.get('tag', 'UNKNOWN') for row in rows('annotations.jsonl'))),
'scope': 'coverage_and_closure_accepted_only; exclusions_and_annotations_all_inputs'}
return Releases(datasets).seal(artifacts, rules=rules,
source_manifest={'attempts': sources, 'materialization_sha256': _sha(_bytes(manifest))},
code_sha256=code_hashes,
validation={'accepted': len(accepted), 'retained': len(retained), 'inputs': len(sources),
'checks': {'source_replay': True, 'hard_gates': True, 'disposition': True, 'source_integrity': True},
'samples': checks, 'data_quality': quality})
def main(argv=None):
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--datasets', required=True)
sub = parser.add_subparsers(dest='command', required=True)
command = sub.add_parser('build')
command.add_argument('--materialized', required=True)
command.add_argument('--rules', required=True)
command.add_argument('--source-root', required=True)
for name in ('publish', 'rollback'):
command = sub.add_parser(name)
command.add_argument('--to', required=True)
command.add_argument('--expected-current', type=int, required=True)
command.add_argument('--event-id', required=True)
command.add_argument('--reason', required=True)
command = sub.add_parser('publication-status'); command.add_argument('--event-id', required=True)
command = sub.add_parser('revoke'); command.add_argument('--version', required=True); command.add_argument('--reason', required=True)
sub.add_parser('current')
args = parser.parse_args(argv)
store = Releases(args.datasets)
try:
if args.command == 'build':
result = build(args.materialized, args.datasets, args.rules, args.source_root)
elif args.command in ('publish', 'rollback'):
result = store.switch(args.to, expected_revision=args.expected_current, event_id=args.event_id,
reason=args.reason, rollback=args.command == 'rollback')
elif args.command == 'publication-status':
result = store.status(args.event_id)
elif args.command == 'revoke':
result = store.revoke(args.version, args.reason)
else:
result = store.current()
print(json.dumps(result, ensure_ascii=False))
return 0
except PublicationError as error:
print(json.dumps({'error': error.reason, 'code': error.code}))
return error.code
if __name__ == '__main__':
raise SystemExit(main())