Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
549 lines
31 KiB
Python
549 lines
31 KiB
Python
"""Selected pipeline/Broker review path; local HTTP only, no provider account.
|
|
|
|
These tests use protected synthetic artifacts, never compilation or a board.
|
|
Only installation/configuration and the upstream response are fixtures. The
|
|
pipeline, selection, reviewer, Broker, receipt and materializer are real code.
|
|
"""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from export import content_review, test_materialization as fixtures, to_agentrl
|
|
from harness import collection_runtime, execution, paths, pipeline
|
|
from scripts.test_quota import Provider
|
|
|
|
|
|
class ContentReviewIntegration(unittest.TestCase):
|
|
def setUp(self):
|
|
self.fixture = fixtures.MaterializationTests()
|
|
self.fixture.setUp()
|
|
self.addCleanup(self.fixture.tearDown)
|
|
self.root = self.fixture.root
|
|
self.addCleanup(self.close_broker)
|
|
self.assertIsNone(collection_runtime._broker)
|
|
lifecycle = dict.fromkeys(execution.POLICY_FIELDS, 10)
|
|
lifecycle.update(version=1, phases={stage: dict(lifecycle) for stage in execution.PHASES})
|
|
self.config = {
|
|
'content_review': {'enabled': True, 'credential_alias': 'reviewer',
|
|
'source_models': ['glm-5.3']},
|
|
'rollouts': {'private_home': True},
|
|
'endpoints': {'reviewer': {'base_url': 'http://127.0.0.1:1/v1',
|
|
'model': 'deepseek-flash', 'api_key_env': 'FAKE_REVIEW_KEY'}},
|
|
'collection_v1': {'version': 1, 'quota_ledger': str(self.root / 'state/quota.sqlite'),
|
|
'credential_groups': {'reviewer': 'review-only'}, 'lifecycle': lifecycle,
|
|
'quota_groups': {'review-only': {'max_concurrency': 1, 'rpm': 1000, 'tpm': 1000000,
|
|
'wait_budget_s': 5, 'connect_timeout_s': 1, 'first_byte_timeout_s': 5,
|
|
'stream_idle_timeout_s': 5, 'request_total_s': 8, 'poll_s': .01}}},
|
|
}
|
|
for key, value in {'BENCH_ROOT': self.root, 'RUNS': self.root / 'runs',
|
|
'DATASETS': self.root / 'datasets', 'STATE': self.root / 'state',
|
|
'AGENT': self.root / 'agent'}.items():
|
|
self.start_patch(patch.object(paths, key, value))
|
|
self.start_patch(patch.object(paths, 'config', return_value=self.config))
|
|
self.start_patch(patch.object(paths, 'secret', return_value='synthetic-local-test-only'))
|
|
self.legacy_transport = content_review._request_once
|
|
self.start_patch(patch.object(content_review, '_request_once',
|
|
side_effect=AssertionError('managed path used legacy SIGALRM transport')))
|
|
|
|
def start_patch(self, patcher):
|
|
value = patcher.start()
|
|
self.addCleanup(patcher.stop)
|
|
return value
|
|
|
|
@staticmethod
|
|
def close_broker():
|
|
if collection_runtime._broker is not None:
|
|
collection_runtime._broker.close()
|
|
collection_runtime._broker = collection_runtime._configuration = None
|
|
|
|
def candidate(self, kk=0, run='r', model='glm-5.3'):
|
|
rd = self.fixture.add(kk=kk)
|
|
shutil.rmtree(rd / 'content-reviews')
|
|
(rd / 'claim.json').unlink() # Natural final plus protected gates is sufficient.
|
|
metadata = json.loads((rd / 'rollout.json').read_text())
|
|
metadata['model'] = model
|
|
(rd / 'rollout.json').write_text(json.dumps(metadata))
|
|
if run != 'r':
|
|
target = self.root / 'runs' / run / 'j' / 'rollouts' / rd.name
|
|
target.parent.mkdir(parents=True)
|
|
shutil.move(str(rd), str(target))
|
|
for name, source, destination in (
|
|
('job.json', self.root / 'runs/r/j/job.json', target.parents[1] / 'job.json'),
|
|
('verdict.json', target / 'verdict.json', target / 'verdict.json')):
|
|
value = json.loads(source.read_text())
|
|
value['run_id'] = run
|
|
destination.write_text(json.dumps(value))
|
|
self.fixture.con.execute('UPDATE rollouts SET run_id=? WHERE run_id=? AND kk=?', (run, 'r', kk))
|
|
self.fixture.con.commit()
|
|
rd = target
|
|
return rd
|
|
|
|
@staticmethod
|
|
def response(finish='stop', decision='PASS'):
|
|
issues = [] if decision != 'FAIL' else [{
|
|
'quote': 'The measured subset passed within the task tolerance.',
|
|
'evidence': 'Synthetic disagreement fixture, not real measurement evidence.',
|
|
'reason': 'Synthetic rejected-review scenario.'}]
|
|
value = {'model': 'deepseek-flash', 'choices': [{'finish_reason': finish,
|
|
'message': {'role': 'assistant', 'content': json.dumps({
|
|
'decision': decision, 'reason': 'Synthetic integration response; not a real review.', 'issues': issues})}}],
|
|
'usage': {'prompt_tokens': 101, 'completion_tokens': 17, 'total_tokens': 118}}
|
|
return 200, {'Content-Type': 'application/json'}, json.dumps(value).encode()
|
|
|
|
def provider(self, responder=None):
|
|
provider = Provider(responder or (lambda *_: self.response()))
|
|
self.addCleanup(provider.close)
|
|
self.config['endpoints']['reviewer']['base_url'] = provider.url
|
|
return provider
|
|
|
|
def refresh(self, run='r', **kwargs):
|
|
def invoke():
|
|
self.assertIsNot(threading.current_thread(), threading.main_thread())
|
|
return pipeline.refresh_run_dataset(run, strict=True, **kwargs)
|
|
with ThreadPoolExecutor(max_workers=1) as workers:
|
|
return workers.submit(invoke).result(timeout=15)
|
|
|
|
def receipt(self, rd):
|
|
files = list((rd / 'content-reviews').glob('*.json'))
|
|
self.assertEqual(len(files), 1)
|
|
return files[0], json.loads(files[0].read_text())
|
|
|
|
def rows(self, name, run='r'):
|
|
return [json.loads(line) for line in
|
|
(self.root / f'datasets/{run}-training/{name}.jsonl').read_text().splitlines()]
|
|
|
|
def assert_manifest(self, result, run='r'):
|
|
dataset = self.root / f'datasets/{run}-training'
|
|
manifest = json.loads((dataset / 'manifest.json').read_text())
|
|
self.assertEqual(manifest, result)
|
|
for name, expected in manifest['files'].items():
|
|
data = (dataset / name).read_bytes()
|
|
self.assertEqual(hashlib.sha256(data).hexdigest(), expected['sha256'], name)
|
|
self.assertEqual(len(data), expected['bytes'], name)
|
|
self.assertEqual(len(data.splitlines()), expected['records'], name)
|
|
|
|
def test_worker_pipeline_real_broker_receipt_and_original_sft(self):
|
|
rd = self.candidate()
|
|
source_before = content_review.trajectory.freeze(rd).manifest
|
|
gates_before = to_agentrl._gate_sources(rd)
|
|
unlocked = []
|
|
def respond(_number, body):
|
|
# HTTP must not run while the exporter holds its dataset write lock.
|
|
with (self.root / 'datasets/r-training/.export.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
unlocked.append(True)
|
|
return self.response()
|
|
provider = self.provider(respond)
|
|
result = self.refresh()
|
|
self.assertEqual((len(provider.requests), unlocked), (1, [True]))
|
|
body = provider.requests[0]['body']
|
|
self.assertEqual(body['model'], 'deepseek-flash')
|
|
self.assertEqual(body['max_tokens'], 32768)
|
|
context = json.loads(body['messages'][1]['content'])
|
|
self.assertEqual(context['task']['run_id'], 'r')
|
|
self.assertEqual(context['measurement_summary']['G3']['quant'], 79.2)
|
|
self.assertEqual(len(context['messages']), len(self.rows('sft')[0]['messages']))
|
|
path, receipt = self.receipt(rd)
|
|
self.assertEqual((receipt['decision'], receipt['state'], receipt['model']),
|
|
('PASS', 'COMPLETE', 'deepseek-flash'))
|
|
self.assertEqual(receipt['usage']['total_tokens'], 118)
|
|
self.assertEqual(receipt['response_model'], 'deepseek-flash')
|
|
self.assertEqual(path.stem, content_review.digest(receipt['binding']))
|
|
self.assertEqual(content_review.trajectory.freeze(rd).manifest, source_before)
|
|
self.assertEqual(to_agentrl._gate_sources(rd), gates_before)
|
|
self.assertEqual(result['stats']['sft_eligible'], 1)
|
|
self.assertEqual(self.rows('retained'), [])
|
|
ledger = collection_runtime.broker().snapshot()
|
|
self.assertEqual((len(ledger['executions']), len(ledger['generations']), len(ledger['attempts'])), (1, 1, 1))
|
|
self.assertEqual(ledger['executions'][0]['execution_id'], receipt['execution_id'])
|
|
self.assertEqual(ledger['executions'][0]['group_id'], 'review-only')
|
|
self.assertEqual(ledger['executions'][0]['cancelled'], 1)
|
|
self.assertEqual(ledger['generations'][0]['generation_id'], receipt['generation_id'])
|
|
self.assertEqual(ledger['generations'][0]['state'], 'COMPLETE')
|
|
self.assertEqual(ledger['attempts'][0]['charged_tokens'], 118)
|
|
self.assertIsNotNone(ledger['attempts'][0]['ended_at'])
|
|
self.assert_manifest(result)
|
|
|
|
def test_repeated_refresh_reuses_receipt_without_new_generation(self):
|
|
rd = self.candidate()
|
|
provider = self.provider()
|
|
first = self.refresh()
|
|
path, _ = self.receipt(rd)
|
|
before = path.read_bytes(), self.rows('sft'), collection_runtime.broker().snapshot()['generations']
|
|
second = self.refresh()
|
|
self.assertEqual(len(provider.requests), 1)
|
|
self.assertEqual(before, (path.read_bytes(), self.rows('sft'), collection_runtime.broker().snapshot()['generations']))
|
|
self.assertEqual(first['files']['sft.jsonl'], second['files']['sft.jsonl'])
|
|
self.assert_manifest(second)
|
|
|
|
def test_explicit_unknown_retry_archives_bytes_and_uses_new_generation(self):
|
|
rd = self.candidate()
|
|
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
|
|
self.refresh()
|
|
path, original = self.receipt(rd)
|
|
original_bytes = path.read_bytes()
|
|
digest = hashlib.sha256(original_bytes).hexdigest()
|
|
result = self.refresh(review=True, retry_unknown=True)
|
|
_, reviewed = self.receipt(rd)
|
|
self.assertEqual((reviewed['decision'], result['stats']['sft_eligible']), ('PASS', 1))
|
|
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
|
|
self.assertEqual(reviewed['previous_receipt_sha256'], digest)
|
|
self.assertNotEqual(reviewed['execution_id'], original['execution_id'])
|
|
self.assertTrue(reviewed['execution_id'].endswith(content_review.digest(
|
|
{'previous': digest, 'started_at': reviewed['started_at']})))
|
|
self.assertEqual(reviewed['request_bytes'], len(json.dumps(provider.requests[-1]['body']).encode()))
|
|
self.assertEqual(reviewed['max_tokens'], 32768)
|
|
self.refresh()
|
|
self.assertEqual(len(provider.requests), 2)
|
|
self.assertEqual(path.read_bytes(), json.dumps(reviewed, ensure_ascii=False, indent=2).encode())
|
|
|
|
def test_local_tpm_refusal_is_explicitly_retryable_after_policy_migration(self):
|
|
rd = self.candidate()
|
|
provider = self.provider()
|
|
self.config['collection_v1']['quota_groups']['review-only']['tpm'] = 1000
|
|
self.refresh()
|
|
path, original = self.receipt(rd)
|
|
original_bytes = path.read_bytes()
|
|
self.assertEqual(original['reason'], 'REQUEST_EXCEEDS_INTERNAL_TPM')
|
|
self.assertEqual(len(provider.requests), 0)
|
|
self.close_broker()
|
|
# This is an isolated, drained ledger migration, not a provider-limit claim.
|
|
policy = self.config['collection_v1']['quota_groups']['review-only']
|
|
policy['tpm'] = 1000000
|
|
from dataclasses import asdict
|
|
from harness.quota import QuotaPolicy
|
|
with sqlite3.connect(self.config['collection_v1']['quota_ledger']) as con:
|
|
con.execute('UPDATE quota_groups SET policy=? WHERE group_id=?',
|
|
(json.dumps(asdict(QuotaPolicy(**policy)), sort_keys=True), 'review-only'))
|
|
con.close()
|
|
result = self.refresh(review=True, retry_unknown=True)
|
|
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (1, 1))
|
|
digest = hashlib.sha256(original_bytes).hexdigest()
|
|
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
|
|
|
|
def test_retry_interrupted_after_archive_preserves_current_receipt_and_can_retry(self):
|
|
rd = self.candidate()
|
|
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
|
|
self.refresh()
|
|
path, _ = self.receipt(rd)
|
|
original_bytes = path.read_bytes()
|
|
digest = hashlib.sha256(original_bytes).hexdigest()
|
|
with patch.object(content_review, '_atomic', side_effect=OSError('injected receipt publication failure')):
|
|
with self.assertRaises(OSError):
|
|
content_review.review_once(rd, managed=True, retry_unknown=True)
|
|
self.assertEqual(path.read_bytes(), original_bytes)
|
|
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
|
|
self.assertEqual(len(provider.requests), 1)
|
|
result = self.refresh(review=True, retry_unknown=True)
|
|
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (2, 1))
|
|
|
|
def test_retry_rejects_started_and_success_receipts_without_request(self):
|
|
rd = self.candidate()
|
|
provider = self.provider()
|
|
self.refresh()
|
|
path, receipt = self.receipt(rd)
|
|
for state, decision in [('COMPLETE', 'PASS'), ('COMPLETE', 'FAIL'), ('STARTED', 'UNKNOWN')]:
|
|
with self.subTest(state=state, decision=decision):
|
|
receipt.update(state=state, decision=decision)
|
|
path.write_text(json.dumps(receipt))
|
|
before = path.read_bytes()
|
|
with self.assertRaisesRegex(ValueError, 'REVIEW_RETRY_REQUIRES_COMPLETE_UNKNOWN'):
|
|
content_review.review_once(rd, managed=True, retry_unknown=True)
|
|
self.assertEqual(path.read_bytes(), before)
|
|
self.assertEqual(len(provider.requests), 1)
|
|
self.assertFalse((path.parent / 'history').exists())
|
|
|
|
def test_retry_cli_requires_explicit_one_off_review(self):
|
|
from harness.cli import cmd_export
|
|
for review, follow in [(False, False), (False, True), (True, True)]:
|
|
with self.subTest(review=review, follow=follow):
|
|
with self.assertRaisesRegex(SystemExit, 'requires --review'):
|
|
cmd_export(SimpleNamespace(retry_unknown=True, review=review, follow=follow))
|
|
|
|
def test_local_tpm_refusal_does_not_starve_smaller_candidate(self):
|
|
large = self.candidate(kk=0)
|
|
self.candidate(kk=1)
|
|
system = (large / 'agent/system.md').read_text() + (' synthetic context' * 6000)
|
|
(large / 'agent/system.md').write_text(system)
|
|
events = [json.loads(line) for line in (large / 'agent/events.jsonl').read_text().splitlines()]
|
|
events[0]['bench']['system_prompt'] = system
|
|
(large / 'agent/events.jsonl').write_text('\n'.join(map(json.dumps, events)))
|
|
self.config['collection_v1']['quota_groups']['review-only']['tpm'] = 50000
|
|
provider = self.provider()
|
|
result = self.refresh()
|
|
self.assertEqual(self.receipt(large)[1]['reason'], 'REQUEST_EXCEEDS_INTERNAL_TPM')
|
|
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (1, 1))
|
|
|
|
def test_pending_only_continues_one_after_unknown_without_repeating_unknown(self):
|
|
first = self.candidate(kk=0)
|
|
self.candidate(kk=1)
|
|
self.candidate(kk=2)
|
|
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
|
|
self.refresh()
|
|
old = self.receipt(first)[0].read_bytes()
|
|
for expected in (2, 3):
|
|
with patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
|
|
result = self.refresh(pending_only=True)
|
|
self.assertEqual(materialize.call_count, 1) # Only publish after one new review.
|
|
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (expected, expected - 1))
|
|
self.assertEqual(self.receipt(first)[0].read_bytes(), old)
|
|
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('idle export rescanned')), \
|
|
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('idle source read')):
|
|
self.assertIsNone(self.refresh(pending_only=True))
|
|
self.assertEqual(len(provider.requests), 3)
|
|
|
|
def test_pending_only_paused_account_preserves_unreviewed_candidates(self):
|
|
self.candidate(kk=0)
|
|
later = self.candidate(kk=1)
|
|
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
|
|
self.refresh()
|
|
with collection_runtime.broker()._db() as con:
|
|
con.execute("UPDATE quota_groups SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN'")
|
|
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('paused export rescanned')), \
|
|
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('paused source read')):
|
|
self.assertIsNone(self.refresh(pending_only=True))
|
|
self.assertEqual(list((later / 'content-reviews').glob('*.json')), [])
|
|
self.assertEqual(len(provider.requests), 1)
|
|
|
|
def test_pending_only_keeps_started_receipt_without_review_or_publication(self):
|
|
rd = self.candidate()
|
|
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
|
|
to_agentrl.materialize(['r'], 'r-training')
|
|
content_review.review_once(rd, managed=True)
|
|
path, receipt = self.receipt(rd)
|
|
receipt.update(state='STARTED', decision='UNKNOWN')
|
|
path.write_text(json.dumps(receipt))
|
|
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('STARTED export rescanned')), \
|
|
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('STARTED source read')):
|
|
self.assertIsNone(self.refresh(pending_only=True))
|
|
self.assertEqual(len(provider.requests), 1)
|
|
|
|
def test_saved_reviews_publish_after_materialization_failure_even_when_account_paused(self):
|
|
decision = ['PASS']
|
|
provider = self.provider(lambda *_: self.response(decision=decision[0]))
|
|
broker = collection_runtime.broker()
|
|
cases = [(value, state) for value in ('PASS', 'FAIL', 'UNKNOWN') for state in ('PAUSED', 'RECONCILING')]
|
|
for kk, (value, state) in enumerate(cases):
|
|
with self.subTest(decision=value, account=state):
|
|
with broker._db() as con:
|
|
con.execute("UPDATE quota_groups SET state='OPEN',reason=NULL")
|
|
rd = self.candidate(kk=kk)
|
|
decision[0] = value
|
|
to_agentrl.materialize(['r'], 'r-training', refresh=True)
|
|
before_db = self.fixture.con.execute('SELECT * FROM rollouts').fetchall()
|
|
with patch.object(to_agentrl, 'materialize', side_effect=OSError('injected publish failure')):
|
|
with self.assertRaisesRegex(OSError, 'injected publish failure'):
|
|
self.refresh(pending_only=True)
|
|
path, receipt = self.receipt(rd)
|
|
self.assertEqual((receipt['state'], receipt['decision']), ('COMPLETE', value))
|
|
saved = path.read_bytes()
|
|
requests = len(provider.requests)
|
|
with broker._db() as con:
|
|
con.execute('UPDATE quota_groups SET state=?,reason=?', (state, 'REQUEST_OUTCOME_UNKNOWN'))
|
|
with patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
|
|
result = self.refresh(pending_only=True)
|
|
self.assertIsNotNone(result)
|
|
self.assertEqual(materialize.call_count, 1)
|
|
disposition = next(row for row in self.rows('disposition') if row['rollout'] == kk)
|
|
self.assertEqual(disposition['reasons'], [] if value == 'PASS' else ['content-review-' + value.lower()])
|
|
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('published receipt rescanned')), \
|
|
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('published source read')):
|
|
for _ in range(3):
|
|
self.assertIsNone(self.refresh(pending_only=True))
|
|
self.assertEqual((len(provider.requests), path.read_bytes()), (requests, saved))
|
|
self.assertEqual(self.fixture.con.execute('SELECT * FROM rollouts').fetchall(), before_db)
|
|
|
|
def test_retried_unknown_receipt_is_republished_once_even_with_same_decision(self):
|
|
rd = self.candidate()
|
|
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
|
|
self.refresh()
|
|
to_agentrl.materialize(['r'], 'r-training', refresh=True, purpose='sft')
|
|
old_hash = self.rows('disposition')[0]['review_receipt_sha256']
|
|
original = to_agentrl.materialize
|
|
calls = []
|
|
def fail_after_review(*args, **kwargs):
|
|
calls.append(True)
|
|
if len(calls) == 2:
|
|
raise OSError('injected publish failure')
|
|
return original(*args, **kwargs)
|
|
with patch.object(to_agentrl, 'materialize', side_effect=fail_after_review):
|
|
with self.assertRaisesRegex(OSError, 'injected publish failure'):
|
|
self.refresh(review=True, retry_unknown=True)
|
|
path, _ = self.receipt(rd)
|
|
new_hash = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
self.assertNotEqual(new_hash, old_hash)
|
|
with collection_runtime.broker()._db() as con:
|
|
con.execute("UPDATE quota_groups SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN'")
|
|
self.assertIsNotNone(self.refresh(pending_only=True))
|
|
self.assertEqual(self.rows('disposition')[0]['review_receipt_sha256'], new_hash)
|
|
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('published receipt rescanned')):
|
|
self.assertIsNone(self.refresh(pending_only=True))
|
|
self.assertEqual(len(provider.requests), 2)
|
|
|
|
def test_follow_unchanged_database_continues_pending_review(self):
|
|
from harness import cli
|
|
self.candidate(kk=0)
|
|
self.candidate(kk=1)
|
|
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
|
|
before = self.fixture.con.execute('SELECT * FROM rollouts').fetchall()
|
|
read_only, sleep = cli.db.readonly, time.sleep
|
|
intervals = []
|
|
def next_interval(seconds):
|
|
if seconds != 30:
|
|
return sleep(seconds)
|
|
intervals.append(seconds)
|
|
if len(intervals) == 2:
|
|
raise SystemExit('follow test complete')
|
|
args = SimpleNamespace(run='r', follow=True, interval=30, review=True, retry_unknown=False, until_pid=None)
|
|
(paths.STATE / 'locks').mkdir(exist_ok=True)
|
|
with (paths.STATE / 'locks/export-follow-r.lock').open('a') as follow_lock, \
|
|
patch('harness.cli.open', return_value=follow_lock, create=True), \
|
|
patch.object(cli.db, 'readonly', side_effect=lambda: read_only(self.root / 'state/jobs.sqlite')), \
|
|
patch.object(time, 'sleep', side_effect=next_interval), \
|
|
patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
|
|
with self.assertRaisesRegex(SystemExit, 'follow test complete'):
|
|
cli.cmd_export(args)
|
|
self.assertEqual(materialize.call_count, 3) # Initial snapshot + UNKNOWN, then one new PASS.
|
|
self.assertEqual(self.fixture.con.execute('SELECT * FROM rollouts').fetchall(), before)
|
|
self.assertEqual((len(provider.requests), len(self.rows('sft'))), (2, 1))
|
|
|
|
def test_pending_only_busy_account_does_not_read_sources(self):
|
|
first = self.candidate(kk=0)
|
|
self.candidate(kk=1)
|
|
entered, release = threading.Event(), threading.Event()
|
|
def response(*_):
|
|
entered.set()
|
|
self.assertTrue(release.wait(5))
|
|
return self.response()
|
|
provider = self.provider(response)
|
|
to_agentrl.materialize(['r'], 'r-training')
|
|
with ThreadPoolExecutor(max_workers=1) as worker:
|
|
future = worker.submit(content_review.review_once, first, managed=True)
|
|
self.assertTrue(entered.wait(5))
|
|
try:
|
|
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('busy export rescanned')), \
|
|
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('busy source read')):
|
|
self.assertIsNone(self.refresh(pending_only=True))
|
|
self.assertEqual(len(provider.requests), 1)
|
|
finally:
|
|
release.set()
|
|
self.assertEqual(future.result(timeout=5)['decision'], 'PASS')
|
|
|
|
def test_incomplete_response_is_unknown_retained_and_never_resent(self):
|
|
rd = self.candidate()
|
|
provider = self.provider(lambda *_: self.response(finish='length'))
|
|
first = self.refresh()
|
|
path, receipt = self.receipt(rd)
|
|
self.assertEqual((receipt['state'], receipt['decision']), ('COMPLETE', 'UNKNOWN'))
|
|
self.assertEqual(first['stats']['sft_eligible'], 0)
|
|
self.assertEqual(len(self.rows('retained')), 1)
|
|
self.assertEqual(self.rows('sft'), [])
|
|
self.assertIn('content-review-unknown', self.rows('disposition')[0]['reasons'])
|
|
before = path.read_bytes()
|
|
self.refresh()
|
|
self.assertEqual((len(provider.requests), path.read_bytes()), (1, before))
|
|
self.assertEqual(len(collection_runtime.broker().snapshot()['generations']), 1)
|
|
|
|
def test_bad_configuration_is_visible_before_receipt_or_http(self):
|
|
rd = self.candidate()
|
|
provider = self.provider()
|
|
self.config['endpoints'].pop('reviewer')
|
|
with self.assertRaisesRegex(ValueError, 'CONTENT_REVIEW_CONFIG_INCOMPLETE'):
|
|
self.refresh()
|
|
self.assertEqual(provider.requests, [])
|
|
self.assertEqual(list((rd / 'content-reviews').glob('*.json')), [])
|
|
self.assertEqual(self.rows('sft'), [])
|
|
self.assertEqual(len(self.rows('retained')), 1)
|
|
self.assertIn('content-review-unreviewed', self.rows('disposition')[0]['reasons'])
|
|
self.assertIsNone(collection_runtime._broker)
|
|
|
|
def test_unknown_stops_remaining_candidates_in_current_refresh(self):
|
|
first = self.candidate(kk=0)
|
|
second = self.candidate(kk=1)
|
|
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
|
|
result = self.refresh()
|
|
self.assertEqual(len(provider.requests), 1)
|
|
self.assertEqual(self.receipt(first)[1]['decision'], 'UNKNOWN')
|
|
self.assertEqual(list((second / 'content-reviews').glob('*.json')), [])
|
|
self.assertEqual((result['stats']['sft_eligible'], len(self.rows('retained'))), (0, 2))
|
|
self.assertEqual([row['reasons'] for row in self.rows('disposition')],
|
|
[['content-review-unknown'], ['content-review-unreviewed']])
|
|
self.assert_manifest(result)
|
|
|
|
def test_explicit_legacy_pipeline_review_remains_compatible(self):
|
|
rd = self.candidate()
|
|
provider = self.provider()
|
|
self.config.pop('content_review')
|
|
with patch.dict('os.environ', {'BENCH_REVIEW_URL': provider.url + '/chat/completions',
|
|
'BENCH_REVIEW_MODEL': 'legacy-fixture', 'BENCH_REVIEW_API_KEY': ''}), \
|
|
patch.object(content_review, '_request_once', side_effect=self.legacy_transport):
|
|
# The historical transport retains its main-thread SIGALRM contract.
|
|
result = pipeline.refresh_run_dataset('r', strict=True, review=True)
|
|
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (1, 1))
|
|
self.assertEqual(provider.requests[0]['body']['model'], 'legacy-fixture')
|
|
self.assertEqual(self.receipt(rd)[1]['decision'], 'PASS')
|
|
self.assertIsNone(collection_runtime._broker)
|
|
self.assert_manifest(result)
|
|
|
|
def test_concurrent_runs_are_serial_and_excluded_source_is_not_reviewed(self):
|
|
first = self.candidate(kk=0)
|
|
second = self.candidate(kk=1, run='r2')
|
|
excluded = self.candidate(kk=2, model='qwen-fixture')
|
|
arrived, release, second_lock_attempt = threading.Event(), threading.Event(), threading.Event()
|
|
self.addCleanup(release.set)
|
|
counts = {'active': 0, 'peak': 0, 'locks': 0}
|
|
mutex = threading.Lock()
|
|
def respond(number, body):
|
|
with mutex:
|
|
counts['active'] += 1
|
|
counts['peak'] = max(counts['peak'], counts['active'])
|
|
try:
|
|
if number == 1:
|
|
arrived.set()
|
|
if not release.wait(5):
|
|
raise AssertionError('local upstream barrier timed out')
|
|
return self.response()
|
|
finally:
|
|
with mutex:
|
|
counts['active'] -= 1
|
|
provider = self.provider(respond)
|
|
original_flock = fcntl.flock
|
|
def observe_lock(file, operation):
|
|
if str(getattr(file, 'name', '')).endswith('/content-review.lock') and operation == fcntl.LOCK_EX:
|
|
with mutex:
|
|
counts['locks'] += 1
|
|
if counts['locks'] == 2:
|
|
second_lock_attempt.set()
|
|
return original_flock(file, operation)
|
|
with patch.object(fcntl, 'flock', side_effect=observe_lock), ThreadPoolExecutor(max_workers=2) as workers:
|
|
jobs = [workers.submit(pipeline.refresh_run_dataset, run, strict=True) for run in ('r', 'r2')]
|
|
try:
|
|
self.assertTrue(arrived.wait(5))
|
|
self.assertTrue(second_lock_attempt.wait(5))
|
|
self.assertEqual(len(provider.requests), 1)
|
|
finally:
|
|
release.set()
|
|
results = [job.result(timeout=15) for job in jobs]
|
|
self.assertEqual((len(provider.requests), counts['peak']), (2, 1))
|
|
self.assertEqual([r['stats']['sft_eligible'] for r in results], [1, 1])
|
|
self.assertNotEqual(self.receipt(first)[1]['execution_id'], self.receipt(second)[1]['execution_id'])
|
|
self.assertEqual(list((excluded / 'content-reviews').glob('*.json')), [])
|
|
self.assertEqual(len(self.rows('retained')), 1)
|
|
ledger = collection_runtime.broker().snapshot()
|
|
self.assertEqual((len(ledger['generations']), len(ledger['attempts'])), (2, 2))
|
|
self.assertTrue(all(row['cancelled'] for row in ledger['executions']))
|
|
for run, result in zip(('r', 'r2'), results):
|
|
self.assert_manifest(result, run)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|