data-gen-bpu/scripts/test_content_review_integration.py
ming01.wang 498187569c fix: close rollout finalization and review publication gaps
Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
2026-09-29 16:09:01 +08:00

549 lines
31 KiB
Python

"""Selected pipeline/Broker review path; local HTTP only, no provider account.
These tests use protected synthetic artifacts, never compilation or a board.
Only installation/configuration and the upstream response are fixtures. The
pipeline, selection, reviewer, Broker, receipt and materializer are real code.
"""
from concurrent.futures import ThreadPoolExecutor
import fcntl
import hashlib
import json
import shutil
import sqlite3
import threading
import time
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from export import content_review, test_materialization as fixtures, to_agentrl
from harness import collection_runtime, execution, paths, pipeline
from scripts.test_quota import Provider
class ContentReviewIntegration(unittest.TestCase):
def setUp(self):
self.fixture = fixtures.MaterializationTests()
self.fixture.setUp()
self.addCleanup(self.fixture.tearDown)
self.root = self.fixture.root
self.addCleanup(self.close_broker)
self.assertIsNone(collection_runtime._broker)
lifecycle = dict.fromkeys(execution.POLICY_FIELDS, 10)
lifecycle.update(version=1, phases={stage: dict(lifecycle) for stage in execution.PHASES})
self.config = {
'content_review': {'enabled': True, 'credential_alias': 'reviewer',
'source_models': ['glm-5.3']},
'rollouts': {'private_home': True},
'endpoints': {'reviewer': {'base_url': 'http://127.0.0.1:1/v1',
'model': 'deepseek-flash', 'api_key_env': 'FAKE_REVIEW_KEY'}},
'collection_v1': {'version': 1, 'quota_ledger': str(self.root / 'state/quota.sqlite'),
'credential_groups': {'reviewer': 'review-only'}, 'lifecycle': lifecycle,
'quota_groups': {'review-only': {'max_concurrency': 1, 'rpm': 1000, 'tpm': 1000000,
'wait_budget_s': 5, 'connect_timeout_s': 1, 'first_byte_timeout_s': 5,
'stream_idle_timeout_s': 5, 'request_total_s': 8, 'poll_s': .01}}},
}
for key, value in {'BENCH_ROOT': self.root, 'RUNS': self.root / 'runs',
'DATASETS': self.root / 'datasets', 'STATE': self.root / 'state',
'AGENT': self.root / 'agent'}.items():
self.start_patch(patch.object(paths, key, value))
self.start_patch(patch.object(paths, 'config', return_value=self.config))
self.start_patch(patch.object(paths, 'secret', return_value='synthetic-local-test-only'))
self.legacy_transport = content_review._request_once
self.start_patch(patch.object(content_review, '_request_once',
side_effect=AssertionError('managed path used legacy SIGALRM transport')))
def start_patch(self, patcher):
value = patcher.start()
self.addCleanup(patcher.stop)
return value
@staticmethod
def close_broker():
if collection_runtime._broker is not None:
collection_runtime._broker.close()
collection_runtime._broker = collection_runtime._configuration = None
def candidate(self, kk=0, run='r', model='glm-5.3'):
rd = self.fixture.add(kk=kk)
shutil.rmtree(rd / 'content-reviews')
(rd / 'claim.json').unlink() # Natural final plus protected gates is sufficient.
metadata = json.loads((rd / 'rollout.json').read_text())
metadata['model'] = model
(rd / 'rollout.json').write_text(json.dumps(metadata))
if run != 'r':
target = self.root / 'runs' / run / 'j' / 'rollouts' / rd.name
target.parent.mkdir(parents=True)
shutil.move(str(rd), str(target))
for name, source, destination in (
('job.json', self.root / 'runs/r/j/job.json', target.parents[1] / 'job.json'),
('verdict.json', target / 'verdict.json', target / 'verdict.json')):
value = json.loads(source.read_text())
value['run_id'] = run
destination.write_text(json.dumps(value))
self.fixture.con.execute('UPDATE rollouts SET run_id=? WHERE run_id=? AND kk=?', (run, 'r', kk))
self.fixture.con.commit()
rd = target
return rd
@staticmethod
def response(finish='stop', decision='PASS'):
issues = [] if decision != 'FAIL' else [{
'quote': 'The measured subset passed within the task tolerance.',
'evidence': 'Synthetic disagreement fixture, not real measurement evidence.',
'reason': 'Synthetic rejected-review scenario.'}]
value = {'model': 'deepseek-flash', 'choices': [{'finish_reason': finish,
'message': {'role': 'assistant', 'content': json.dumps({
'decision': decision, 'reason': 'Synthetic integration response; not a real review.', 'issues': issues})}}],
'usage': {'prompt_tokens': 101, 'completion_tokens': 17, 'total_tokens': 118}}
return 200, {'Content-Type': 'application/json'}, json.dumps(value).encode()
def provider(self, responder=None):
provider = Provider(responder or (lambda *_: self.response()))
self.addCleanup(provider.close)
self.config['endpoints']['reviewer']['base_url'] = provider.url
return provider
def refresh(self, run='r', **kwargs):
def invoke():
self.assertIsNot(threading.current_thread(), threading.main_thread())
return pipeline.refresh_run_dataset(run, strict=True, **kwargs)
with ThreadPoolExecutor(max_workers=1) as workers:
return workers.submit(invoke).result(timeout=15)
def receipt(self, rd):
files = list((rd / 'content-reviews').glob('*.json'))
self.assertEqual(len(files), 1)
return files[0], json.loads(files[0].read_text())
def rows(self, name, run='r'):
return [json.loads(line) for line in
(self.root / f'datasets/{run}-training/{name}.jsonl').read_text().splitlines()]
def assert_manifest(self, result, run='r'):
dataset = self.root / f'datasets/{run}-training'
manifest = json.loads((dataset / 'manifest.json').read_text())
self.assertEqual(manifest, result)
for name, expected in manifest['files'].items():
data = (dataset / name).read_bytes()
self.assertEqual(hashlib.sha256(data).hexdigest(), expected['sha256'], name)
self.assertEqual(len(data), expected['bytes'], name)
self.assertEqual(len(data.splitlines()), expected['records'], name)
def test_worker_pipeline_real_broker_receipt_and_original_sft(self):
rd = self.candidate()
source_before = content_review.trajectory.freeze(rd).manifest
gates_before = to_agentrl._gate_sources(rd)
unlocked = []
def respond(_number, body):
# HTTP must not run while the exporter holds its dataset write lock.
with (self.root / 'datasets/r-training/.export.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
unlocked.append(True)
return self.response()
provider = self.provider(respond)
result = self.refresh()
self.assertEqual((len(provider.requests), unlocked), (1, [True]))
body = provider.requests[0]['body']
self.assertEqual(body['model'], 'deepseek-flash')
self.assertEqual(body['max_tokens'], 32768)
context = json.loads(body['messages'][1]['content'])
self.assertEqual(context['task']['run_id'], 'r')
self.assertEqual(context['measurement_summary']['G3']['quant'], 79.2)
self.assertEqual(len(context['messages']), len(self.rows('sft')[0]['messages']))
path, receipt = self.receipt(rd)
self.assertEqual((receipt['decision'], receipt['state'], receipt['model']),
('PASS', 'COMPLETE', 'deepseek-flash'))
self.assertEqual(receipt['usage']['total_tokens'], 118)
self.assertEqual(receipt['response_model'], 'deepseek-flash')
self.assertEqual(path.stem, content_review.digest(receipt['binding']))
self.assertEqual(content_review.trajectory.freeze(rd).manifest, source_before)
self.assertEqual(to_agentrl._gate_sources(rd), gates_before)
self.assertEqual(result['stats']['sft_eligible'], 1)
self.assertEqual(self.rows('retained'), [])
ledger = collection_runtime.broker().snapshot()
self.assertEqual((len(ledger['executions']), len(ledger['generations']), len(ledger['attempts'])), (1, 1, 1))
self.assertEqual(ledger['executions'][0]['execution_id'], receipt['execution_id'])
self.assertEqual(ledger['executions'][0]['group_id'], 'review-only')
self.assertEqual(ledger['executions'][0]['cancelled'], 1)
self.assertEqual(ledger['generations'][0]['generation_id'], receipt['generation_id'])
self.assertEqual(ledger['generations'][0]['state'], 'COMPLETE')
self.assertEqual(ledger['attempts'][0]['charged_tokens'], 118)
self.assertIsNotNone(ledger['attempts'][0]['ended_at'])
self.assert_manifest(result)
def test_repeated_refresh_reuses_receipt_without_new_generation(self):
rd = self.candidate()
provider = self.provider()
first = self.refresh()
path, _ = self.receipt(rd)
before = path.read_bytes(), self.rows('sft'), collection_runtime.broker().snapshot()['generations']
second = self.refresh()
self.assertEqual(len(provider.requests), 1)
self.assertEqual(before, (path.read_bytes(), self.rows('sft'), collection_runtime.broker().snapshot()['generations']))
self.assertEqual(first['files']['sft.jsonl'], second['files']['sft.jsonl'])
self.assert_manifest(second)
def test_explicit_unknown_retry_archives_bytes_and_uses_new_generation(self):
rd = self.candidate()
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
self.refresh()
path, original = self.receipt(rd)
original_bytes = path.read_bytes()
digest = hashlib.sha256(original_bytes).hexdigest()
result = self.refresh(review=True, retry_unknown=True)
_, reviewed = self.receipt(rd)
self.assertEqual((reviewed['decision'], result['stats']['sft_eligible']), ('PASS', 1))
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
self.assertEqual(reviewed['previous_receipt_sha256'], digest)
self.assertNotEqual(reviewed['execution_id'], original['execution_id'])
self.assertTrue(reviewed['execution_id'].endswith(content_review.digest(
{'previous': digest, 'started_at': reviewed['started_at']})))
self.assertEqual(reviewed['request_bytes'], len(json.dumps(provider.requests[-1]['body']).encode()))
self.assertEqual(reviewed['max_tokens'], 32768)
self.refresh()
self.assertEqual(len(provider.requests), 2)
self.assertEqual(path.read_bytes(), json.dumps(reviewed, ensure_ascii=False, indent=2).encode())
def test_local_tpm_refusal_is_explicitly_retryable_after_policy_migration(self):
rd = self.candidate()
provider = self.provider()
self.config['collection_v1']['quota_groups']['review-only']['tpm'] = 1000
self.refresh()
path, original = self.receipt(rd)
original_bytes = path.read_bytes()
self.assertEqual(original['reason'], 'REQUEST_EXCEEDS_INTERNAL_TPM')
self.assertEqual(len(provider.requests), 0)
self.close_broker()
# This is an isolated, drained ledger migration, not a provider-limit claim.
policy = self.config['collection_v1']['quota_groups']['review-only']
policy['tpm'] = 1000000
from dataclasses import asdict
from harness.quota import QuotaPolicy
with sqlite3.connect(self.config['collection_v1']['quota_ledger']) as con:
con.execute('UPDATE quota_groups SET policy=? WHERE group_id=?',
(json.dumps(asdict(QuotaPolicy(**policy)), sort_keys=True), 'review-only'))
con.close()
result = self.refresh(review=True, retry_unknown=True)
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (1, 1))
digest = hashlib.sha256(original_bytes).hexdigest()
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
def test_retry_interrupted_after_archive_preserves_current_receipt_and_can_retry(self):
rd = self.candidate()
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
self.refresh()
path, _ = self.receipt(rd)
original_bytes = path.read_bytes()
digest = hashlib.sha256(original_bytes).hexdigest()
with patch.object(content_review, '_atomic', side_effect=OSError('injected receipt publication failure')):
with self.assertRaises(OSError):
content_review.review_once(rd, managed=True, retry_unknown=True)
self.assertEqual(path.read_bytes(), original_bytes)
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
self.assertEqual(len(provider.requests), 1)
result = self.refresh(review=True, retry_unknown=True)
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (2, 1))
def test_retry_rejects_started_and_success_receipts_without_request(self):
rd = self.candidate()
provider = self.provider()
self.refresh()
path, receipt = self.receipt(rd)
for state, decision in [('COMPLETE', 'PASS'), ('COMPLETE', 'FAIL'), ('STARTED', 'UNKNOWN')]:
with self.subTest(state=state, decision=decision):
receipt.update(state=state, decision=decision)
path.write_text(json.dumps(receipt))
before = path.read_bytes()
with self.assertRaisesRegex(ValueError, 'REVIEW_RETRY_REQUIRES_COMPLETE_UNKNOWN'):
content_review.review_once(rd, managed=True, retry_unknown=True)
self.assertEqual(path.read_bytes(), before)
self.assertEqual(len(provider.requests), 1)
self.assertFalse((path.parent / 'history').exists())
def test_retry_cli_requires_explicit_one_off_review(self):
from harness.cli import cmd_export
for review, follow in [(False, False), (False, True), (True, True)]:
with self.subTest(review=review, follow=follow):
with self.assertRaisesRegex(SystemExit, 'requires --review'):
cmd_export(SimpleNamespace(retry_unknown=True, review=review, follow=follow))
def test_local_tpm_refusal_does_not_starve_smaller_candidate(self):
large = self.candidate(kk=0)
self.candidate(kk=1)
system = (large / 'agent/system.md').read_text() + (' synthetic context' * 6000)
(large / 'agent/system.md').write_text(system)
events = [json.loads(line) for line in (large / 'agent/events.jsonl').read_text().splitlines()]
events[0]['bench']['system_prompt'] = system
(large / 'agent/events.jsonl').write_text('\n'.join(map(json.dumps, events)))
self.config['collection_v1']['quota_groups']['review-only']['tpm'] = 50000
provider = self.provider()
result = self.refresh()
self.assertEqual(self.receipt(large)[1]['reason'], 'REQUEST_EXCEEDS_INTERNAL_TPM')
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (1, 1))
def test_pending_only_continues_one_after_unknown_without_repeating_unknown(self):
first = self.candidate(kk=0)
self.candidate(kk=1)
self.candidate(kk=2)
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
self.refresh()
old = self.receipt(first)[0].read_bytes()
for expected in (2, 3):
with patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
result = self.refresh(pending_only=True)
self.assertEqual(materialize.call_count, 1) # Only publish after one new review.
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (expected, expected - 1))
self.assertEqual(self.receipt(first)[0].read_bytes(), old)
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('idle export rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('idle source read')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(len(provider.requests), 3)
def test_pending_only_paused_account_preserves_unreviewed_candidates(self):
self.candidate(kk=0)
later = self.candidate(kk=1)
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
self.refresh()
with collection_runtime.broker()._db() as con:
con.execute("UPDATE quota_groups SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN'")
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('paused export rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('paused source read')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(list((later / 'content-reviews').glob('*.json')), [])
self.assertEqual(len(provider.requests), 1)
def test_pending_only_keeps_started_receipt_without_review_or_publication(self):
rd = self.candidate()
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
to_agentrl.materialize(['r'], 'r-training')
content_review.review_once(rd, managed=True)
path, receipt = self.receipt(rd)
receipt.update(state='STARTED', decision='UNKNOWN')
path.write_text(json.dumps(receipt))
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('STARTED export rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('STARTED source read')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(len(provider.requests), 1)
def test_saved_reviews_publish_after_materialization_failure_even_when_account_paused(self):
decision = ['PASS']
provider = self.provider(lambda *_: self.response(decision=decision[0]))
broker = collection_runtime.broker()
cases = [(value, state) for value in ('PASS', 'FAIL', 'UNKNOWN') for state in ('PAUSED', 'RECONCILING')]
for kk, (value, state) in enumerate(cases):
with self.subTest(decision=value, account=state):
with broker._db() as con:
con.execute("UPDATE quota_groups SET state='OPEN',reason=NULL")
rd = self.candidate(kk=kk)
decision[0] = value
to_agentrl.materialize(['r'], 'r-training', refresh=True)
before_db = self.fixture.con.execute('SELECT * FROM rollouts').fetchall()
with patch.object(to_agentrl, 'materialize', side_effect=OSError('injected publish failure')):
with self.assertRaisesRegex(OSError, 'injected publish failure'):
self.refresh(pending_only=True)
path, receipt = self.receipt(rd)
self.assertEqual((receipt['state'], receipt['decision']), ('COMPLETE', value))
saved = path.read_bytes()
requests = len(provider.requests)
with broker._db() as con:
con.execute('UPDATE quota_groups SET state=?,reason=?', (state, 'REQUEST_OUTCOME_UNKNOWN'))
with patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
result = self.refresh(pending_only=True)
self.assertIsNotNone(result)
self.assertEqual(materialize.call_count, 1)
disposition = next(row for row in self.rows('disposition') if row['rollout'] == kk)
self.assertEqual(disposition['reasons'], [] if value == 'PASS' else ['content-review-' + value.lower()])
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('published receipt rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('published source read')):
for _ in range(3):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual((len(provider.requests), path.read_bytes()), (requests, saved))
self.assertEqual(self.fixture.con.execute('SELECT * FROM rollouts').fetchall(), before_db)
def test_retried_unknown_receipt_is_republished_once_even_with_same_decision(self):
rd = self.candidate()
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
self.refresh()
to_agentrl.materialize(['r'], 'r-training', refresh=True, purpose='sft')
old_hash = self.rows('disposition')[0]['review_receipt_sha256']
original = to_agentrl.materialize
calls = []
def fail_after_review(*args, **kwargs):
calls.append(True)
if len(calls) == 2:
raise OSError('injected publish failure')
return original(*args, **kwargs)
with patch.object(to_agentrl, 'materialize', side_effect=fail_after_review):
with self.assertRaisesRegex(OSError, 'injected publish failure'):
self.refresh(review=True, retry_unknown=True)
path, _ = self.receipt(rd)
new_hash = hashlib.sha256(path.read_bytes()).hexdigest()
self.assertNotEqual(new_hash, old_hash)
with collection_runtime.broker()._db() as con:
con.execute("UPDATE quota_groups SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN'")
self.assertIsNotNone(self.refresh(pending_only=True))
self.assertEqual(self.rows('disposition')[0]['review_receipt_sha256'], new_hash)
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('published receipt rescanned')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(len(provider.requests), 2)
def test_follow_unchanged_database_continues_pending_review(self):
from harness import cli
self.candidate(kk=0)
self.candidate(kk=1)
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
before = self.fixture.con.execute('SELECT * FROM rollouts').fetchall()
read_only, sleep = cli.db.readonly, time.sleep
intervals = []
def next_interval(seconds):
if seconds != 30:
return sleep(seconds)
intervals.append(seconds)
if len(intervals) == 2:
raise SystemExit('follow test complete')
args = SimpleNamespace(run='r', follow=True, interval=30, review=True, retry_unknown=False, until_pid=None)
(paths.STATE / 'locks').mkdir(exist_ok=True)
with (paths.STATE / 'locks/export-follow-r.lock').open('a') as follow_lock, \
patch('harness.cli.open', return_value=follow_lock, create=True), \
patch.object(cli.db, 'readonly', side_effect=lambda: read_only(self.root / 'state/jobs.sqlite')), \
patch.object(time, 'sleep', side_effect=next_interval), \
patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
with self.assertRaisesRegex(SystemExit, 'follow test complete'):
cli.cmd_export(args)
self.assertEqual(materialize.call_count, 3) # Initial snapshot + UNKNOWN, then one new PASS.
self.assertEqual(self.fixture.con.execute('SELECT * FROM rollouts').fetchall(), before)
self.assertEqual((len(provider.requests), len(self.rows('sft'))), (2, 1))
def test_pending_only_busy_account_does_not_read_sources(self):
first = self.candidate(kk=0)
self.candidate(kk=1)
entered, release = threading.Event(), threading.Event()
def response(*_):
entered.set()
self.assertTrue(release.wait(5))
return self.response()
provider = self.provider(response)
to_agentrl.materialize(['r'], 'r-training')
with ThreadPoolExecutor(max_workers=1) as worker:
future = worker.submit(content_review.review_once, first, managed=True)
self.assertTrue(entered.wait(5))
try:
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('busy export rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('busy source read')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(len(provider.requests), 1)
finally:
release.set()
self.assertEqual(future.result(timeout=5)['decision'], 'PASS')
def test_incomplete_response_is_unknown_retained_and_never_resent(self):
rd = self.candidate()
provider = self.provider(lambda *_: self.response(finish='length'))
first = self.refresh()
path, receipt = self.receipt(rd)
self.assertEqual((receipt['state'], receipt['decision']), ('COMPLETE', 'UNKNOWN'))
self.assertEqual(first['stats']['sft_eligible'], 0)
self.assertEqual(len(self.rows('retained')), 1)
self.assertEqual(self.rows('sft'), [])
self.assertIn('content-review-unknown', self.rows('disposition')[0]['reasons'])
before = path.read_bytes()
self.refresh()
self.assertEqual((len(provider.requests), path.read_bytes()), (1, before))
self.assertEqual(len(collection_runtime.broker().snapshot()['generations']), 1)
def test_bad_configuration_is_visible_before_receipt_or_http(self):
rd = self.candidate()
provider = self.provider()
self.config['endpoints'].pop('reviewer')
with self.assertRaisesRegex(ValueError, 'CONTENT_REVIEW_CONFIG_INCOMPLETE'):
self.refresh()
self.assertEqual(provider.requests, [])
self.assertEqual(list((rd / 'content-reviews').glob('*.json')), [])
self.assertEqual(self.rows('sft'), [])
self.assertEqual(len(self.rows('retained')), 1)
self.assertIn('content-review-unreviewed', self.rows('disposition')[0]['reasons'])
self.assertIsNone(collection_runtime._broker)
def test_unknown_stops_remaining_candidates_in_current_refresh(self):
first = self.candidate(kk=0)
second = self.candidate(kk=1)
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
result = self.refresh()
self.assertEqual(len(provider.requests), 1)
self.assertEqual(self.receipt(first)[1]['decision'], 'UNKNOWN')
self.assertEqual(list((second / 'content-reviews').glob('*.json')), [])
self.assertEqual((result['stats']['sft_eligible'], len(self.rows('retained'))), (0, 2))
self.assertEqual([row['reasons'] for row in self.rows('disposition')],
[['content-review-unknown'], ['content-review-unreviewed']])
self.assert_manifest(result)
def test_explicit_legacy_pipeline_review_remains_compatible(self):
rd = self.candidate()
provider = self.provider()
self.config.pop('content_review')
with patch.dict('os.environ', {'BENCH_REVIEW_URL': provider.url + '/chat/completions',
'BENCH_REVIEW_MODEL': 'legacy-fixture', 'BENCH_REVIEW_API_KEY': ''}), \
patch.object(content_review, '_request_once', side_effect=self.legacy_transport):
# The historical transport retains its main-thread SIGALRM contract.
result = pipeline.refresh_run_dataset('r', strict=True, review=True)
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (1, 1))
self.assertEqual(provider.requests[0]['body']['model'], 'legacy-fixture')
self.assertEqual(self.receipt(rd)[1]['decision'], 'PASS')
self.assertIsNone(collection_runtime._broker)
self.assert_manifest(result)
def test_concurrent_runs_are_serial_and_excluded_source_is_not_reviewed(self):
first = self.candidate(kk=0)
second = self.candidate(kk=1, run='r2')
excluded = self.candidate(kk=2, model='qwen-fixture')
arrived, release, second_lock_attempt = threading.Event(), threading.Event(), threading.Event()
self.addCleanup(release.set)
counts = {'active': 0, 'peak': 0, 'locks': 0}
mutex = threading.Lock()
def respond(number, body):
with mutex:
counts['active'] += 1
counts['peak'] = max(counts['peak'], counts['active'])
try:
if number == 1:
arrived.set()
if not release.wait(5):
raise AssertionError('local upstream barrier timed out')
return self.response()
finally:
with mutex:
counts['active'] -= 1
provider = self.provider(respond)
original_flock = fcntl.flock
def observe_lock(file, operation):
if str(getattr(file, 'name', '')).endswith('/content-review.lock') and operation == fcntl.LOCK_EX:
with mutex:
counts['locks'] += 1
if counts['locks'] == 2:
second_lock_attempt.set()
return original_flock(file, operation)
with patch.object(fcntl, 'flock', side_effect=observe_lock), ThreadPoolExecutor(max_workers=2) as workers:
jobs = [workers.submit(pipeline.refresh_run_dataset, run, strict=True) for run in ('r', 'r2')]
try:
self.assertTrue(arrived.wait(5))
self.assertTrue(second_lock_attempt.wait(5))
self.assertEqual(len(provider.requests), 1)
finally:
release.set()
results = [job.result(timeout=15) for job in jobs]
self.assertEqual((len(provider.requests), counts['peak']), (2, 1))
self.assertEqual([r['stats']['sft_eligible'] for r in results], [1, 1])
self.assertNotEqual(self.receipt(first)[1]['execution_id'], self.receipt(second)[1]['execution_id'])
self.assertEqual(list((excluded / 'content-reviews').glob('*.json')), [])
self.assertEqual(len(self.rows('retained')), 1)
ledger = collection_runtime.broker().snapshot()
self.assertEqual((len(ledger['generations']), len(ledger['attempts'])), (2, 2))
self.assertTrue(all(row['cancelled'] for row in ledger['executions']))
for run, result in zip(('r', 'r2'), results):
self.assert_manifest(result, run)
if __name__ == '__main__':
unittest.main()