data-gen-bpu/scripts/run_natural_content_probe.py
bench db5a7a839d Export: read the running checkout's own state wherever it lives
A checkout whose state is elsewhere (BENCH_STATE_DIR, staging) was read as
<checkout>/state, so content review and export found no database and called
a PASS rollout unfinished. One helper, paths.state_dir, used by every lookup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 16:52:42 +08:00

183 lines
11 KiB
Python

"""One real historical negative calibration, read-only source, shared quota only.
Writes only a new staging report directory and its own shared quota accounting.
This is calibration evidence, never an admission receipt or publication action.
"""
from contextlib import closing
import argparse
import hashlib
import json
import os
from pathlib import Path
import sqlite3
import sys
import time
import urllib.request
import uuid
import yaml
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from export import content_review as cr, to_agentrl as ex, trajectory as tr
from harness import paths
from harness.verify import collect_measurements, verify_rollout
from harness.quota import Broker, QuotaPolicy, Upstream
def save(path, data):
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
path.chmod(0o600)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--production-root', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--case', choices=['negative', 'sdk-positive'], default='negative')
args = parser.parse_args()
root, output = args.production_root.resolve(), args.output.resolve()
if not output.is_relative_to(ROOT):
raise ValueError('REPORT_OUTPUT_MUST_BE_IN_STAGING')
output.mkdir(parents=True, exist_ok=False) # Never repeat this paid calibration.
if args.case == 'negative':
rd = root / 'runs/r20260922-02/cls--resnet18--nash-p--ptq--fp16int8/rollouts/00'
frozen = tr.freeze(rd)
reconstructed = tr.reconstruct(frozen)
validation = tr.validate(frozen, reconstructed)
validation['checks']['NORMAL_FINAL_RESPONSE'] = ex.normal_final_response(reconstructed['messages'])
validation['pass'] = all(c['pass'] for c in validation['checks'].values())
verdict = verify_rollout(rd, write=False)
gates = ex._gate_sources(rd)
job = json.loads((rd.parent.parent / 'job.json').read_text())
hard = ex._hard_gate_check(job, verdict, gates, gates, collect_measurements(rd)[0])
with closing(sqlite3.connect((paths.state_dir(root) / 'jobs.sqlite').as_uri() + '?mode=ro', uri=True)) as con:
rows = con.execute('SELECT run_id,job_id,note FROM annotations WHERE run_id=?', (verdict['run_id'],)).fetchall()
reasons = sorted(ex.annotation_reasons(rows, verdict['run_id'], verdict['job_id'], 0))
qualified = (validation['pass'] and hard['pass'] and not reasons and ex._database_terminal(rd, root)
and ex.sft_eligibility(verdict, 0)[0])
bound = cr.binding(rd, verdict, frozen, gates)
old_verdict_sha = hashlib.sha256((rd / 'verdict.json').read_bytes()).hexdigest()
before = {'binding': bound, 'old_verdict_sha256': old_verdict_sha, 'annotation_rows_sha256': cr.digest(rows)}
save(output / 'prescreen.json', {'qualified': qualified, 'validation': validation, 'hard_gates': hard,
'annotation_reasons': reasons, 'source': str(rd), **before})
if not qualified:
save(output / 'summary.json', {'decision': 'NOT_SCRIPT_QUALIFIED', 'model_calls': 0})
return
context = cr.review_context(job, reconstructed, bound, gates)
unchanged_check = lambda: (tr.source_unchanged(rd, frozen) and ex._gate_sources(rd) == gates
and hashlib.sha256((rd / "verdict.json").read_bytes()).hexdigest() == old_verdict_sha)
else:
rd = ROOT / 'outputs/a1-live-budget-20260922/two-turn-read-final'
names = ['events.jsonl', 'native-session.jsonl', 'checkpoint.json', 'messages.json', 'system.md', 'tools.json', 'prompt.md']
original = {name: (rd / name).read_bytes() for name in names}
manifest = {name: {'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data)} for name, data in original.items()}
checkpoint = json.loads(original['checkpoint.json'])
messages = json.loads(original['messages.json'])
events = [json.loads(line) for line in original['events.jsonl'].splitlines()]
header = events[0]['bench']
qualified = (checkpoint.get('phase') == 'COMPLETED' and checkpoint.get('complete') is True
and checkpoint.get('closure') is True and tr.closure(messages)['pass']
and ex.normal_final_response(messages)['pass']
and [event['message'] for event in events if event.get('type') == 'message_end'] == messages
and events[-1].get('type') == 'agent_end'
and hashlib.sha256(original['native-session.jsonl']).hexdigest() == checkpoint['sha256']
and hashlib.sha256(Path(checkpoint['session_path']).read_bytes()).hexdigest() == checkpoint['sha256']
and header.get('system_prompt') == original['system.md'].decode()
and header.get('tools') == json.loads(original['tools.json']))
results = [message for message in messages if message.get('role') == 'toolResult']
calls = [call for message in messages for call in tr._calls(message)]
qualified = qualified and len(calls) == len(results) == 1 and calls[0]['name'] == 'read' and not results[0].get('isError')
reconstructed = {'messages': [{'role': 'system', 'content': original['system.md'].decode()}] + messages,
'tools': json.loads(original['tools.json'])}
authority = {'execution_state': 'COMPLETED', 'tool_calls': calls, 'tool_results': results,
'compile_calls': 0, 'inference_calls': 0, 'board_calls': 0,
'scope': 'Real SDK read-only diagnostic task; synthetic input file, no hardware/model claims.'}
bound = {'rule': cr.RULE, 'review_prompt_sha256': hashlib.sha256(cr.SYSTEM.encode()).hexdigest(),
'source_manifest': manifest, 'authoritative_result': authority, 'measurement_summary': {}}
context = cr.review_context({'user_request': original['prompt.md'].decode(), 'kind': 'SDK diagnostic'}, reconstructed, bound, manifest)
before = {'binding': bound}
save(output / 'prescreen.json', {'qualified': qualified, 'scope': 'SDK content calibration only; not a board SFT admission candidate', **before})
if not qualified:
save(output / 'summary.json', {'decision': 'NOT_SCRIPT_QUALIFIED', 'model_calls': 0})
return
unchanged_check = lambda: all((rd / name).read_bytes() == data for name, data in original.items())
save(output / 'input.json', context)
config = yaml.safe_load((root / 'config.yaml').read_text())
endpoint, shared = config['endpoints']['teacher'], config['collection_v1']
if endpoint['model'] != 'glm-5.3':
raise ValueError('UNEXPECTED_TEACHER_MODEL')
prior_root = paths.BENCH_ROOT
try:
paths.BENCH_ROOT = root
paths.load_env_local.cache_clear()
key = paths.secret(endpoint['api_key_env'])
finally:
paths.BENCH_ROOT = prior_root
paths.load_env_local.cache_clear()
if not key:
raise ValueError('TEACHER_CREDENTIAL_UNAVAILABLE')
groups = {name: QuotaPolicy(**value) for name, value in shared['quota_groups'].items()}
broker = Broker(Path(shared['quota_ledger']), groups, {'teacher': Upstream(
shared['credential_groups']['teacher'], endpoint['base_url'], key, (endpoint['model'],))})
broker.start()
execution_id = 'natural-content-' + args.case + '-' + uuid.uuid4().hex
scope = broker.issue_scope(execution_id, 'teacher', endpoint['model'])
generation_id = execution_id + ':0'
record = {'purpose': 'READONLY_CALIBRATION_NOT_PUBLICATION', 'case': args.case,
'expected_decision': 'FAIL' if args.case == 'negative' else 'PASS',
'source': str(rd), 'model': endpoint['model'], 'fixture_provider': False, 'execution_id': execution_id,
'generation_id': generation_id, 'shared_quota_ledger': str(broker.path), 'qualified': qualified,
'decision': 'UNKNOWN', 'model_requests': 1, 'usage': None, 'monetary_cost': 'UNKNOWN',
'compile_calls': 0, 'inference_calls': 0, 'board_calls': 0,
'rule': cr.RULE, 'prompt_sha256': hashlib.sha256(cr.SYSTEM.encode()).hexdigest()}
save(output / 'started.json', record)
started = time.monotonic()
try:
body = json.dumps({'model': endpoint['model'], 'messages': [{'role': 'system', 'content': cr.SYSTEM},
{'role': 'user', 'content': json.dumps(context, ensure_ascii=False)}],
'max_tokens': 8192, 'temperature': 0}).encode()
if len(body) > 2_000_000:
raise ValueError('CONTEXT_TOO_LARGE_NO_TRUNCATION')
request = urllib.request.Request(scope.base_url + '/chat/completions', body,
{'Content-Type': 'application/json', 'Authorization': 'Bearer ' + scope.api_key,
'X-Bench-Generation-Id': generation_id})
response = cr._request_once(request, 120)
save(output / 'response.json', json.loads(response))
record['usage'] = json.loads(response).get('usage')
review, usage = cr.parse_response(response, reconstructed['messages'])
record.update(review, usage=usage, response_sha256=hashlib.sha256(response).hexdigest())
except Exception as error:
record.update(decision='UNKNOWN', error=type(error).__name__)
finally:
# Cancel just this scope before observing final ledger rows or closing server.
broker.cancel(execution_id)
save(output / 'quota-status.json', broker.execution_status(execution_id))
with closing(sqlite3.connect(broker.path)) as con:
con.row_factory = sqlite3.Row
attempts = [dict(row) for row in con.execute('''SELECT attempt_id,generation_id,state,sent,http_status,
charged_tokens,usage_certainty,reason,transport_reason FROM attempts WHERE generation_id IN
(SELECT generation_id FROM generations WHERE execution_id=?)''', (execution_id,))]
save(output / 'quota-attempts.json', attempts)
broker.close()
unchanged = unchanged_check()
if args.case == 'negative':
with closing(sqlite3.connect((paths.state_dir(root) / 'jobs.sqlite').as_uri() + '?mode=ro', uri=True)) as con:
after_rows = con.execute('SELECT run_id,job_id,note FROM annotations WHERE run_id=?', (verdict['run_id'],)).fetchall()
unchanged = unchanged and cr.digest(after_rows) == before['annotation_rows_sha256']
record.update(elapsed_s=round(time.monotonic() - started, 3), source_unchanged=unchanged,
original_verdict_unchanged=True if unchanged else None,
upstream_attempts=len(attempts), charged_tokens=sum(a['charged_tokens'] or 0 for a in attempts),
usage_certainty=sorted(set(a['usage_certainty'] for a in attempts)),
false_accept=args.case == 'negative' and record['decision'] == 'PASS',
negative_rejected=args.case == 'negative' and record['decision'] == 'FAIL',
positive_accepted=args.case == 'sdk-positive' and record['decision'] == 'PASS',
review_binding_sha256=cr.digest(bound))
if not unchanged:
record['decision'] = 'UNKNOWN'
save(output / 'summary.json', record)
print(json.dumps(record, ensure_ascii=False), flush=True)
if __name__ == '__main__':
main()