183 lines
11 KiB
Python
183 lines
11 KiB
Python
"""One real historical negative calibration, read-only source, shared quota only.
|
|
|
|
Writes only a new staging report directory and its own shared quota accounting.
|
|
This is calibration evidence, never an admission receipt or publication action.
|
|
"""
|
|
from contextlib import closing
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
import uuid
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
from export import content_review as cr, to_agentrl as ex, trajectory as tr
|
|
from harness import paths
|
|
from harness.verify import collect_measurements, verify_rollout
|
|
from harness.quota import Broker, QuotaPolicy, Upstream
|
|
|
|
|
|
def save(path, data):
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
|
path.chmod(0o600)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--production-root', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--case', choices=['negative', 'sdk-positive'], default='negative')
|
|
args = parser.parse_args()
|
|
root, output = args.production_root.resolve(), args.output.resolve()
|
|
if not output.is_relative_to(ROOT):
|
|
raise ValueError('REPORT_OUTPUT_MUST_BE_IN_STAGING')
|
|
output.mkdir(parents=True, exist_ok=False) # Never repeat this paid calibration.
|
|
if args.case == 'negative':
|
|
rd = root / 'runs/r20260922-02/cls--resnet18--nash-p--ptq--fp16int8/rollouts/00'
|
|
frozen = tr.freeze(rd)
|
|
reconstructed = tr.reconstruct(frozen)
|
|
validation = tr.validate(frozen, reconstructed)
|
|
validation['checks']['NORMAL_FINAL_RESPONSE'] = ex.normal_final_response(reconstructed['messages'])
|
|
validation['pass'] = all(c['pass'] for c in validation['checks'].values())
|
|
verdict = verify_rollout(rd, write=False)
|
|
gates = ex._gate_sources(rd)
|
|
job = json.loads((rd.parent.parent / 'job.json').read_text())
|
|
hard = ex._hard_gate_check(job, verdict, gates, gates, collect_measurements(rd)[0])
|
|
with closing(sqlite3.connect((root / 'state/jobs.sqlite').as_uri() + '?mode=ro', uri=True)) as con:
|
|
rows = con.execute('SELECT run_id,job_id,note FROM annotations WHERE run_id=?', (verdict['run_id'],)).fetchall()
|
|
reasons = sorted(ex.annotation_reasons(rows, verdict['run_id'], verdict['job_id'], 0))
|
|
qualified = (validation['pass'] and hard['pass'] and not reasons and ex._database_terminal(rd, root)
|
|
and ex.sft_eligibility(verdict, 0)[0])
|
|
bound = cr.binding(rd, verdict, frozen, gates)
|
|
old_verdict_sha = hashlib.sha256((rd / 'verdict.json').read_bytes()).hexdigest()
|
|
before = {'binding': bound, 'old_verdict_sha256': old_verdict_sha, 'annotation_rows_sha256': cr.digest(rows)}
|
|
save(output / 'prescreen.json', {'qualified': qualified, 'validation': validation, 'hard_gates': hard,
|
|
'annotation_reasons': reasons, 'source': str(rd), **before})
|
|
if not qualified:
|
|
save(output / 'summary.json', {'decision': 'NOT_SCRIPT_QUALIFIED', 'model_calls': 0})
|
|
return
|
|
context = cr.review_context(job, reconstructed, bound, gates)
|
|
unchanged_check = lambda: (tr.source_unchanged(rd, frozen) and ex._gate_sources(rd) == gates
|
|
and hashlib.sha256((rd / "verdict.json").read_bytes()).hexdigest() == old_verdict_sha)
|
|
else:
|
|
rd = ROOT / 'outputs/a1-live-budget-20260922/two-turn-read-final'
|
|
names = ['events.jsonl', 'native-session.jsonl', 'checkpoint.json', 'messages.json', 'system.md', 'tools.json', 'prompt.md']
|
|
original = {name: (rd / name).read_bytes() for name in names}
|
|
manifest = {name: {'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data)} for name, data in original.items()}
|
|
checkpoint = json.loads(original['checkpoint.json'])
|
|
messages = json.loads(original['messages.json'])
|
|
events = [json.loads(line) for line in original['events.jsonl'].splitlines()]
|
|
header = events[0]['bench']
|
|
qualified = (checkpoint.get('phase') == 'COMPLETED' and checkpoint.get('complete') is True
|
|
and checkpoint.get('closure') is True and tr.closure(messages)['pass']
|
|
and ex.normal_final_response(messages)['pass']
|
|
and [event['message'] for event in events if event.get('type') == 'message_end'] == messages
|
|
and events[-1].get('type') == 'agent_end'
|
|
and hashlib.sha256(original['native-session.jsonl']).hexdigest() == checkpoint['sha256']
|
|
and hashlib.sha256(Path(checkpoint['session_path']).read_bytes()).hexdigest() == checkpoint['sha256']
|
|
and header.get('system_prompt') == original['system.md'].decode()
|
|
and header.get('tools') == json.loads(original['tools.json']))
|
|
results = [message for message in messages if message.get('role') == 'toolResult']
|
|
calls = [call for message in messages for call in tr._calls(message)]
|
|
qualified = qualified and len(calls) == len(results) == 1 and calls[0]['name'] == 'read' and not results[0].get('isError')
|
|
reconstructed = {'messages': [{'role': 'system', 'content': original['system.md'].decode()}] + messages,
|
|
'tools': json.loads(original['tools.json'])}
|
|
authority = {'execution_state': 'COMPLETED', 'tool_calls': calls, 'tool_results': results,
|
|
'compile_calls': 0, 'inference_calls': 0, 'board_calls': 0,
|
|
'scope': 'Real SDK read-only diagnostic task; synthetic input file, no hardware/model claims.'}
|
|
bound = {'rule': cr.RULE, 'review_prompt_sha256': hashlib.sha256(cr.SYSTEM.encode()).hexdigest(),
|
|
'source_manifest': manifest, 'authoritative_result': authority, 'measurement_summary': {}}
|
|
context = cr.review_context({'user_request': original['prompt.md'].decode(), 'kind': 'SDK diagnostic'}, reconstructed, bound, manifest)
|
|
before = {'binding': bound}
|
|
save(output / 'prescreen.json', {'qualified': qualified, 'scope': 'SDK content calibration only; not a board SFT admission candidate', **before})
|
|
if not qualified:
|
|
save(output / 'summary.json', {'decision': 'NOT_SCRIPT_QUALIFIED', 'model_calls': 0})
|
|
return
|
|
unchanged_check = lambda: all((rd / name).read_bytes() == data for name, data in original.items())
|
|
save(output / 'input.json', context)
|
|
config = yaml.safe_load((root / 'config.yaml').read_text())
|
|
endpoint, shared = config['endpoints']['teacher'], config['collection_v1']
|
|
if endpoint['model'] != 'glm-5.3':
|
|
raise ValueError('UNEXPECTED_TEACHER_MODEL')
|
|
prior_root = paths.BENCH_ROOT
|
|
try:
|
|
paths.BENCH_ROOT = root
|
|
paths.load_env_local.cache_clear()
|
|
key = paths.secret(endpoint['api_key_env'])
|
|
finally:
|
|
paths.BENCH_ROOT = prior_root
|
|
paths.load_env_local.cache_clear()
|
|
if not key:
|
|
raise ValueError('TEACHER_CREDENTIAL_UNAVAILABLE')
|
|
groups = {name: QuotaPolicy(**value) for name, value in shared['quota_groups'].items()}
|
|
broker = Broker(Path(shared['quota_ledger']), groups, {'teacher': Upstream(
|
|
shared['credential_groups']['teacher'], endpoint['base_url'], key, (endpoint['model'],))})
|
|
broker.start()
|
|
execution_id = 'natural-content-' + args.case + '-' + uuid.uuid4().hex
|
|
scope = broker.issue_scope(execution_id, 'teacher', endpoint['model'])
|
|
generation_id = execution_id + ':0'
|
|
record = {'purpose': 'READONLY_CALIBRATION_NOT_PUBLICATION', 'case': args.case,
|
|
'expected_decision': 'FAIL' if args.case == 'negative' else 'PASS',
|
|
'source': str(rd), 'model': endpoint['model'], 'fixture_provider': False, 'execution_id': execution_id,
|
|
'generation_id': generation_id, 'shared_quota_ledger': str(broker.path), 'qualified': qualified,
|
|
'decision': 'UNKNOWN', 'model_requests': 1, 'usage': None, 'monetary_cost': 'UNKNOWN',
|
|
'compile_calls': 0, 'inference_calls': 0, 'board_calls': 0,
|
|
'rule': cr.RULE, 'prompt_sha256': hashlib.sha256(cr.SYSTEM.encode()).hexdigest()}
|
|
save(output / 'started.json', record)
|
|
started = time.monotonic()
|
|
try:
|
|
body = json.dumps({'model': endpoint['model'], 'messages': [{'role': 'system', 'content': cr.SYSTEM},
|
|
{'role': 'user', 'content': json.dumps(context, ensure_ascii=False)}],
|
|
'max_tokens': 8192, 'temperature': 0}).encode()
|
|
if len(body) > 2_000_000:
|
|
raise ValueError('CONTEXT_TOO_LARGE_NO_TRUNCATION')
|
|
request = urllib.request.Request(scope.base_url + '/chat/completions', body,
|
|
{'Content-Type': 'application/json', 'Authorization': 'Bearer ' + scope.api_key,
|
|
'X-Bench-Generation-Id': generation_id})
|
|
response = cr._request_once(request, 120)
|
|
save(output / 'response.json', json.loads(response))
|
|
record['usage'] = json.loads(response).get('usage')
|
|
review, usage = cr.parse_response(response, reconstructed['messages'])
|
|
record.update(review, usage=usage, response_sha256=hashlib.sha256(response).hexdigest())
|
|
except Exception as error:
|
|
record.update(decision='UNKNOWN', error=type(error).__name__)
|
|
finally:
|
|
# Cancel just this scope before observing final ledger rows or closing server.
|
|
broker.cancel(execution_id)
|
|
save(output / 'quota-status.json', broker.execution_status(execution_id))
|
|
with closing(sqlite3.connect(broker.path)) as con:
|
|
con.row_factory = sqlite3.Row
|
|
attempts = [dict(row) for row in con.execute('''SELECT attempt_id,generation_id,state,sent,http_status,
|
|
charged_tokens,usage_certainty,reason,transport_reason FROM attempts WHERE generation_id IN
|
|
(SELECT generation_id FROM generations WHERE execution_id=?)''', (execution_id,))]
|
|
save(output / 'quota-attempts.json', attempts)
|
|
broker.close()
|
|
unchanged = unchanged_check()
|
|
if args.case == 'negative':
|
|
with closing(sqlite3.connect((root / 'state/jobs.sqlite').as_uri() + '?mode=ro', uri=True)) as con:
|
|
after_rows = con.execute('SELECT run_id,job_id,note FROM annotations WHERE run_id=?', (verdict['run_id'],)).fetchall()
|
|
unchanged = unchanged and cr.digest(after_rows) == before['annotation_rows_sha256']
|
|
record.update(elapsed_s=round(time.monotonic() - started, 3), source_unchanged=unchanged,
|
|
original_verdict_unchanged=True if unchanged else None,
|
|
upstream_attempts=len(attempts), charged_tokens=sum(a['charged_tokens'] or 0 for a in attempts),
|
|
usage_certainty=sorted(set(a['usage_certainty'] for a in attempts)),
|
|
false_accept=args.case == 'negative' and record['decision'] == 'PASS',
|
|
negative_rejected=args.case == 'negative' and record['decision'] == 'FAIL',
|
|
positive_accepted=args.case == 'sdk-positive' and record['decision'] == 'PASS',
|
|
review_binding_sha256=cr.digest(bound))
|
|
if not unchanged:
|
|
record['decision'] = 'UNKNOWN'
|
|
save(output / 'summary.json', record)
|
|
print(json.dumps(record, ensure_ascii=False), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|