data-gen-bpu/scripts/test_codex.py
bench 6437b572a2 fix: freeze long Pi sessions at rollout end (no 1 MiB cap)
freeze_checkpoint read the session through config_bytes, which caps files at
1 MiB for tickets/configs. Long rollouts pass that, so finalization failed:
10-15..10-17, 17 rollouts, 4 recorded INFRA_ERROR instead of their real
outcome. The session read keeps the symlink/regular-file checks, no cap.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 10:33:37 +08:00

214 lines
14 KiB
Python

"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI."""
import importlib.util
import json
import os
import tempfile
import unittest
import yaml
from pathlib import Path
from unittest.mock import patch
from harness.stageworker import Worker, config_bytes, publish, snapshot_config
from harness import watch
ROOT = Path(__file__).resolve().parents[1]
def module(name, path):
spec = importlib.util.spec_from_file_location(name, ROOT/path)
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m
exporter = module('exporter', 'export/to_agentrl.py')
audit = module('audit', 'scripts/audit_rollouts.py')
class Regression(unittest.TestCase):
def test_audit_does_not_offer_historical_repair_or_blocked_as_training(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'runs/r/job/rollouts/00';(rd/'agent').mkdir(parents=True)
(rd/'rollout.json').write_text('{"agent_config_version":7,"state":"DONE"}')
(rd/'agent/events.jsonl').write_text('{"type":"agent_end","messages":[]}')
for mode, terminal, expected in [('e2e','PASS',True),('repair','PASS',False),('e2e','BLOCKED',False)]:
(rd/'verdict.json').write_text(json.dumps({'terminal':terminal,'claim_consistent':True}))
(rd/'claim.json').write_text(json.dumps({'terminal':terminal}))
result=audit.audit_one(rd,0,7,mode)
self.assertEqual(result['rl_ok'],expected)
self.assertEqual(result['sft_ok'],expected)
def test_annotations_scope(self):
rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')]
self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'})
self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'})
self.assertFalse(exporter.annotation_reasons(rows,'run','other',2))
def test_config_symlink_and_traversal(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x')
(ws/'link').symlink_to('/etc/passwd')
self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x')
for p in [ws/'link', ws/'../outside']:
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p))
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p),limit=None)
# A long Pi session is several MiB: capped for configs, whole for the frozen checkpoint.
(ws/'session.jsonl').write_bytes(b'x'*(3<<20))
with self.assertRaises(ValueError): config_bytes(ws,str(ws/'session.jsonl'))
self.assertEqual(len(config_bytes(ws,str(ws/'session.jsonl'),limit=None)),3<<20)
def test_worker_config_file_paths(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)
p=ws/'config.yaml';p.write_text('calibration_parameters:\n cal_data_dir: /state\n')
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
p.write_text('calibration_parameters:\n quant_config: /etc/passwd\n')
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
(ws/'q.json').write_text('{"model_config":{"all_node_type":"int8"}}')
p.write_text('calibration_parameters:\n quant_config: q.json\n')
self.assertIn(b'all_node_type: int8',snapshot_config(ws,str(p)))
p.write_text('verification_parameters: {hbm_bc_samples: 2}\n')
self.assertIn(b'hbm_bc_samples: 2',snapshot_config(ws,str(p)))
for bad in ('-1', '17', 'true', '"2"'):
p.write_text('verification_parameters: {hbm_bc_samples: %s}\n' % bad)
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
# The agent's own graph/calibration: kept workspace-relative (04 resolves them in bench dev's copy).
p.write_text(f'model_parameters: {{onnx_model: {ws}/g/m.onnx, march: x}}\ncalibration_parameters: {{cal_data_dir: calib}}\n')
out = yaml.safe_load(snapshot_config(ws,str(p)))
self.assertEqual(out['model_parameters']['onnx_model'], 'g/m.onnx')
self.assertEqual(out['calibration_parameters']['cal_data_dir'], 'calib')
for bad in ('../m.onnx', '/etc/m.onnx', '""', '3'):
p.write_text('model_parameters: {onnx_model: %s}\n' % bad)
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
p.write_text('calibration_parameters: {per_channel: true, bogus: 1}\n')
with self.assertRaisesRegex(ValueError, 'bogus'): snapshot_config(ws,str(p))
def test_status_symlink_does_not_write_target(self):
with tempfile.TemporaryDirectory() as d:
hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe')
(hd/'status.json').symlink_to(secret)
fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY)
try: publish(fd,{'state':'DONE'})
finally: os.close(fd)
self.assertEqual(secret.read_text(),'safe')
self.assertFalse((hd/'status.json').is_symlink())
def test_noop_production_rejected(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}')
Worker(rd,test_noop=True).scan()
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL')
def test_content_and_state_other_job(self):
with tempfile.TemporaryDirectory() as d:
jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir()
(jd/'job.json').write_text('{"threshold":0.9900}')
(jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}')
events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}]
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
s=audit.scan_events(rd,1)
for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1)
self.assertNotIn('0.9900',audit.oracle_needles(rd))
events[1]['result']={'content':[{'text':'path-guard: denied'}]}
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0)
def test_export_own_budget_and_version(self):
import sqlite3, sys
from harness.db import SCHEMA
with tempfile.TemporaryDirectory() as d:
root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5')
(root/'state').mkdir()
con=sqlite3.connect(root/'state/jobs.sqlite')
try:
con.executescript(SCHEMA)
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','job',1,'FAIL')")
con.commit()
finally:
con.close()
rd=root/'runs/r/job/rollouts/01'; (rd/'agent').mkdir(parents=True)
(rd/'rollout.json').write_text(json.dumps({
'agent_config_version':6, 'max_turns':60, 'max_tokens':100,
'state':'DONE', 'mode':'e2e', 'policy':'teacher',
'runner_version':'pi-sdk-v1', 'execution_id':'budget-fixture'}))
# A legacy verdict is historical metadata. The exporter must obtain
# both outcome and cost from this rollout's protected source files.
original_verdict=json.dumps({'schema':'bench.verdict/3','terminal':'PASS',
'run_id':'r','job_id':'job','rollout':1,'gates':{},
'cost':{'turns':0,'tokens':0}})
(rd/'verdict.json').write_text(original_verdict)
(rd.parents[1]/'job.json').write_text(json.dumps({
'run_id':'r', 'job_id':'job', 'task':'cls', 'march':'nash-m', 'backend':'ptq',
'gates':{'required':['G1','G3'], 'G1':{}, 'G2':{},
'G3':{'kind':'rel_pct','tol':5,'metric':'top1','require_source':'board'}, 'G4':{}}}))
stage=rd/'stages/05-compile'; stage.mkdir(parents=True)
(stage/'meta.json').write_text(json.dumps({'state':'FAIL','error_class':'compile_error'}))
system='Report only observed results.'
tools=[{'name':'bash','parameters':{'type':'object'}}]
(rd/'agent/system.md').write_text(system)
(rd/'agent/tools.json').write_text(json.dumps(tools))
messages=[{'role':'user','content':'Compile the model.'}]
events=[{'type':'session','version':3,'id':'budget-fixture','bench':{
'runner_version':'pi-sdk-v1','execution_id':'budget-fixture',
'system_prompt':system,'tools':tools}}, {'type':'agent_start'},
{'type':'message_end','message':messages[0]}]
for turn in range(30):
message={'role':'assistant','content':f'Compilation failed; diagnostic turn {turn + 1}.',
'usage':{'input':20 if turn == 29 else 0, 'output':1}}
messages.append(message)
events.extend([{'type':'message_end','message':message},
{'type':'turn_end','message':message}])
events.append({'type':'agent_end','messages':messages})
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
with patch.dict(os.environ, {'BENCH_STATE_DIR':str(root/'state')}), \
patch.object(exporter,'BENCH',root), \
patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']):
self.assertEqual(exporter.main(),0)
rec=json.loads((root/'datasets/test/teacher.jsonl').read_text())
self.assertEqual(rec['reward_terms']['cost'],-0.5)
self.assertEqual((rec['cost']['turns'],rec['cost']['tokens']),(30,50))
self.assertEqual(rec['verdict']['terminal'],'FAIL')
self.assertFalse(rec['sft_eligible'])
self.assertEqual((root/'datasets/test/sft.jsonl').read_text(),'')
self.assertEqual((rd/'verdict.json').read_text(),original_verdict)
manifest=json.loads((root/'datasets/test/manifest.json').read_text())
self.assertEqual(manifest['filters']['agent_version_mode'],'minimum')
self.assertEqual(manifest['filters']['min_agent_version'],5)
self.assertEqual(rec['agent_config_version'],'6')
self.assertEqual(manifest['stats']['written'],1)
self.assertEqual(manifest['stats']['reverified'],1)
def test_long_lane_preserved(self):
from harness import stagerun
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"05-compile"}')
ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={})
stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu'
stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={})
con=MagicMock()
with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'), patch.object(stagerun,'compile_attempts_used',return_value=0):
stagerun._exec(hd)
self.assertEqual(ctx.lane,'long')
self.assertEqual(con.execute.call_args[0][1][0],'agent-long')
def test_heartbeat_healthy_zombie_and_stall(self):
import sqlite3, time, io
from harness import heartbeat, db
from contextlib import redirect_stdout
with tempfile.TemporaryDirectory() as d:
root=Path(d); dbpath=root/'jobs.sqlite'
con=sqlite3.connect(dbpath);con.executescript(db.SCHEMA);con.commit()
cfg={'watchdog':{'stall_s':900,'cpu_idle_pct':5}}
with patch.object(heartbeat.paths,'DB_PATH',dbpath), patch.object(heartbeat.paths,'RUNS',root/'runs'), patch.object(heartbeat.paths,'config',return_value=cfg), patch.object(heartbeat,'processes',return_value={}), redirect_stdout(io.StringIO()) as out:
self.assertEqual(heartbeat.heartbeat('r'),0);self.assertEqual(out.getvalue(),'')
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state,started_at) VALUES('r','job',1,'RUNNING',?)",(time.time()-5000,));con.commit()
self.assertEqual(heartbeat.heartbeat('r'),1)
self.assertIn('ZOMBIE',out.getvalue());self.assertIn('events idle',out.getvalue())
con.execute("INSERT INTO stages(run_id,job_id,stage,state,lane,started_at) VALUES('r','job','05-compile','RUNNING','agent-long',?)",(time.time()-5000,));con.commit()
with patch.object(heartbeat,'matching',return_value=[42]), patch.object(heartbeat.watchdog,'_proc_tree_stat',return_value=(1000,0)), patch.object(heartbeat.time,'sleep'):
self.assertEqual(heartbeat.heartbeat('r'),1)
self.assertIn('STALLED stdout',out.getvalue())
con.close()
def test_non_tty_watch_once(self):
with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep:
self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called()
if __name__=='__main__': unittest.main()