data-gen-bpu/scripts/test_codex.py
ming01.wang 4332f43dab feat: cap each trajectory at three started compiles
Compile attempts are now the primary per-trajectory budget: they map to
the real compute cost, unlike turns or tokens, and make the stop reason
obvious. stagerun refuses a 4th 05-compile with COMPILE_BUDGET_EXHAUSTED
before inline calibration or slot acquisition.

- Counted from the stages ledger (05-compile rows that got a slot), so
  resume/recovery cannot reset it; a compile that timed out in the queue
  is free; recompiles after an accuracy miss count.
- Board/eval/review of the last artifact still run after the cap.
- Turn/token/wall budgets, per-phase deadlines, cleanup and API quota
  protection are unchanged as backstops.
- e2e prompt drops "no retry limit"; SYSTEM.md names compiles as the
  primary budget; agent/VERSION 12 -> 13.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 22:37:15 +08:00

193 lines
12 KiB
Python

"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI."""
import importlib.util
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from harness.stageworker import Worker, config_bytes, publish, snapshot_config
from harness import watch
ROOT = Path(__file__).resolve().parents[1]
def module(name, path):
spec = importlib.util.spec_from_file_location(name, ROOT/path)
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m
exporter = module('exporter', 'export/to_agentrl.py')
audit = module('audit', 'scripts/audit_rollouts.py')
class Regression(unittest.TestCase):
def test_audit_does_not_offer_historical_repair_or_blocked_as_training(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'runs/r/job/rollouts/00';(rd/'agent').mkdir(parents=True)
(rd/'rollout.json').write_text('{"agent_config_version":7,"state":"DONE"}')
(rd/'agent/events.jsonl').write_text('{"type":"agent_end","messages":[]}')
for mode, terminal, expected in [('e2e','PASS',True),('repair','PASS',False),('e2e','BLOCKED',False)]:
(rd/'verdict.json').write_text(json.dumps({'terminal':terminal,'claim_consistent':True}))
(rd/'claim.json').write_text(json.dumps({'terminal':terminal}))
result=audit.audit_one(rd,0,7,mode)
self.assertEqual(result['rl_ok'],expected)
self.assertEqual(result['sft_ok'],expected)
def test_annotations_scope(self):
rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')]
self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'})
self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'})
self.assertFalse(exporter.annotation_reasons(rows,'run','other',2))
def test_config_symlink_and_traversal(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x')
(ws/'link').symlink_to('/etc/passwd')
self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x')
for p in [ws/'link', ws/'../outside']:
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p))
def test_worker_config_file_paths(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)
p=ws/'config.yaml';p.write_text('calibration_parameters:\n cal_data_dir: /state\n')
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
p.write_text('calibration_parameters:\n quant_config: /etc/passwd\n')
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
(ws/'q.json').write_text('{"model_config":{"all_node_type":"int8"}}')
p.write_text('calibration_parameters:\n quant_config: q.json\n')
self.assertIn(b'all_node_type: int8',snapshot_config(ws,str(p)))
def test_status_symlink_does_not_write_target(self):
with tempfile.TemporaryDirectory() as d:
hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe')
(hd/'status.json').symlink_to(secret)
fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY)
try: publish(fd,{'state':'DONE'})
finally: os.close(fd)
self.assertEqual(secret.read_text(),'safe')
self.assertFalse((hd/'status.json').is_symlink())
def test_noop_production_rejected(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}')
Worker(rd,test_noop=True).scan()
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL')
def test_content_and_state_other_job(self):
with tempfile.TemporaryDirectory() as d:
jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir()
(jd/'job.json').write_text('{"threshold":0.9900}')
(jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}')
events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}]
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
s=audit.scan_events(rd,1)
for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1)
self.assertNotIn('0.9900',audit.oracle_needles(rd))
events[1]['result']={'content':[{'text':'path-guard: denied'}]}
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0)
def test_export_own_budget_and_version(self):
import sqlite3, sys
from harness.db import SCHEMA
with tempfile.TemporaryDirectory() as d:
root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5')
(root/'state').mkdir()
con=sqlite3.connect(root/'state/jobs.sqlite')
try:
con.executescript(SCHEMA)
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','job',1,'FAIL')")
con.commit()
finally:
con.close()
rd=root/'runs/r/job/rollouts/01'; (rd/'agent').mkdir(parents=True)
(rd/'rollout.json').write_text(json.dumps({
'agent_config_version':6, 'max_turns':60, 'max_tokens':100,
'state':'DONE', 'mode':'e2e', 'policy':'teacher',
'runner_version':'pi-sdk-v1', 'execution_id':'budget-fixture'}))
# A legacy verdict is historical metadata. The exporter must obtain
# both outcome and cost from this rollout's protected source files.
original_verdict=json.dumps({'schema':'bench.verdict/3','terminal':'PASS',
'run_id':'r','job_id':'job','rollout':1,'gates':{},
'cost':{'turns':0,'tokens':0}})
(rd/'verdict.json').write_text(original_verdict)
(rd.parents[1]/'job.json').write_text(json.dumps({
'run_id':'r', 'job_id':'job', 'task':'cls', 'march':'nash-m', 'backend':'ptq',
'gates':{'required':['G1','G3'], 'G1':{}, 'G2':{},
'G3':{'kind':'rel_pct','tol':5,'metric':'top1','require_source':'board'}, 'G4':{}}}))
stage=rd/'stages/05-compile'; stage.mkdir(parents=True)
(stage/'meta.json').write_text(json.dumps({'state':'FAIL','error_class':'compile_error'}))
system='Report only observed results.'
tools=[{'name':'bash','parameters':{'type':'object'}}]
(rd/'agent/system.md').write_text(system)
(rd/'agent/tools.json').write_text(json.dumps(tools))
messages=[{'role':'user','content':'Compile the model.'}]
events=[{'type':'session','version':3,'id':'budget-fixture','bench':{
'runner_version':'pi-sdk-v1','execution_id':'budget-fixture',
'system_prompt':system,'tools':tools}}, {'type':'agent_start'},
{'type':'message_end','message':messages[0]}]
for turn in range(30):
message={'role':'assistant','content':f'Compilation failed; diagnostic turn {turn + 1}.',
'usage':{'input':20 if turn == 29 else 0, 'output':1}}
messages.append(message)
events.extend([{'type':'message_end','message':message},
{'type':'turn_end','message':message}])
events.append({'type':'agent_end','messages':messages})
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
with patch.dict(os.environ, {'BENCH_STATE_DIR':str(root/'state')}), \
patch.object(exporter,'BENCH',root), \
patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']):
self.assertEqual(exporter.main(),0)
rec=json.loads((root/'datasets/test/teacher.jsonl').read_text())
self.assertEqual(rec['reward_terms']['cost'],-0.5)
self.assertEqual((rec['cost']['turns'],rec['cost']['tokens']),(30,50))
self.assertEqual(rec['verdict']['terminal'],'FAIL')
self.assertFalse(rec['sft_eligible'])
self.assertEqual((root/'datasets/test/sft.jsonl').read_text(),'')
self.assertEqual((rd/'verdict.json').read_text(),original_verdict)
manifest=json.loads((root/'datasets/test/manifest.json').read_text())
self.assertEqual(manifest['filters']['agent_version_mode'],'minimum')
self.assertEqual(manifest['filters']['min_agent_version'],5)
self.assertEqual(rec['agent_config_version'],'6')
self.assertEqual(manifest['stats']['written'],1)
self.assertEqual(manifest['stats']['reverified'],1)
def test_long_lane_preserved(self):
from harness import stagerun
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"05-compile"}')
ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={})
stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu'
stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={})
con=MagicMock()
with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'), patch.object(stagerun,'compile_attempts_used',return_value=0):
stagerun._exec(hd)
self.assertEqual(ctx.lane,'long')
self.assertEqual(con.execute.call_args[0][1][0],'agent-long')
def test_heartbeat_healthy_zombie_and_stall(self):
import sqlite3, time, io
from harness import heartbeat, db
from contextlib import redirect_stdout
with tempfile.TemporaryDirectory() as d:
root=Path(d); dbpath=root/'jobs.sqlite'
con=sqlite3.connect(dbpath);con.executescript(db.SCHEMA);con.commit()
cfg={'watchdog':{'stall_s':900,'cpu_idle_pct':5}}
with patch.object(heartbeat.paths,'DB_PATH',dbpath), patch.object(heartbeat.paths,'RUNS',root/'runs'), patch.object(heartbeat.paths,'config',return_value=cfg), patch.object(heartbeat,'processes',return_value={}), redirect_stdout(io.StringIO()) as out:
self.assertEqual(heartbeat.heartbeat('r'),0);self.assertEqual(out.getvalue(),'')
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state,started_at) VALUES('r','job',1,'RUNNING',?)",(time.time()-5000,));con.commit()
self.assertEqual(heartbeat.heartbeat('r'),1)
self.assertIn('ZOMBIE',out.getvalue());self.assertIn('events idle',out.getvalue())
con.execute("INSERT INTO stages(run_id,job_id,stage,state,lane,started_at) VALUES('r','job','05-compile','RUNNING','agent-long',?)",(time.time()-5000,));con.commit()
with patch.object(heartbeat,'matching',return_value=[42]), patch.object(heartbeat.watchdog,'_proc_tree_stat',return_value=(1000,0)), patch.object(heartbeat.time,'sleep'):
self.assertEqual(heartbeat.heartbeat('r'),1)
self.assertIn('STALLED stdout',out.getvalue())
con.close()
def test_non_tty_watch_once(self):
with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep:
self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called()
if __name__=='__main__': unittest.main()