data-gen-bpu/scripts/test_codex.py
bench f74fe00aaf hbm/bc 模拟器对比改为 agent 按需开启,只作诊断
阶段配置新增 verification_parameters.hbm_bc_samples(0–16,默认 0)。agent 用
bench stage run hbm --config <yaml> 指定 N 张校准样本,用 x86 模拟器对比 HBM 与 BC,
结果写在 G1.hbm_bc_cos_min,不改 verifier_ok:开了不会被扣分,真正把关的仍是 08-eval 板端实测。
编译配置只取三段,所以同一个 yaml 拿去编译也不受影响。适用任何有 .bc 的后端;跑失败记 None。
s06_hbm 随阶段打包,部署后马上生效(默认不跑);配置校验在 stageworker、说明在 SYSTEM.md,
都要下次重启时一起上线,否则按说明提交会被旧校验拒绝。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 12:48:17 +08:00

198 lines
13 KiB
Python

"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI."""
import importlib.util
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from harness.stageworker import Worker, config_bytes, publish, snapshot_config
from harness import watch
ROOT = Path(__file__).resolve().parents[1]
def module(name, path):
spec = importlib.util.spec_from_file_location(name, ROOT/path)
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m
exporter = module('exporter', 'export/to_agentrl.py')
audit = module('audit', 'scripts/audit_rollouts.py')
class Regression(unittest.TestCase):
def test_audit_does_not_offer_historical_repair_or_blocked_as_training(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'runs/r/job/rollouts/00';(rd/'agent').mkdir(parents=True)
(rd/'rollout.json').write_text('{"agent_config_version":7,"state":"DONE"}')
(rd/'agent/events.jsonl').write_text('{"type":"agent_end","messages":[]}')
for mode, terminal, expected in [('e2e','PASS',True),('repair','PASS',False),('e2e','BLOCKED',False)]:
(rd/'verdict.json').write_text(json.dumps({'terminal':terminal,'claim_consistent':True}))
(rd/'claim.json').write_text(json.dumps({'terminal':terminal}))
result=audit.audit_one(rd,0,7,mode)
self.assertEqual(result['rl_ok'],expected)
self.assertEqual(result['sft_ok'],expected)
def test_annotations_scope(self):
rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')]
self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'})
self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'})
self.assertFalse(exporter.annotation_reasons(rows,'run','other',2))
def test_config_symlink_and_traversal(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x')
(ws/'link').symlink_to('/etc/passwd')
self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x')
for p in [ws/'link', ws/'../outside']:
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p))
def test_worker_config_file_paths(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)
p=ws/'config.yaml';p.write_text('calibration_parameters:\n cal_data_dir: /state\n')
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
p.write_text('calibration_parameters:\n quant_config: /etc/passwd\n')
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
(ws/'q.json').write_text('{"model_config":{"all_node_type":"int8"}}')
p.write_text('calibration_parameters:\n quant_config: q.json\n')
self.assertIn(b'all_node_type: int8',snapshot_config(ws,str(p)))
p.write_text('verification_parameters: {hbm_bc_samples: 2}\n')
self.assertIn(b'hbm_bc_samples: 2',snapshot_config(ws,str(p)))
for bad in ('-1', '17', 'true', '"2"'):
p.write_text('verification_parameters: {hbm_bc_samples: %s}\n' % bad)
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
def test_status_symlink_does_not_write_target(self):
with tempfile.TemporaryDirectory() as d:
hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe')
(hd/'status.json').symlink_to(secret)
fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY)
try: publish(fd,{'state':'DONE'})
finally: os.close(fd)
self.assertEqual(secret.read_text(),'safe')
self.assertFalse((hd/'status.json').is_symlink())
def test_noop_production_rejected(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}')
Worker(rd,test_noop=True).scan()
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL')
def test_content_and_state_other_job(self):
with tempfile.TemporaryDirectory() as d:
jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir()
(jd/'job.json').write_text('{"threshold":0.9900}')
(jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}')
events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}]
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
s=audit.scan_events(rd,1)
for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1)
self.assertNotIn('0.9900',audit.oracle_needles(rd))
events[1]['result']={'content':[{'text':'path-guard: denied'}]}
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0)
def test_export_own_budget_and_version(self):
import sqlite3, sys
from harness.db import SCHEMA
with tempfile.TemporaryDirectory() as d:
root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5')
(root/'state').mkdir()
con=sqlite3.connect(root/'state/jobs.sqlite')
try:
con.executescript(SCHEMA)
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','job',1,'FAIL')")
con.commit()
finally:
con.close()
rd=root/'runs/r/job/rollouts/01'; (rd/'agent').mkdir(parents=True)
(rd/'rollout.json').write_text(json.dumps({
'agent_config_version':6, 'max_turns':60, 'max_tokens':100,
'state':'DONE', 'mode':'e2e', 'policy':'teacher',
'runner_version':'pi-sdk-v1', 'execution_id':'budget-fixture'}))
# A legacy verdict is historical metadata. The exporter must obtain
# both outcome and cost from this rollout's protected source files.
original_verdict=json.dumps({'schema':'bench.verdict/3','terminal':'PASS',
'run_id':'r','job_id':'job','rollout':1,'gates':{},
'cost':{'turns':0,'tokens':0}})
(rd/'verdict.json').write_text(original_verdict)
(rd.parents[1]/'job.json').write_text(json.dumps({
'run_id':'r', 'job_id':'job', 'task':'cls', 'march':'nash-m', 'backend':'ptq',
'gates':{'required':['G1','G3'], 'G1':{}, 'G2':{},
'G3':{'kind':'rel_pct','tol':5,'metric':'top1','require_source':'board'}, 'G4':{}}}))
stage=rd/'stages/05-compile'; stage.mkdir(parents=True)
(stage/'meta.json').write_text(json.dumps({'state':'FAIL','error_class':'compile_error'}))
system='Report only observed results.'
tools=[{'name':'bash','parameters':{'type':'object'}}]
(rd/'agent/system.md').write_text(system)
(rd/'agent/tools.json').write_text(json.dumps(tools))
messages=[{'role':'user','content':'Compile the model.'}]
events=[{'type':'session','version':3,'id':'budget-fixture','bench':{
'runner_version':'pi-sdk-v1','execution_id':'budget-fixture',
'system_prompt':system,'tools':tools}}, {'type':'agent_start'},
{'type':'message_end','message':messages[0]}]
for turn in range(30):
message={'role':'assistant','content':f'Compilation failed; diagnostic turn {turn + 1}.',
'usage':{'input':20 if turn == 29 else 0, 'output':1}}
messages.append(message)
events.extend([{'type':'message_end','message':message},
{'type':'turn_end','message':message}])
events.append({'type':'agent_end','messages':messages})
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
with patch.dict(os.environ, {'BENCH_STATE_DIR':str(root/'state')}), \
patch.object(exporter,'BENCH',root), \
patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']):
self.assertEqual(exporter.main(),0)
rec=json.loads((root/'datasets/test/teacher.jsonl').read_text())
self.assertEqual(rec['reward_terms']['cost'],-0.5)
self.assertEqual((rec['cost']['turns'],rec['cost']['tokens']),(30,50))
self.assertEqual(rec['verdict']['terminal'],'FAIL')
self.assertFalse(rec['sft_eligible'])
self.assertEqual((root/'datasets/test/sft.jsonl').read_text(),'')
self.assertEqual((rd/'verdict.json').read_text(),original_verdict)
manifest=json.loads((root/'datasets/test/manifest.json').read_text())
self.assertEqual(manifest['filters']['agent_version_mode'],'minimum')
self.assertEqual(manifest['filters']['min_agent_version'],5)
self.assertEqual(rec['agent_config_version'],'6')
self.assertEqual(manifest['stats']['written'],1)
self.assertEqual(manifest['stats']['reverified'],1)
def test_long_lane_preserved(self):
from harness import stagerun
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"05-compile"}')
ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={})
stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu'
stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={})
con=MagicMock()
with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'), patch.object(stagerun,'compile_attempts_used',return_value=0):
stagerun._exec(hd)
self.assertEqual(ctx.lane,'long')
self.assertEqual(con.execute.call_args[0][1][0],'agent-long')
def test_heartbeat_healthy_zombie_and_stall(self):
import sqlite3, time, io
from harness import heartbeat, db
from contextlib import redirect_stdout
with tempfile.TemporaryDirectory() as d:
root=Path(d); dbpath=root/'jobs.sqlite'
con=sqlite3.connect(dbpath);con.executescript(db.SCHEMA);con.commit()
cfg={'watchdog':{'stall_s':900,'cpu_idle_pct':5}}
with patch.object(heartbeat.paths,'DB_PATH',dbpath), patch.object(heartbeat.paths,'RUNS',root/'runs'), patch.object(heartbeat.paths,'config',return_value=cfg), patch.object(heartbeat,'processes',return_value={}), redirect_stdout(io.StringIO()) as out:
self.assertEqual(heartbeat.heartbeat('r'),0);self.assertEqual(out.getvalue(),'')
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state,started_at) VALUES('r','job',1,'RUNNING',?)",(time.time()-5000,));con.commit()
self.assertEqual(heartbeat.heartbeat('r'),1)
self.assertIn('ZOMBIE',out.getvalue());self.assertIn('events idle',out.getvalue())
con.execute("INSERT INTO stages(run_id,job_id,stage,state,lane,started_at) VALUES('r','job','05-compile','RUNNING','agent-long',?)",(time.time()-5000,));con.commit()
with patch.object(heartbeat,'matching',return_value=[42]), patch.object(heartbeat.watchdog,'_proc_tree_stat',return_value=(1000,0)), patch.object(heartbeat.time,'sleep'):
self.assertEqual(heartbeat.heartbeat('r'),1)
self.assertIn('STALLED stdout',out.getvalue())
con.close()
def test_non_tty_watch_once(self):
with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep:
self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called()
if __name__=='__main__': unittest.main()