data-gen-bpu/scripts/test_codex.py
bench a92510dab8 智能体开发环境与自带图:bench dev 在 Salty 沙箱里跑命令,compile.yaml 可用 workspace 里自己的图和校准集
- bench dev:受管终端的 development 目标改走 Salty 容器(有完整工具链)。
  容器里用 user+mount 命名空间、uid nobody、去掉全部能力,只露工具链/缓存/数据(只读)和本轨迹 workspace;
  板卡密钥、其他轨迹、个人空间其余内容不可见(10-14 在 Salty 上实测,含 umount 逃逸被挡)。
  本地改动下次 bench dev 时上传,小文件拷回,大文件留在开发环境并列出。
- compile.yaml:model_parameters.onnx_model / calibration_parameters.cal_data_dir 可指向 workspace;
  04 在容器里核对输入输出与平台图一致(不一致直接拒、不占编译次数),记录图的 sha256 和校准来源。
- 配置白名单报错列出具体字段;SYSTEM.md 写明开发环境和自带图;agent/VERSION 15→16。
- 生效:stageworker/terminal/stageclient/SYSTEM.md 要重启;新配置要加 salty_remote.stages: terminal、
  terminal.enabled: true、pools.dev。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:49:14 +08:00

209 lines
14 KiB
Python

"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI."""
import importlib.util
import json
import os
import tempfile
import unittest
import yaml
from pathlib import Path
from unittest.mock import patch
from harness.stageworker import Worker, config_bytes, publish, snapshot_config
from harness import watch
ROOT = Path(__file__).resolve().parents[1]
def module(name, path):
spec = importlib.util.spec_from_file_location(name, ROOT/path)
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m
exporter = module('exporter', 'export/to_agentrl.py')
audit = module('audit', 'scripts/audit_rollouts.py')
class Regression(unittest.TestCase):
def test_audit_does_not_offer_historical_repair_or_blocked_as_training(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'runs/r/job/rollouts/00';(rd/'agent').mkdir(parents=True)
(rd/'rollout.json').write_text('{"agent_config_version":7,"state":"DONE"}')
(rd/'agent/events.jsonl').write_text('{"type":"agent_end","messages":[]}')
for mode, terminal, expected in [('e2e','PASS',True),('repair','PASS',False),('e2e','BLOCKED',False)]:
(rd/'verdict.json').write_text(json.dumps({'terminal':terminal,'claim_consistent':True}))
(rd/'claim.json').write_text(json.dumps({'terminal':terminal}))
result=audit.audit_one(rd,0,7,mode)
self.assertEqual(result['rl_ok'],expected)
self.assertEqual(result['sft_ok'],expected)
def test_annotations_scope(self):
rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')]
self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'})
self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'})
self.assertFalse(exporter.annotation_reasons(rows,'run','other',2))
def test_config_symlink_and_traversal(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x')
(ws/'link').symlink_to('/etc/passwd')
self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x')
for p in [ws/'link', ws/'../outside']:
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p))
def test_worker_config_file_paths(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)
p=ws/'config.yaml';p.write_text('calibration_parameters:\n cal_data_dir: /state\n')
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
p.write_text('calibration_parameters:\n quant_config: /etc/passwd\n')
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
(ws/'q.json').write_text('{"model_config":{"all_node_type":"int8"}}')
p.write_text('calibration_parameters:\n quant_config: q.json\n')
self.assertIn(b'all_node_type: int8',snapshot_config(ws,str(p)))
p.write_text('verification_parameters: {hbm_bc_samples: 2}\n')
self.assertIn(b'hbm_bc_samples: 2',snapshot_config(ws,str(p)))
for bad in ('-1', '17', 'true', '"2"'):
p.write_text('verification_parameters: {hbm_bc_samples: %s}\n' % bad)
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
# The agent's own graph/calibration: kept workspace-relative (04 resolves them in bench dev's copy).
p.write_text(f'model_parameters: {{onnx_model: {ws}/g/m.onnx, march: x}}\ncalibration_parameters: {{cal_data_dir: calib}}\n')
out = yaml.safe_load(snapshot_config(ws,str(p)))
self.assertEqual(out['model_parameters']['onnx_model'], 'g/m.onnx')
self.assertEqual(out['calibration_parameters']['cal_data_dir'], 'calib')
for bad in ('../m.onnx', '/etc/m.onnx', '""', '3'):
p.write_text('model_parameters: {onnx_model: %s}\n' % bad)
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
p.write_text('calibration_parameters: {per_channel: true, bogus: 1}\n')
with self.assertRaisesRegex(ValueError, 'bogus'): snapshot_config(ws,str(p))
def test_status_symlink_does_not_write_target(self):
with tempfile.TemporaryDirectory() as d:
hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe')
(hd/'status.json').symlink_to(secret)
fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY)
try: publish(fd,{'state':'DONE'})
finally: os.close(fd)
self.assertEqual(secret.read_text(),'safe')
self.assertFalse((hd/'status.json').is_symlink())
def test_noop_production_rejected(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}')
Worker(rd,test_noop=True).scan()
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL')
def test_content_and_state_other_job(self):
with tempfile.TemporaryDirectory() as d:
jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir()
(jd/'job.json').write_text('{"threshold":0.9900}')
(jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}')
events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}]
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
s=audit.scan_events(rd,1)
for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1)
self.assertNotIn('0.9900',audit.oracle_needles(rd))
events[1]['result']={'content':[{'text':'path-guard: denied'}]}
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0)
def test_export_own_budget_and_version(self):
import sqlite3, sys
from harness.db import SCHEMA
with tempfile.TemporaryDirectory() as d:
root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5')
(root/'state').mkdir()
con=sqlite3.connect(root/'state/jobs.sqlite')
try:
con.executescript(SCHEMA)
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','job',1,'FAIL')")
con.commit()
finally:
con.close()
rd=root/'runs/r/job/rollouts/01'; (rd/'agent').mkdir(parents=True)
(rd/'rollout.json').write_text(json.dumps({
'agent_config_version':6, 'max_turns':60, 'max_tokens':100,
'state':'DONE', 'mode':'e2e', 'policy':'teacher',
'runner_version':'pi-sdk-v1', 'execution_id':'budget-fixture'}))
# A legacy verdict is historical metadata. The exporter must obtain
# both outcome and cost from this rollout's protected source files.
original_verdict=json.dumps({'schema':'bench.verdict/3','terminal':'PASS',
'run_id':'r','job_id':'job','rollout':1,'gates':{},
'cost':{'turns':0,'tokens':0}})
(rd/'verdict.json').write_text(original_verdict)
(rd.parents[1]/'job.json').write_text(json.dumps({
'run_id':'r', 'job_id':'job', 'task':'cls', 'march':'nash-m', 'backend':'ptq',
'gates':{'required':['G1','G3'], 'G1':{}, 'G2':{},
'G3':{'kind':'rel_pct','tol':5,'metric':'top1','require_source':'board'}, 'G4':{}}}))
stage=rd/'stages/05-compile'; stage.mkdir(parents=True)
(stage/'meta.json').write_text(json.dumps({'state':'FAIL','error_class':'compile_error'}))
system='Report only observed results.'
tools=[{'name':'bash','parameters':{'type':'object'}}]
(rd/'agent/system.md').write_text(system)
(rd/'agent/tools.json').write_text(json.dumps(tools))
messages=[{'role':'user','content':'Compile the model.'}]
events=[{'type':'session','version':3,'id':'budget-fixture','bench':{
'runner_version':'pi-sdk-v1','execution_id':'budget-fixture',
'system_prompt':system,'tools':tools}}, {'type':'agent_start'},
{'type':'message_end','message':messages[0]}]
for turn in range(30):
message={'role':'assistant','content':f'Compilation failed; diagnostic turn {turn + 1}.',
'usage':{'input':20 if turn == 29 else 0, 'output':1}}
messages.append(message)
events.extend([{'type':'message_end','message':message},
{'type':'turn_end','message':message}])
events.append({'type':'agent_end','messages':messages})
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
with patch.dict(os.environ, {'BENCH_STATE_DIR':str(root/'state')}), \
patch.object(exporter,'BENCH',root), \
patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']):
self.assertEqual(exporter.main(),0)
rec=json.loads((root/'datasets/test/teacher.jsonl').read_text())
self.assertEqual(rec['reward_terms']['cost'],-0.5)
self.assertEqual((rec['cost']['turns'],rec['cost']['tokens']),(30,50))
self.assertEqual(rec['verdict']['terminal'],'FAIL')
self.assertFalse(rec['sft_eligible'])
self.assertEqual((root/'datasets/test/sft.jsonl').read_text(),'')
self.assertEqual((rd/'verdict.json').read_text(),original_verdict)
manifest=json.loads((root/'datasets/test/manifest.json').read_text())
self.assertEqual(manifest['filters']['agent_version_mode'],'minimum')
self.assertEqual(manifest['filters']['min_agent_version'],5)
self.assertEqual(rec['agent_config_version'],'6')
self.assertEqual(manifest['stats']['written'],1)
self.assertEqual(manifest['stats']['reverified'],1)
def test_long_lane_preserved(self):
from harness import stagerun
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"05-compile"}')
ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={})
stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu'
stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={})
con=MagicMock()
with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'), patch.object(stagerun,'compile_attempts_used',return_value=0):
stagerun._exec(hd)
self.assertEqual(ctx.lane,'long')
self.assertEqual(con.execute.call_args[0][1][0],'agent-long')
def test_heartbeat_healthy_zombie_and_stall(self):
import sqlite3, time, io
from harness import heartbeat, db
from contextlib import redirect_stdout
with tempfile.TemporaryDirectory() as d:
root=Path(d); dbpath=root/'jobs.sqlite'
con=sqlite3.connect(dbpath);con.executescript(db.SCHEMA);con.commit()
cfg={'watchdog':{'stall_s':900,'cpu_idle_pct':5}}
with patch.object(heartbeat.paths,'DB_PATH',dbpath), patch.object(heartbeat.paths,'RUNS',root/'runs'), patch.object(heartbeat.paths,'config',return_value=cfg), patch.object(heartbeat,'processes',return_value={}), redirect_stdout(io.StringIO()) as out:
self.assertEqual(heartbeat.heartbeat('r'),0);self.assertEqual(out.getvalue(),'')
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state,started_at) VALUES('r','job',1,'RUNNING',?)",(time.time()-5000,));con.commit()
self.assertEqual(heartbeat.heartbeat('r'),1)
self.assertIn('ZOMBIE',out.getvalue());self.assertIn('events idle',out.getvalue())
con.execute("INSERT INTO stages(run_id,job_id,stage,state,lane,started_at) VALUES('r','job','05-compile','RUNNING','agent-long',?)",(time.time()-5000,));con.commit()
with patch.object(heartbeat,'matching',return_value=[42]), patch.object(heartbeat.watchdog,'_proc_tree_stat',return_value=(1000,0)), patch.object(heartbeat.time,'sleep'):
self.assertEqual(heartbeat.heartbeat('r'),1)
self.assertIn('STALLED stdout',out.getvalue())
con.close()
def test_non_tty_watch_once(self):
with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep:
self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called()
if __name__=='__main__': unittest.main()