193 lines
12 KiB
Python
193 lines
12 KiB
Python
"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI."""
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from harness.stageworker import Worker, config_bytes, publish, snapshot_config
|
|
from harness import watch
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
def module(name, path):
|
|
spec = importlib.util.spec_from_file_location(name, ROOT/path)
|
|
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m
|
|
exporter = module('exporter', 'export/to_agentrl.py')
|
|
audit = module('audit', 'scripts/audit_rollouts.py')
|
|
|
|
class Regression(unittest.TestCase):
|
|
def test_audit_does_not_offer_historical_repair_or_blocked_as_training(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
rd=Path(d)/'runs/r/job/rollouts/00';(rd/'agent').mkdir(parents=True)
|
|
(rd/'rollout.json').write_text('{"agent_config_version":7,"state":"DONE"}')
|
|
(rd/'agent/events.jsonl').write_text('{"type":"agent_end","messages":[]}')
|
|
for mode, terminal, expected in [('e2e','PASS',True),('repair','PASS',False),('e2e','BLOCKED',False)]:
|
|
(rd/'verdict.json').write_text(json.dumps({'terminal':terminal,'claim_consistent':True}))
|
|
(rd/'claim.json').write_text(json.dumps({'terminal':terminal}))
|
|
result=audit.audit_one(rd,0,7,mode)
|
|
self.assertEqual(result['rl_ok'],expected)
|
|
self.assertEqual(result['sft_ok'],expected)
|
|
|
|
def test_annotations_scope(self):
|
|
rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')]
|
|
self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'})
|
|
self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'})
|
|
self.assertFalse(exporter.annotation_reasons(rows,'run','other',2))
|
|
|
|
def test_config_symlink_and_traversal(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x')
|
|
(ws/'link').symlink_to('/etc/passwd')
|
|
self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x')
|
|
for p in [ws/'link', ws/'../outside']:
|
|
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p))
|
|
|
|
def test_worker_config_file_paths(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
ws=Path(d)
|
|
p=ws/'config.yaml';p.write_text('calibration_parameters:\n cal_data_dir: /state\n')
|
|
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
|
|
p.write_text('calibration_parameters:\n quant_config: /etc/passwd\n')
|
|
with self.assertRaises(ValueError): snapshot_config(ws,str(p))
|
|
(ws/'q.json').write_text('{"model_config":{"all_node_type":"int8"}}')
|
|
p.write_text('calibration_parameters:\n quant_config: q.json\n')
|
|
self.assertIn(b'all_node_type: int8',snapshot_config(ws,str(p)))
|
|
|
|
def test_status_symlink_does_not_write_target(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe')
|
|
(hd/'status.json').symlink_to(secret)
|
|
fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY)
|
|
try: publish(fd,{'state':'DONE'})
|
|
finally: os.close(fd)
|
|
self.assertEqual(secret.read_text(),'safe')
|
|
self.assertFalse((hd/'status.json').is_symlink())
|
|
|
|
def test_noop_production_rejected(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}')
|
|
Worker(rd,test_noop=True).scan()
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL')
|
|
|
|
def test_content_and_state_other_job(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir()
|
|
(jd/'job.json').write_text('{"threshold":0.9900}')
|
|
(jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}')
|
|
events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}]
|
|
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
|
|
s=audit.scan_events(rd,1)
|
|
for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1)
|
|
self.assertNotIn('0.9900',audit.oracle_needles(rd))
|
|
events[1]['result']={'content':[{'text':'path-guard: denied'}]}
|
|
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
|
|
self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0)
|
|
|
|
def test_export_own_budget_and_version(self):
|
|
import sqlite3, sys
|
|
from harness.db import SCHEMA
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5')
|
|
(root/'state').mkdir()
|
|
con=sqlite3.connect(root/'state/jobs.sqlite')
|
|
try:
|
|
con.executescript(SCHEMA)
|
|
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','job',1,'FAIL')")
|
|
con.commit()
|
|
finally:
|
|
con.close()
|
|
rd=root/'runs/r/job/rollouts/01'; (rd/'agent').mkdir(parents=True)
|
|
(rd/'rollout.json').write_text(json.dumps({
|
|
'agent_config_version':6, 'max_turns':60, 'max_tokens':100,
|
|
'state':'DONE', 'mode':'e2e', 'policy':'teacher',
|
|
'runner_version':'pi-sdk-v1', 'execution_id':'budget-fixture'}))
|
|
# A legacy verdict is historical metadata. The exporter must obtain
|
|
# both outcome and cost from this rollout's protected source files.
|
|
original_verdict=json.dumps({'schema':'bench.verdict/3','terminal':'PASS',
|
|
'run_id':'r','job_id':'job','rollout':1,'gates':{},
|
|
'cost':{'turns':0,'tokens':0}})
|
|
(rd/'verdict.json').write_text(original_verdict)
|
|
(rd.parents[1]/'job.json').write_text(json.dumps({
|
|
'run_id':'r', 'job_id':'job', 'task':'cls', 'march':'nash-m', 'backend':'ptq',
|
|
'gates':{'required':['G1','G3'], 'G1':{}, 'G2':{},
|
|
'G3':{'kind':'rel_pct','tol':5,'metric':'top1','require_source':'board'}, 'G4':{}}}))
|
|
stage=rd/'stages/05-compile'; stage.mkdir(parents=True)
|
|
(stage/'meta.json').write_text(json.dumps({'state':'FAIL','error_class':'compile_error'}))
|
|
system='Report only observed results.'
|
|
tools=[{'name':'bash','parameters':{'type':'object'}}]
|
|
(rd/'agent/system.md').write_text(system)
|
|
(rd/'agent/tools.json').write_text(json.dumps(tools))
|
|
messages=[{'role':'user','content':'Compile the model.'}]
|
|
events=[{'type':'session','version':3,'id':'budget-fixture','bench':{
|
|
'runner_version':'pi-sdk-v1','execution_id':'budget-fixture',
|
|
'system_prompt':system,'tools':tools}}, {'type':'agent_start'},
|
|
{'type':'message_end','message':messages[0]}]
|
|
for turn in range(30):
|
|
message={'role':'assistant','content':f'Compilation failed; diagnostic turn {turn + 1}.',
|
|
'usage':{'input':20 if turn == 29 else 0, 'output':1}}
|
|
messages.append(message)
|
|
events.extend([{'type':'message_end','message':message},
|
|
{'type':'turn_end','message':message}])
|
|
events.append({'type':'agent_end','messages':messages})
|
|
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
|
|
with patch.dict(os.environ, {'BENCH_STATE_DIR':str(root/'state')}), \
|
|
patch.object(exporter,'BENCH',root), \
|
|
patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']):
|
|
self.assertEqual(exporter.main(),0)
|
|
rec=json.loads((root/'datasets/test/teacher.jsonl').read_text())
|
|
self.assertEqual(rec['reward_terms']['cost'],-0.5)
|
|
self.assertEqual((rec['cost']['turns'],rec['cost']['tokens']),(30,50))
|
|
self.assertEqual(rec['verdict']['terminal'],'FAIL')
|
|
self.assertFalse(rec['sft_eligible'])
|
|
self.assertEqual((root/'datasets/test/sft.jsonl').read_text(),'')
|
|
self.assertEqual((rd/'verdict.json').read_text(),original_verdict)
|
|
manifest=json.loads((root/'datasets/test/manifest.json').read_text())
|
|
self.assertEqual(manifest['filters']['agent_version_mode'],'minimum')
|
|
self.assertEqual(manifest['filters']['min_agent_version'],5)
|
|
self.assertEqual(rec['agent_config_version'],'6')
|
|
self.assertEqual(manifest['stats']['written'],1)
|
|
self.assertEqual(manifest['stats']['reverified'],1)
|
|
|
|
def test_long_lane_preserved(self):
|
|
from harness import stagerun
|
|
from contextlib import nullcontext
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
with tempfile.TemporaryDirectory() as d:
|
|
rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True)
|
|
(hd/'request.json').write_text('{"stage":"05-compile"}')
|
|
ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={})
|
|
stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu'
|
|
stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={})
|
|
con=MagicMock()
|
|
with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'):
|
|
stagerun._exec(hd)
|
|
self.assertEqual(ctx.lane,'long')
|
|
self.assertEqual(con.execute.call_args[0][1][0],'agent-long')
|
|
|
|
def test_heartbeat_healthy_zombie_and_stall(self):
|
|
import sqlite3, time, io
|
|
from harness import heartbeat, db
|
|
from contextlib import redirect_stdout
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d); dbpath=root/'jobs.sqlite'
|
|
con=sqlite3.connect(dbpath);con.executescript(db.SCHEMA);con.commit()
|
|
cfg={'watchdog':{'stall_s':900,'cpu_idle_pct':5}}
|
|
with patch.object(heartbeat.paths,'DB_PATH',dbpath), patch.object(heartbeat.paths,'RUNS',root/'runs'), patch.object(heartbeat.paths,'config',return_value=cfg), patch.object(heartbeat,'processes',return_value={}), redirect_stdout(io.StringIO()) as out:
|
|
self.assertEqual(heartbeat.heartbeat('r'),0);self.assertEqual(out.getvalue(),'')
|
|
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state,started_at) VALUES('r','job',1,'RUNNING',?)",(time.time()-5000,));con.commit()
|
|
self.assertEqual(heartbeat.heartbeat('r'),1)
|
|
self.assertIn('ZOMBIE',out.getvalue());self.assertIn('events idle',out.getvalue())
|
|
con.execute("INSERT INTO stages(run_id,job_id,stage,state,lane,started_at) VALUES('r','job','05-compile','RUNNING','agent-long',?)",(time.time()-5000,));con.commit()
|
|
with patch.object(heartbeat,'matching',return_value=[42]), patch.object(heartbeat.watchdog,'_proc_tree_stat',return_value=(1000,0)), patch.object(heartbeat.time,'sleep'):
|
|
self.assertEqual(heartbeat.heartbeat('r'),1)
|
|
self.assertIn('STALLED stdout',out.getvalue())
|
|
con.close()
|
|
|
|
def test_non_tty_watch_once(self):
|
|
with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep:
|
|
self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called()
|
|
|
|
if __name__=='__main__': unittest.main()
|