"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI.""" import importlib.util import json import os import tempfile import unittest from pathlib import Path from unittest.mock import patch from harness.stageworker import Worker, config_bytes, publish, snapshot_config from harness import watch ROOT = Path(__file__).resolve().parents[1] def module(name, path): spec = importlib.util.spec_from_file_location(name, ROOT/path) m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m exporter = module('exporter', 'export/to_agentrl.py') audit = module('audit', 'scripts/audit_rollouts.py') class Regression(unittest.TestCase): def test_audit_does_not_offer_historical_repair_or_blocked_as_training(self): with tempfile.TemporaryDirectory() as d: rd=Path(d)/'runs/r/job/rollouts/00';(rd/'agent').mkdir(parents=True) (rd/'rollout.json').write_text('{"agent_config_version":7,"state":"DONE"}') (rd/'agent/events.jsonl').write_text('{"type":"agent_end","messages":[]}') for mode, terminal, expected in [('e2e','PASS',True),('repair','PASS',False),('e2e','BLOCKED',False)]: (rd/'verdict.json').write_text(json.dumps({'terminal':terminal,'claim_consistent':True})) (rd/'claim.json').write_text(json.dumps({'terminal':terminal})) result=audit.audit_one(rd,0,7,mode) self.assertEqual(result['rl_ok'],expected) self.assertEqual(result['sft_ok'],expected) def test_annotations_scope(self): rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')] self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'}) self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'}) self.assertFalse(exporter.annotation_reasons(rows,'run','other',2)) def test_config_symlink_and_traversal(self): with tempfile.TemporaryDirectory() as d: ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x') (ws/'link').symlink_to('/etc/passwd') self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x') for p in [ws/'link', ws/'../outside']: with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p)) def test_worker_config_file_paths(self): with tempfile.TemporaryDirectory() as d: ws=Path(d) p=ws/'config.yaml';p.write_text('calibration_parameters:\n cal_data_dir: /state\n') with self.assertRaises(ValueError): snapshot_config(ws,str(p)) p.write_text('calibration_parameters:\n quant_config: /etc/passwd\n') with self.assertRaises(ValueError): snapshot_config(ws,str(p)) (ws/'q.json').write_text('{"model_config":{"all_node_type":"int8"}}') p.write_text('calibration_parameters:\n quant_config: q.json\n') self.assertIn(b'all_node_type: int8',snapshot_config(ws,str(p))) def test_status_symlink_does_not_write_target(self): with tempfile.TemporaryDirectory() as d: hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe') (hd/'status.json').symlink_to(secret) fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY) try: publish(fd,{'state':'DONE'}) finally: os.close(fd) self.assertEqual(secret.read_text(),'safe') self.assertFalse((hd/'status.json').is_symlink()) def test_noop_production_rejected(self): with tempfile.TemporaryDirectory() as d: rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}') Worker(rd,test_noop=True).scan() self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL') def test_content_and_state_other_job(self): with tempfile.TemporaryDirectory() as d: jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir() (jd/'job.json').write_text('{"threshold":0.9900}') (jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}') events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}] (rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events))) s=audit.scan_events(rd,1) for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1) self.assertNotIn('0.9900',audit.oracle_needles(rd)) events[1]['result']={'content':[{'text':'path-guard: denied'}]} (rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events))) self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0) def test_export_own_budget_and_version(self): import sqlite3, sys from harness.db import SCHEMA with tempfile.TemporaryDirectory() as d: root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5') (root/'state').mkdir() con=sqlite3.connect(root/'state/jobs.sqlite') try: con.executescript(SCHEMA) con.execute("INSERT INTO rollouts(run_id,job_id,kk,state) VALUES('r','job',1,'FAIL')") con.commit() finally: con.close() rd=root/'runs/r/job/rollouts/01'; (rd/'agent').mkdir(parents=True) (rd/'rollout.json').write_text(json.dumps({ 'agent_config_version':6, 'max_turns':60, 'max_tokens':100, 'state':'DONE', 'mode':'e2e', 'policy':'teacher', 'runner_version':'pi-sdk-v1', 'execution_id':'budget-fixture'})) # A legacy verdict is historical metadata. The exporter must obtain # both outcome and cost from this rollout's protected source files. original_verdict=json.dumps({'schema':'bench.verdict/3','terminal':'PASS', 'run_id':'r','job_id':'job','rollout':1,'gates':{}, 'cost':{'turns':0,'tokens':0}}) (rd/'verdict.json').write_text(original_verdict) (rd.parents[1]/'job.json').write_text(json.dumps({ 'run_id':'r', 'job_id':'job', 'task':'cls', 'march':'nash-m', 'backend':'ptq', 'gates':{'required':['G1','G3'], 'G1':{}, 'G2':{}, 'G3':{'kind':'rel_pct','tol':5,'metric':'top1','require_source':'board'}, 'G4':{}}})) stage=rd/'stages/05-compile'; stage.mkdir(parents=True) (stage/'meta.json').write_text(json.dumps({'state':'FAIL','error_class':'compile_error'})) system='Report only observed results.' tools=[{'name':'bash','parameters':{'type':'object'}}] (rd/'agent/system.md').write_text(system) (rd/'agent/tools.json').write_text(json.dumps(tools)) messages=[{'role':'user','content':'Compile the model.'}] events=[{'type':'session','version':3,'id':'budget-fixture','bench':{ 'runner_version':'pi-sdk-v1','execution_id':'budget-fixture', 'system_prompt':system,'tools':tools}}, {'type':'agent_start'}, {'type':'message_end','message':messages[0]}] for turn in range(30): message={'role':'assistant','content':f'Compilation failed; diagnostic turn {turn + 1}.', 'usage':{'input':20 if turn == 29 else 0, 'output':1}} messages.append(message) events.extend([{'type':'message_end','message':message}, {'type':'turn_end','message':message}]) events.append({'type':'agent_end','messages':messages}) (rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events))) with patch.dict(os.environ, {'BENCH_STATE_DIR':str(root/'state')}), \ patch.object(exporter,'BENCH',root), \ patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']): self.assertEqual(exporter.main(),0) rec=json.loads((root/'datasets/test/teacher.jsonl').read_text()) self.assertEqual(rec['reward_terms']['cost'],-0.5) self.assertEqual((rec['cost']['turns'],rec['cost']['tokens']),(30,50)) self.assertEqual(rec['verdict']['terminal'],'FAIL') self.assertFalse(rec['sft_eligible']) self.assertEqual((root/'datasets/test/sft.jsonl').read_text(),'') self.assertEqual((rd/'verdict.json').read_text(),original_verdict) manifest=json.loads((root/'datasets/test/manifest.json').read_text()) self.assertEqual(manifest['filters']['agent_version_mode'],'minimum') self.assertEqual(manifest['filters']['min_agent_version'],5) self.assertEqual(rec['agent_config_version'],'6') self.assertEqual(manifest['stats']['written'],1) self.assertEqual(manifest['stats']['reverified'],1) def test_long_lane_preserved(self): from harness import stagerun from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import MagicMock with tempfile.TemporaryDirectory() as d: rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True) (hd/'request.json').write_text('{"stage":"05-compile"}') ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={}) stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu' stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={}) con=MagicMock() with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'), patch.object(stagerun,'compile_attempts_used',return_value=0): stagerun._exec(hd) self.assertEqual(ctx.lane,'long') self.assertEqual(con.execute.call_args[0][1][0],'agent-long') def test_heartbeat_healthy_zombie_and_stall(self): import sqlite3, time, io from harness import heartbeat, db from contextlib import redirect_stdout with tempfile.TemporaryDirectory() as d: root=Path(d); dbpath=root/'jobs.sqlite' con=sqlite3.connect(dbpath);con.executescript(db.SCHEMA);con.commit() cfg={'watchdog':{'stall_s':900,'cpu_idle_pct':5}} with patch.object(heartbeat.paths,'DB_PATH',dbpath), patch.object(heartbeat.paths,'RUNS',root/'runs'), patch.object(heartbeat.paths,'config',return_value=cfg), patch.object(heartbeat,'processes',return_value={}), redirect_stdout(io.StringIO()) as out: self.assertEqual(heartbeat.heartbeat('r'),0);self.assertEqual(out.getvalue(),'') con.execute("INSERT INTO rollouts(run_id,job_id,kk,state,started_at) VALUES('r','job',1,'RUNNING',?)",(time.time()-5000,));con.commit() self.assertEqual(heartbeat.heartbeat('r'),1) self.assertIn('ZOMBIE',out.getvalue());self.assertIn('events idle',out.getvalue()) con.execute("INSERT INTO stages(run_id,job_id,stage,state,lane,started_at) VALUES('r','job','05-compile','RUNNING','agent-long',?)",(time.time()-5000,));con.commit() with patch.object(heartbeat,'matching',return_value=[42]), patch.object(heartbeat.watchdog,'_proc_tree_stat',return_value=(1000,0)), patch.object(heartbeat.time,'sleep'): self.assertEqual(heartbeat.heartbeat('r'),1) self.assertIn('STALLED stdout',out.getvalue()) con.close() def test_non_tty_watch_once(self): with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep: self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called() if __name__=='__main__': unittest.main()