"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI.""" import importlib.util import json import os import tempfile import unittest from pathlib import Path from unittest.mock import patch from harness.stageworker import Worker, config_bytes, publish from harness import watch ROOT = Path(__file__).resolve().parents[1] def module(name, path): spec = importlib.util.spec_from_file_location(name, ROOT/path) m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m exporter = module('exporter', 'export/to_agentrl.py') audit = module('audit', 'scripts/audit_rollouts.py') class Regression(unittest.TestCase): def test_annotations_scope(self): rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')] self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'}) self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'}) self.assertFalse(exporter.annotation_reasons(rows,'run','other',2)) def test_config_symlink_and_traversal(self): with tempfile.TemporaryDirectory() as d: ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x') (ws/'link').symlink_to('/etc/passwd') self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x') for p in [ws/'link', ws/'../outside']: with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p)) def test_status_symlink_does_not_write_target(self): with tempfile.TemporaryDirectory() as d: hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe') (hd/'status.json').symlink_to(secret) fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY) try: publish(fd,{'state':'DONE'}) finally: os.close(fd) self.assertEqual(secret.read_text(),'safe') self.assertFalse((hd/'status.json').is_symlink()) def test_noop_production_rejected(self): with tempfile.TemporaryDirectory() as d: rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}') Worker(rd,test_noop=True).scan() self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL') def test_content_and_state_other_job(self): with tempfile.TemporaryDirectory() as d: jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir() (jd/'job.json').write_text('{"threshold":0.9900}') (jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}') events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}] (rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events))) s=audit.scan_events(rd,1) for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1) self.assertNotIn('0.9900',audit.oracle_needles(rd)) events[1]['result']={'content':[{'text':'path-guard: denied'}]} (rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events))) self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0) def test_export_own_budget_and_version(self): import sqlite3, sys with tempfile.TemporaryDirectory() as d: root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5') (root/'state').mkdir() con=sqlite3.connect(root/'state/jobs.sqlite') con.execute('CREATE TABLE annotations (run_id TEXT,job_id TEXT,note TEXT)');con.commit();con.close() rd=root/'runs/r/job/rollouts/01';rd.mkdir(parents=True) (rd/'rollout.json').write_text(json.dumps({'agent_config_version':6,'max_turns':60,'max_tokens':100})) (rd/'verdict.json').write_text(json.dumps({'terminal':'FAIL','run_id':'r','job_id':'job','rollout':1,'gates':{},'cost':{'turns':30,'tokens':50}})) (rd.parents[1]/'job.json').write_text(json.dumps({'task':'cls','march':'e','backend':'ptq'})) with patch.object(exporter,'BENCH',root), patch.object(exporter,'scan',return_value=[rd]), patch.object(exporter,'load_messages',return_value=[{}]), patch.object(exporter,'clean',return_value=([{'role':'assistant','content':'test'}],{'drop':False})), patch.object(exporter,'system_prompt',return_value=('', 'none')), patch.object(exporter,'tool_defs',return_value=([], 'none')), patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']): self.assertEqual(exporter.main(),0) rec=json.loads((root/'datasets/test/teacher.jsonl').read_text()) self.assertEqual(rec['reward_terms']['cost'],-0.5) manifest=json.loads((root/'datasets/test/manifest.json').read_text()) self.assertEqual(manifest['filters']['agent_version_mode'],'minimum') self.assertEqual(manifest['stats']['written'],1) def test_long_lane_preserved(self): from harness import stagerun from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import MagicMock with tempfile.TemporaryDirectory() as d: rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True) (hd/'request.json').write_text('{"stage":"05-compile"}') ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={}) stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu' stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={}) con=MagicMock() with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'): stagerun._exec(hd) self.assertEqual(ctx.lane,'long') self.assertEqual(con.execute.call_args[0][1][0],'agent-long') def test_heartbeat_healthy_zombie_and_stall(self): import sqlite3, time, io from harness import heartbeat, db from contextlib import redirect_stdout with tempfile.TemporaryDirectory() as d: root=Path(d); dbpath=root/'jobs.sqlite' con=sqlite3.connect(dbpath);con.executescript(db.SCHEMA);con.commit() cfg={'watchdog':{'stall_s':900,'cpu_idle_pct':5}} with patch.object(heartbeat.paths,'DB_PATH',dbpath), patch.object(heartbeat.paths,'RUNS',root/'runs'), patch.object(heartbeat.paths,'config',return_value=cfg), patch.object(heartbeat,'processes',return_value={}), redirect_stdout(io.StringIO()) as out: self.assertEqual(heartbeat.heartbeat('r'),0);self.assertEqual(out.getvalue(),'') con.execute("INSERT INTO rollouts(run_id,job_id,kk,state,started_at) VALUES('r','job',1,'RUNNING',?)",(time.time()-5000,));con.commit() self.assertEqual(heartbeat.heartbeat('r'),1) self.assertIn('ZOMBIE',out.getvalue());self.assertIn('events idle',out.getvalue()) con.execute("INSERT INTO stages(run_id,job_id,stage,state,lane,started_at) VALUES('r','job','05-compile','RUNNING','agent-long',?)",(time.time()-5000,));con.commit() with patch.object(heartbeat,'matching',return_value=[42]), patch.object(heartbeat.watchdog,'_proc_tree_stat',return_value=(1000,0)), patch.object(heartbeat.time,'sleep'): self.assertEqual(heartbeat.heartbeat('r'),1) self.assertIn('STALLED stdout',out.getvalue()) con.close() def test_non_tty_watch_once(self): with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep: self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called() if __name__=='__main__': unittest.main()