124 lines
8.3 KiB
Python
124 lines
8.3 KiB
Python
"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI."""
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from harness.stageworker import Worker, config_bytes, publish
|
|
from harness import watch
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
def module(name, path):
|
|
spec = importlib.util.spec_from_file_location(name, ROOT/path)
|
|
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m
|
|
exporter = module('exporter', 'export/to_agentrl.py')
|
|
audit = module('audit', 'scripts/audit_rollouts.py')
|
|
|
|
class Regression(unittest.TestCase):
|
|
def test_annotations_scope(self):
|
|
rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')]
|
|
self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'})
|
|
self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'})
|
|
self.assertFalse(exporter.annotation_reasons(rows,'run','other',2))
|
|
|
|
def test_config_symlink_and_traversal(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x')
|
|
(ws/'link').symlink_to('/etc/passwd')
|
|
self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x')
|
|
for p in [ws/'link', ws/'../outside']:
|
|
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p))
|
|
|
|
def test_status_symlink_does_not_write_target(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe')
|
|
(hd/'status.json').symlink_to(secret)
|
|
fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY)
|
|
try: publish(fd,{'state':'DONE'})
|
|
finally: os.close(fd)
|
|
self.assertEqual(secret.read_text(),'safe')
|
|
self.assertFalse((hd/'status.json').is_symlink())
|
|
|
|
def test_noop_production_rejected(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}')
|
|
Worker(rd,test_noop=True).scan()
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL')
|
|
|
|
def test_content_and_state_other_job(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir()
|
|
(jd/'job.json').write_text('{"threshold":0.9900}')
|
|
(jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}')
|
|
events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}]
|
|
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
|
|
s=audit.scan_events(rd,1)
|
|
for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1)
|
|
self.assertNotIn('0.9900',audit.oracle_needles(rd))
|
|
events[1]['result']={'content':[{'text':'path-guard: denied'}]}
|
|
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
|
|
self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0)
|
|
|
|
def test_export_own_budget_and_version(self):
|
|
import sqlite3, sys
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5')
|
|
(root/'state').mkdir()
|
|
con=sqlite3.connect(root/'state/jobs.sqlite')
|
|
con.execute('CREATE TABLE annotations (run_id TEXT,job_id TEXT,note TEXT)');con.commit();con.close()
|
|
rd=root/'runs/r/job/rollouts/01';rd.mkdir(parents=True)
|
|
(rd/'rollout.json').write_text(json.dumps({'agent_config_version':6,'max_turns':60,'max_tokens':100}))
|
|
(rd/'verdict.json').write_text(json.dumps({'terminal':'FAIL','run_id':'r','job_id':'job','rollout':1,'gates':{},'cost':{'turns':30,'tokens':50}}))
|
|
(rd.parents[1]/'job.json').write_text(json.dumps({'task':'cls','march':'e','backend':'ptq'}))
|
|
with patch.object(exporter,'BENCH',root), patch.object(exporter,'scan',return_value=[rd]), patch.object(exporter,'load_messages',return_value=[{}]), patch.object(exporter,'clean',return_value=([{'role':'assistant','content':'test'}],{'drop':False})), patch.object(exporter,'system_prompt',return_value=('', 'none')), patch.object(exporter,'tool_defs',return_value=([], 'none')), patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']):
|
|
self.assertEqual(exporter.main(),0)
|
|
rec=json.loads((root/'datasets/test/teacher.jsonl').read_text())
|
|
self.assertEqual(rec['reward_terms']['cost'],-0.5)
|
|
manifest=json.loads((root/'datasets/test/manifest.json').read_text())
|
|
self.assertEqual(manifest['filters']['agent_version_mode'],'minimum')
|
|
self.assertEqual(manifest['stats']['written'],1)
|
|
|
|
def test_long_lane_preserved(self):
|
|
from harness import stagerun
|
|
from contextlib import nullcontext
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
with tempfile.TemporaryDirectory() as d:
|
|
rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True)
|
|
(hd/'request.json').write_text('{"stage":"05-compile"}')
|
|
ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={})
|
|
stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu'
|
|
stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={})
|
|
con=MagicMock()
|
|
with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'):
|
|
stagerun._exec(hd)
|
|
self.assertEqual(ctx.lane,'long')
|
|
self.assertEqual(con.execute.call_args[0][1][0],'agent-long')
|
|
|
|
def test_heartbeat_healthy_zombie_and_stall(self):
|
|
import sqlite3, time, io
|
|
from harness import heartbeat, db
|
|
from contextlib import redirect_stdout
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d); dbpath=root/'jobs.sqlite'
|
|
con=sqlite3.connect(dbpath);con.executescript(db.SCHEMA);con.commit()
|
|
cfg={'watchdog':{'stall_s':900,'cpu_idle_pct':5}}
|
|
with patch.object(heartbeat.paths,'DB_PATH',dbpath), patch.object(heartbeat.paths,'RUNS',root/'runs'), patch.object(heartbeat.paths,'config',return_value=cfg), patch.object(heartbeat,'processes',return_value={}), redirect_stdout(io.StringIO()) as out:
|
|
self.assertEqual(heartbeat.heartbeat('r'),0);self.assertEqual(out.getvalue(),'')
|
|
con.execute("INSERT INTO rollouts(run_id,job_id,kk,state,started_at) VALUES('r','job',1,'RUNNING',?)",(time.time()-5000,));con.commit()
|
|
self.assertEqual(heartbeat.heartbeat('r'),1)
|
|
self.assertIn('ZOMBIE',out.getvalue());self.assertIn('events idle',out.getvalue())
|
|
con.execute("INSERT INTO stages(run_id,job_id,stage,state,lane,started_at) VALUES('r','job','05-compile','RUNNING','agent-long',?)",(time.time()-5000,));con.commit()
|
|
with patch.object(heartbeat,'matching',return_value=[42]), patch.object(heartbeat.watchdog,'_proc_tree_stat',return_value=(1000,0)), patch.object(heartbeat.time,'sleep'):
|
|
self.assertEqual(heartbeat.heartbeat('r'),1)
|
|
self.assertIn('STALLED stdout',out.getvalue())
|
|
con.close()
|
|
|
|
def test_non_tty_watch_once(self):
|
|
with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep:
|
|
self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called()
|
|
|
|
if __name__=='__main__': unittest.main()
|