data-gen-bpu/scripts/test_codex.py

105 lines
6.7 KiB
Python

"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI."""
import importlib.util
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from harness.stageworker import Worker, config_bytes, publish
from harness import watch
ROOT = Path(__file__).resolve().parents[1]
def module(name, path):
spec = importlib.util.spec_from_file_location(name, ROOT/path)
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m
exporter = module('exporter', 'export/to_agentrl.py')
audit = module('audit', 'scripts/audit_rollouts.py')
class Regression(unittest.TestCase):
def test_annotations_scope(self):
rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')]
self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'})
self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'})
self.assertFalse(exporter.annotation_reasons(rows,'run','other',2))
def test_config_symlink_and_traversal(self):
with tempfile.TemporaryDirectory() as d:
ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x')
(ws/'link').symlink_to('/etc/passwd')
self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x')
for p in [ws/'link', ws/'../outside']:
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p))
def test_status_symlink_does_not_write_target(self):
with tempfile.TemporaryDirectory() as d:
hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe')
(hd/'status.json').symlink_to(secret)
fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY)
try: publish(fd,{'state':'DONE'})
finally: os.close(fd)
self.assertEqual(secret.read_text(),'safe')
self.assertFalse((hd/'status.json').is_symlink())
def test_noop_production_rejected(self):
with tempfile.TemporaryDirectory() as d:
rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}')
Worker(rd,test_noop=True).scan()
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL')
def test_content_and_state_other_job(self):
with tempfile.TemporaryDirectory() as d:
jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir()
(jd/'job.json').write_text('{"threshold":0.9900}')
(jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}')
events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}]
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
s=audit.scan_events(rd,1)
for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1)
self.assertNotIn('0.9900',audit.oracle_needles(rd))
events[1]['result']={'content':[{'text':'path-guard: denied'}]}
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0)
def test_export_own_budget_and_version(self):
import sqlite3, sys
with tempfile.TemporaryDirectory() as d:
root=Path(d); (root/'agent').mkdir(); (root/'agent/VERSION').write_text('5')
(root/'state').mkdir()
con=sqlite3.connect(root/'state/jobs.sqlite')
con.execute('CREATE TABLE annotations (run_id TEXT,job_id TEXT,note TEXT)');con.commit();con.close()
rd=root/'runs/r/job/rollouts/01';rd.mkdir(parents=True)
(rd/'rollout.json').write_text(json.dumps({'agent_config_version':6,'max_turns':60,'max_tokens':100}))
(rd/'verdict.json').write_text(json.dumps({'terminal':'FAIL','run_id':'r','job_id':'job','rollout':1,'gates':{},'cost':{'turns':30,'tokens':50}}))
(rd.parents[1]/'job.json').write_text(json.dumps({'task':'cls','march':'e','backend':'ptq'}))
with patch.object(exporter,'BENCH',root), patch.object(exporter,'scan',return_value=[rd]), patch.object(exporter,'load_messages',return_value=[{}]), patch.object(exporter,'clean',return_value=([{'role':'assistant','content':'test'}],{'drop':False})), patch.object(exporter,'system_prompt',return_value=('', 'none')), patch.object(exporter,'tool_defs',return_value=([], 'none')), patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']):
self.assertEqual(exporter.main(),0)
rec=json.loads((root/'datasets/test/teacher.jsonl').read_text())
self.assertEqual(rec['reward_terms']['cost'],-0.5)
manifest=json.loads((root/'datasets/test/manifest.json').read_text())
self.assertEqual(manifest['filters']['agent_version_mode'],'minimum')
self.assertEqual(manifest['stats']['written'],1)
def test_long_lane_preserved(self):
from harness import stagerun
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock
with tempfile.TemporaryDirectory() as d:
rd=Path(d)/'01';hd=rd/'handles/h';hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"05-compile"}')
ctx=SimpleNamespace(run_id='r',job={'job_id':'j'},rollout=1,lane='normal',extra={})
stage=MagicMock(name='stage');stage.name='05-compile';stage.pool='cpu'
stage.run.return_value=SimpleNamespace(state='OK',exit_code=0,error_class=None,duration_s=0,measurements={})
con=MagicMock()
with patch.object(stagerun,'_ctx_from_rollout',return_value=ctx), patch.dict(stagerun.BY_NAME,{'05-compile':lambda:stage}), patch.object(stagerun.db,'connect',side_effect=lambda:nullcontext(con)), patch.object(stagerun.scheduler,'acquire',return_value=nullcontext(1)), patch.object(stagerun.db,'stage_start'), patch.object(stagerun.db,'stage_finish'):
stagerun._exec(hd)
self.assertEqual(ctx.lane,'long')
self.assertEqual(con.execute.call_args[0][1][0],'agent-long')
def test_non_tty_watch_once(self):
with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep:
self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called()
if __name__=='__main__': unittest.main()