69 lines
3.8 KiB
Python
69 lines
3.8 KiB
Python
"""Regression checks: trust boundary, exporter filters, audit evidence and cron UI."""
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from harness.stageworker import Worker, config_bytes, publish
|
|
from harness import watch
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
def module(name, path):
|
|
spec = importlib.util.spec_from_file_location(name, ROOT/path)
|
|
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m
|
|
exporter = module('exporter', 'export/to_agentrl.py')
|
|
audit = module('audit', 'scripts/audit_rollouts.py')
|
|
|
|
class Regression(unittest.TestCase):
|
|
def test_annotations_scope(self):
|
|
rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')]
|
|
self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'})
|
|
self.assertEqual(exporter.annotation_reasons(rows,'run','job',1),{'stalled_killed'})
|
|
self.assertFalse(exporter.annotation_reasons(rows,'run','other',2))
|
|
|
|
def test_config_symlink_and_traversal(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
ws=Path(d)/'workspace'; ws.mkdir(); (ws/'ok').write_text('x')
|
|
(ws/'link').symlink_to('/etc/passwd')
|
|
self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x')
|
|
for p in [ws/'link', ws/'../outside']:
|
|
with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p))
|
|
|
|
def test_status_symlink_does_not_write_target(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
hd=Path(d)/'handle'; hd.mkdir(); secret=Path(d)/'secret';secret.write_text('safe')
|
|
(hd/'status.json').symlink_to(secret)
|
|
fd=os.open(hd,os.O_RDONLY|os.O_DIRECTORY)
|
|
try: publish(fd,{'state':'DONE'})
|
|
finally: os.close(fd)
|
|
self.assertEqual(secret.read_text(),'safe')
|
|
self.assertFalse((hd/'status.json').is_symlink())
|
|
|
|
def test_noop_production_rejected(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
rd=Path(d);(rd/'handles/h').mkdir(parents=True);(rd/'handles/h/request.json').write_text('{"stage":"noop"}')
|
|
Worker(rd,test_noop=True).scan()
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['result'],'FAIL')
|
|
|
|
def test_content_and_state_other_job(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
jd=Path(d)/'runs/r1/job';rd=jd/'rollouts/01';(rd/'agent').mkdir(parents=True);(jd/'oracle').mkdir()
|
|
(jd/'job.json').write_text('{"threshold":0.9900}')
|
|
(jd/'oracle/result.json').write_text('{"cos_min":0.987654321,"threshold":0.9900}')
|
|
events=[{'type':'tool_execution_start','toolCallId':'a','args':{'command':'python state/jobs.sqlite runs/r1/other/oracle/result.json'}}, {'type':'tool_execution_end','toolCallId':'a','result':{'content':[{'text':'cos 0.987654321'}]}}]
|
|
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
|
|
s=audit.scan_events(rd,1)
|
|
for k in ['oracle_content_match','state_access','other_job_read']: self.assertEqual(s[k],1)
|
|
self.assertNotIn('0.9900',audit.oracle_needles(rd))
|
|
events[1]['result']={'content':[{'text':'path-guard: denied'}]}
|
|
(rd/'agent/events.jsonl').write_text('\n'.join(map(json.dumps,events)))
|
|
self.assertEqual(audit.scan_events(rd,1)['oracle_content_match'],0)
|
|
|
|
def test_non_tty_watch_once(self):
|
|
with patch.object(watch.sys.stdout,'isatty',return_value=False), patch.object(watch,'snapshot',return_value='snapshot'), patch.object(watch.os,'system') as clear, patch.object(watch.time,'sleep') as sleep:
|
|
self.assertEqual(watch.watch('r'),0);clear.assert_not_called();sleep.assert_not_called()
|
|
|
|
if __name__=='__main__': unittest.main()
|