119 lines
6.5 KiB
Python
119 lines
6.5 KiB
Python
"""Real Linux namespace/worker regressions; no model/provider/board is contacted."""
|
|
import json
|
|
import hashlib
|
|
import io
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import tarfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import execution, paths, stageworker, terminal
|
|
|
|
|
|
class PredictionContract(unittest.TestCase):
|
|
def test_exact_ordered_coverage(self):
|
|
raw = b'{"sample_id":"b","top5":[2,1,3,4,5]}\n{"sample_id":"a","top5":[1,2,3,4,5]}\n'
|
|
self.assertEqual(terminal.score_predictions(raw, {'a': 1, 'b': 2})['top1'], 100)
|
|
for broken in (raw.splitlines()[0], raw + raw.splitlines()[0], raw.replace(b'"b"', b'"x"'),
|
|
raw.replace(b'[2,1,3,4,5]', b'[2,2,3,4,5]')):
|
|
with self.assertRaises(ValueError):
|
|
terminal.score_predictions(broken, {'a': 1, 'b': 2})
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required')
|
|
class TerminalWorker(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(prefix='terminal-acceptance-')
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.rd = self.root / 'runs/r/j/rollouts/00'
|
|
for name in ('workspace', 'handles', 'stages'):
|
|
(self.rd / name).mkdir(parents=True)
|
|
(self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j', 'march': 'nash-p'}))
|
|
(self.rd / 'rollout.json').write_text(json.dumps({'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'}))
|
|
self.cfg = dict(paths.config())
|
|
self.cfg.update(terminal={'enabled': True}, pools={'cpu': {'workers': 1}})
|
|
self.cfg['paths'] = dict(self.cfg['paths'], harness_python=sys.executable)
|
|
self.config = self.root / 'config.yaml'
|
|
import yaml
|
|
self.config.write_text(yaml.safe_dump(self.cfg))
|
|
policy = dict(version='terminal-test', queue_deadline_s=20, execution_deadline_s=40,
|
|
suspect_after_s=5, cancel_grace_s=.1, cleanup_deadline_s=4,
|
|
ordinary_tool_deadline_s=.1)
|
|
policy['phases'] = {name: {k: policy[k] for k in execution.POLICY_FIELDS} for name in execution.PHASES}
|
|
self.ex = execution.Execution(self.rd); self.ex.start(policy)
|
|
for override in (patch.object(paths, 'STATE', self.root / 'state'),
|
|
patch.object(paths, 'config', return_value=self.cfg),
|
|
patch.dict(os.environ, {'BENCH_CONFIG': str(self.config), 'BENCH_STATE_DIR': str(self.root / 'state')})):
|
|
override.start(); self.addCleanup(override.stop)
|
|
|
|
def submit(self, command, timeout=10, **extra):
|
|
hid = 'terminal-test'
|
|
hd = self.rd / 'handles' / hid; hd.mkdir()
|
|
req = dict(stage='terminal', target='development', command=command, timeout=timeout, **extra)
|
|
(hd / 'request.json').write_text(json.dumps(req))
|
|
worker = stageworker.Worker(self.rd, execution=self.ex)
|
|
worker.scan()
|
|
result = json.loads((hd / 'status.json').read_text())
|
|
return result
|
|
|
|
def test_long_command_no_stage_and_private_filesystem(self):
|
|
marker = self.root / 'hidden-labels.txt'; marker.write_text('must-not-read')
|
|
result = self.submit(f'test ! -e {marker}; sleep .5; printf terminal-ok > result.txt; cat result.txt')
|
|
self.assertEqual(result['result'], 'OK', result)
|
|
self.assertEqual((self.rd / 'workspace/result.txt').read_text(), 'terminal-ok')
|
|
self.assertIn('terminal-ok', result['stdout_tail'])
|
|
self.assertEqual(result['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(result['model_execution_source'], 'UNVERIFIED')
|
|
scopes = self.ex.snapshot()['process_scopes']
|
|
self.assertTrue(scopes)
|
|
self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values()))
|
|
|
|
def test_request_deadline_reaps_detached_child(self):
|
|
result = self.submit('setsid sh -c "sleep 60" & wait', timeout=1)
|
|
self.assertNotEqual(result['result'], 'OK', result)
|
|
scopes = self.ex.snapshot()['process_scopes']
|
|
self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values()))
|
|
|
|
def test_explicit_error_is_returned(self):
|
|
result = self.submit('echo missing-private-package >&2; exit 9')
|
|
self.assertEqual(result['exit_code'], 9, result)
|
|
self.assertIn('missing-private-package', result['stderr_tail'])
|
|
|
|
def test_valid_predictions_do_not_prove_model_execution(self):
|
|
# Adversarial candidate: writes perfect predictions without loading a
|
|
# model. Scoring is real, but the formal model/HBM claim must stay unknown.
|
|
package = self.root / 'secret-evaluation.tar'
|
|
manifest = {'sha256': 'a' * 64, 'items': [
|
|
{'file': 'a.jpg', 'label': 1}, {'file': 'b.jpg', 'label': 1}]}
|
|
with tarfile.open(package, 'w') as tar:
|
|
for name, raw in [('manifest.json', json.dumps(manifest).encode()), ('a.jpg', b'fixture'), ('b.jpg', b'fixture')]:
|
|
info = tarfile.TarInfo(name); info.size = len(raw)
|
|
tar.addfile(info, io.BytesIO(raw))
|
|
jobpath = self.rd.parent.parent / 'job.json'
|
|
job = json.loads(jobpath.read_text())
|
|
job['data'] = {'t1': {'path': str(package), 'manifest_sha': hashlib.sha256(package.read_bytes()).hexdigest(),
|
|
'content_sha': 'a' * 64, 'n': 2}}
|
|
jobpath.write_text(json.dumps(job))
|
|
(self.rd / 'workspace/fake.py').write_text(
|
|
"import json,pathlib\n"
|
|
"items=json.loads(pathlib.Path('/input/samples.json').read_text())['items']\n"
|
|
"assert all('label' not in x for x in items)\n"
|
|
"pathlib.Path('/out/predictions.jsonl').write_text(''.join(json.dumps({'sample_id':x['sample_id'],'top5':[1,2,3,4,5]})+'\\n' for x in items))\n")
|
|
result = self.submit('/usr/bin/python3 /workspace/fake.py', evaluate=True, predictions='predictions.jsonl')
|
|
self.assertEqual(result['result'], 'OK', result)
|
|
record = json.loads((self.rd / 'terminal_result.json').read_text())
|
|
self.assertEqual(record['metrics']['top1'], 100)
|
|
self.assertEqual(record['measurement_terminal'], 'UNVERIFIED')
|
|
self.assertEqual(record['model_execution_source'], 'UNVERIFIED')
|
|
raw = Path(record['predictions']['path']).read_bytes()
|
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), record['predictions']['sha256'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|