"""Real Linux namespace/worker regressions; no model/provider/board is contacted.""" import json import hashlib import io import os from pathlib import Path import sys import tempfile import tarfile import time import unittest from unittest.mock import patch from harness import execution, paths, stageworker, terminal class PredictionContract(unittest.TestCase): def test_exact_ordered_coverage(self): raw = b'{"sample_id":"b","top5":[2,1,3,4,5]}\n{"sample_id":"a","top5":[1,2,3,4,5]}\n' self.assertEqual(terminal.score_predictions(raw, {'a': 1, 'b': 2})['top1'], 100) for broken in (raw.splitlines()[0], raw + raw.splitlines()[0], raw.replace(b'"b"', b'"x"'), raw.replace(b'[2,1,3,4,5]', b'[2,2,3,4,5]')): with self.assertRaises(ValueError): terminal.score_predictions(broken, {'a': 1, 'b': 2}) @unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required') class TerminalWorker(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix='terminal-acceptance-') self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.rd = self.root / 'runs/r/j/rollouts/00' for name in ('workspace', 'handles', 'stages'): (self.rd / name).mkdir(parents=True) (self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j', 'march': 'nash-p'})) (self.rd / 'rollout.json').write_text(json.dumps({'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'})) self.cfg = dict(paths.config()) self.cfg.update(terminal={'enabled': True}, pools={'cpu': {'workers': 1}}) self.cfg['paths'] = dict(self.cfg['paths'], harness_python=sys.executable) self.config = self.root / 'config.yaml' import yaml self.config.write_text(yaml.safe_dump(self.cfg)) policy = dict(version='terminal-test', queue_deadline_s=20, execution_deadline_s=40, suspect_after_s=5, cancel_grace_s=.1, cleanup_deadline_s=4, ordinary_tool_deadline_s=.1) policy['phases'] = {name: {k: policy[k] for k in execution.POLICY_FIELDS} for name in execution.PHASES} self.ex = execution.Execution(self.rd); self.ex.start(policy) for override in (patch.object(paths, 'STATE', self.root / 'state'), patch.object(paths, 'config', return_value=self.cfg), patch.dict(os.environ, {'BENCH_CONFIG': str(self.config), 'BENCH_STATE_DIR': str(self.root / 'state')})): override.start(); self.addCleanup(override.stop) def submit(self, command, timeout=10, **extra): hid = 'terminal-test' hd = self.rd / 'handles' / hid; hd.mkdir() req = dict(stage='terminal', target='development', command=command, timeout=timeout, **extra) (hd / 'request.json').write_text(json.dumps(req)) worker = stageworker.Worker(self.rd, execution=self.ex) worker.scan() result = json.loads((hd / 'status.json').read_text()) return result def test_long_command_no_stage_and_private_filesystem(self): marker = self.root / 'hidden-labels.txt'; marker.write_text('must-not-read') result = self.submit(f'test ! -e {marker}; sleep .5; printf terminal-ok > result.txt; cat result.txt') self.assertEqual(result['result'], 'OK', result) self.assertEqual((self.rd / 'workspace/result.txt').read_text(), 'terminal-ok') self.assertIn('terminal-ok', result['stdout_tail']) self.assertEqual(result['cleanup_state'], 'VERIFIED') self.assertEqual(result['model_execution_source'], 'UNVERIFIED') scopes = self.ex.snapshot()['process_scopes'] self.assertTrue(scopes) self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values())) def test_request_deadline_reaps_detached_child(self): result = self.submit('setsid sh -c "sleep 60" & wait', timeout=1) self.assertNotEqual(result['result'], 'OK', result) scopes = self.ex.snapshot()['process_scopes'] self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values())) def test_explicit_error_is_returned(self): result = self.submit('echo missing-private-package >&2; exit 9') self.assertEqual(result['exit_code'], 9, result) self.assertIn('missing-private-package', result['stderr_tail']) def test_valid_predictions_do_not_prove_model_execution(self): # Adversarial candidate: writes perfect predictions without loading a # model. Scoring is real, but the formal model/HBM claim must stay unknown. package = self.root / 'secret-evaluation.tar' manifest = {'sha256': 'a' * 64, 'items': [ {'file': 'a.jpg', 'label': 1}, {'file': 'b.jpg', 'label': 1}]} with tarfile.open(package, 'w') as tar: for name, raw in [('manifest.json', json.dumps(manifest).encode()), ('a.jpg', b'fixture'), ('b.jpg', b'fixture')]: info = tarfile.TarInfo(name); info.size = len(raw) tar.addfile(info, io.BytesIO(raw)) jobpath = self.rd.parent.parent / 'job.json' job = json.loads(jobpath.read_text()) job['data'] = {'t1': {'path': str(package), 'manifest_sha': hashlib.sha256(package.read_bytes()).hexdigest(), 'content_sha': 'a' * 64, 'n': 2}} jobpath.write_text(json.dumps(job)) (self.rd / 'workspace/fake.py').write_text( "import json,pathlib\n" "items=json.loads(pathlib.Path('/input/samples.json').read_text())['items']\n" "assert all('label' not in x for x in items)\n" "pathlib.Path('/out/predictions.jsonl').write_text(''.join(json.dumps({'sample_id':x['sample_id'],'top5':[1,2,3,4,5]})+'\\n' for x in items))\n") result = self.submit('/usr/bin/python3 /workspace/fake.py', evaluate=True, predictions='predictions.jsonl') self.assertEqual(result['result'], 'OK', result) record = json.loads((self.rd / 'terminal_result.json').read_text()) self.assertEqual(record['metrics']['top1'], 100) self.assertEqual(record['measurement_terminal'], 'UNVERIFIED') self.assertEqual(record['model_execution_source'], 'UNVERIFIED') raw = Path(record['predictions']['path']).read_bytes() self.assertEqual(hashlib.sha256(raw).hexdigest(), record['predictions']['sha256']) if __name__ == '__main__': unittest.main()