"""Real Linux namespace/worker regressions; no model/provider/board is contacted.""" import json import hashlib import io import os from pathlib import Path import sys import tempfile import tarfile import time import unittest from unittest.mock import patch from harness import execution, paths, stageworker, terminal class PredictionContract(unittest.TestCase): def test_exact_ordered_coverage(self): raw = b'{"sample_id":"b","top5":[2,1,3,4,5]}\n{"sample_id":"a","top5":[1,2,3,4,5]}\n' self.assertEqual(terminal.score_predictions(raw, {'a': 1, 'b': 2})['top1'], 100) for broken in (raw.splitlines()[0], raw + raw.splitlines()[0], raw.replace(b'"b"', b'"x"'), raw.replace(b'[2,1,3,4,5]', b'[2,2,3,4,5]')): with self.assertRaises(ValueError): terminal.score_predictions(broken, {'a': 1, 'b': 2}) @unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required') class TerminalWorker(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix='terminal-acceptance-') self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.rd = self.root / 'runs/r/j/rollouts/00' for name in ('workspace', 'handles', 'stages'): (self.rd / name).mkdir(parents=True) (self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j', 'march': 'nash-p'})) (self.rd / 'rollout.json').write_text(json.dumps({'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'})) self.cfg = dict(paths.config()) self.cfg.update(terminal={'enabled': True}, pools={'cpu': {'workers': 1}}) self.cfg['paths'] = dict(self.cfg['paths'], harness_python=sys.executable) self.config = self.root / 'config.yaml' import yaml self.config.write_text(yaml.safe_dump(self.cfg)) policy = dict(version='terminal-test', queue_deadline_s=20, execution_deadline_s=40, suspect_after_s=5, cancel_grace_s=.1, cleanup_deadline_s=4, ordinary_tool_deadline_s=.1) policy['phases'] = {name: {k: policy[k] for k in execution.POLICY_FIELDS} for name in execution.PHASES} self.ex = execution.Execution(self.rd); self.ex.start(policy) for override in (patch.object(paths, 'STATE', self.root / 'state'), patch.object(paths, 'config', return_value=self.cfg), patch.dict(os.environ, {'BENCH_CONFIG': str(self.config), 'BENCH_STATE_DIR': str(self.root / 'state')})): override.start(); self.addCleanup(override.stop) def submit(self, command, timeout=10, **extra): hid = 'terminal-test' hd = self.rd / 'handles' / hid; hd.mkdir() req = dict(stage='terminal', target='development', command=command, timeout=timeout, **extra) (hd / 'request.json').write_text(json.dumps(req)) worker = stageworker.Worker(self.rd, execution=self.ex) worker.scan() result = json.loads((hd / 'status.json').read_text()) return result def test_long_command_no_stage_and_private_filesystem(self): marker = self.root / 'hidden-labels.txt'; marker.write_text('must-not-read') result = self.submit(f'test ! -e {marker}; sleep .5; printf terminal-ok > result.txt; cat result.txt') self.assertEqual(result['result'], 'OK', result) self.assertEqual((self.rd / 'workspace/result.txt').read_text(), 'terminal-ok') self.assertIn('terminal-ok', result['stdout_tail']) self.assertEqual(result['cleanup_state'], 'VERIFIED') self.assertEqual(result['model_execution_source'], 'UNVERIFIED') scopes = self.ex.snapshot()['process_scopes'] self.assertTrue(scopes) self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values())) def test_request_deadline_reaps_detached_child(self): result = self.submit('setsid sh -c "sleep 60" & wait', timeout=1) self.assertNotEqual(result['result'], 'OK', result) scopes = self.ex.snapshot()['process_scopes'] self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values())) def test_timing_out_in_the_queue_says_so(self): # 10-19 staging: the child's TERMINAL_DEADLINE_EXCEEDED was overwritten by a bare STAGE_EXECUTOR_EXIT. from harness import scheduler with scheduler.acquire('cpu', lane='agent'): # the only slot is taken for the whole request result = self.submit('true', timeout=1) self.assertEqual((result['result'], result['error_class']), ('FAIL', 'TERMINAL_DEADLINE_EXCEEDED'), result) def test_explicit_error_is_returned(self): result = self.submit('echo missing-private-package >&2; exit 9') self.assertEqual(result['exit_code'], 9, result) self.assertIn('missing-private-package', result['stderr_tail']) def test_valid_predictions_do_not_prove_model_execution(self): # Adversarial candidate: writes perfect predictions without loading a # model. Scoring is real, but the formal model/HBM claim must stay unknown. package = self.root / 'secret-evaluation.tar' manifest = {'sha256': 'a' * 64, 'items': [ {'file': 'a.jpg', 'label': 1}, {'file': 'b.jpg', 'label': 1}]} with tarfile.open(package, 'w') as tar: for name, raw in [('manifest.json', json.dumps(manifest).encode()), ('a.jpg', b'fixture'), ('b.jpg', b'fixture')]: info = tarfile.TarInfo(name); info.size = len(raw) tar.addfile(info, io.BytesIO(raw)) jobpath = self.rd.parent.parent / 'job.json' job = json.loads(jobpath.read_text()) job['data'] = {'t1': {'path': str(package), 'manifest_sha': hashlib.sha256(package.read_bytes()).hexdigest(), 'content_sha': 'a' * 64, 'n': 2}} jobpath.write_text(json.dumps(job)) (self.rd / 'workspace/fake.py').write_text( "import json,pathlib\n" "items=json.loads(pathlib.Path('/input/samples.json').read_text())['items']\n" "assert all('label' not in x for x in items)\n" "pathlib.Path('/out/predictions.jsonl').write_text(''.join(json.dumps({'sample_id':x['sample_id'],'top5':[1,2,3,4,5]})+'\\n' for x in items))\n") result = self.submit('/usr/bin/python3 /workspace/fake.py', evaluate=True, predictions='predictions.jsonl') self.assertEqual(result['result'], 'OK', result) record = json.loads((self.rd / 'terminal_result.json').read_text()) self.assertEqual(record['metrics']['top1'], 100) self.assertEqual(record['measurement_terminal'], 'UNVERIFIED') self.assertEqual(record['model_execution_source'], 'UNVERIFIED') raw = Path(record['predictions']['path']).read_bytes() self.assertEqual(hashlib.sha256(raw).hexdigest(), record['predictions']['sha256']) class BoardCommand(unittest.TestCase): """bench dev --board: a marked development request becomes a board request only for v18+ tasks.""" def test_marked_request_goes_to_the_board_for_v18_only(self): from types import SimpleNamespace req = {'target': 'development', 'command': terminal.BOARD_MARK + 'python3 run.py', 'cwd': 'board', 'source': '.', 'evaluate': False, 'timeout': 3600, 'deadline_monotonic': time.monotonic() + 7200} with tempfile.TemporaryDirectory() as d, \ patch('harness.salty_dev.remote_size', return_value=1 << 20) as size, \ patch('harness.salty_dev.download_dir', side_effect=lambda rd, rel, dest: dest.mkdir()) as fetch: hd = Path(d) ctx = SimpleNamespace(job={'march': 'nash-p'}, extra={'own_inputs_required': False}) with self.assertRaisesRegex(terminal.BoardRefused, 'BOARD_TERMINAL_UNAVAILABLE'): terminal.board_request(ctx, hd, hd, req) ctx = SimpleNamespace(job={'march': 'nash-p'}, extra={'own_inputs_required': True}) board, files = terminal.board_request(ctx, hd, hd, req) self.assertEqual((board['target'], board['command'], board['source'], board['cwd']), ('board', 'python3 run.py', 'board', '.')) # The queue for a board shared with production is not the command's time; that starts with the board. self.assertEqual((board['deadline_monotonic'], board['run_s']), (req['deadline_monotonic'], terminal.BOARD_MAX_S)) self.assertEqual(fetch.call_args.args[1], 'board') self.assertEqual(size.call_args.args[1:], ('board', terminal.BOARD_MAX_BYTES)) def test_too_large_is_refused_before_fetching_and_slow_fetch_is_explained(self): from types import SimpleNamespace ctx = SimpleNamespace(job={'march': 'nash-p'}, extra={'own_inputs_required': True}) req = {'target': 'development', 'command': terminal.BOARD_MARK + 'python3 run.py', 'cwd': '.', 'source': '.', 'evaluate': False, 'timeout': 1500, 'deadline_monotonic': time.monotonic() + 1500} with tempfile.TemporaryDirectory() as d, patch('harness.salty_dev.remote_size', return_value=None), \ patch('harness.salty_dev.download_dir') as fetch: with self.assertRaisesRegex(terminal.BoardRefused, 'BOARD_INPUT_TOO_LARGE.*小目录'): terminal.board_request(ctx, Path(d), Path(d), req) fetch.assert_not_called() late = dict(req, deadline_monotonic=time.monotonic() + 30) with tempfile.TemporaryDirectory() as d, patch('harness.salty_dev.remote_size', return_value=1), \ patch('harness.salty_dev.download_dir', side_effect=lambda rd, rel, dest: dest.mkdir()): with self.assertRaisesRegex(terminal.BoardRefused, 'BOARD_TRANSFER_TOO_SLOW'): terminal.board_request(ctx, Path(d), Path(d), late) def test_refusal_reaches_the_agent_and_copies_are_dropped(self): with tempfile.TemporaryDirectory() as d: hd = Path(d) (hd / 'request.json').write_text(json.dumps({'target': 'development', 'command': terminal.BOARD_MARK + 'x'})) (hd / 'snapshot').mkdir(); (hd / 'dev-files').mkdir() published = [] with patch.object(terminal, '_run', side_effect=terminal.BoardRefused('BOARD_INPUT_TOO_LARGE: 太大')): self.assertEqual(terminal.run(hd, hd, published.append), 0) self.assertEqual((published[-1]['result'], published[-1]['error_class']), ('FAIL', 'BOARD_INPUT_TOO_LARGE')) self.assertIn('太大', published[-1]['stderr_tail']) self.assertFalse((hd / 'snapshot').exists() or (hd / 'dev-files').exists()) class RemoteSize(unittest.TestCase): """salty storage ls, one directory level per call: stop as soon as the cap or the directory budget is passed.""" def walk(self, listing, cap, max_dirs=40): from harness import salty_dev calls = [] class Cli: def call(self, *args, **kw): calls.append(args[2]); return 0, listing.get(args[2], '名称\t类型\t大小\t修改时间\n'), '' with patch.object(salty_dev, '_storage', return_value=(Cli(), 'R')): return salty_dev.remote_size(Path('/x'), 'd', cap, max_dirs), calls def test_sizes_and_limits(self): head = '名称\t类型\t大小\t修改时间\n' listing = {'R': head + 'a.hbm\t文件\t300\t-\nsub\t目录\t-\t-\n', 'R/sub': head + 'b.npy\t文件\t50\t-\n'} self.assertEqual(self.walk(listing, 1000), (350, ['R', 'R/sub'])) self.assertEqual(self.walk(listing, 100)[0], None) # past the cap after the first listing self.assertEqual(self.walk(listing, 1000, max_dirs=1)[0], None) @unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required') class SaltyDevAdmission(TerminalWorker): """bench dev on a current (non natural-agent) rollout is admitted only when it goes to Salty, whose container sandbox is the isolation; the container run itself was exercised on Salty (10-14).""" def setUp(self): super().setUp() (self.rd / 'rollout.json').write_text(json.dumps({'jail_policy': 'private-home'})) self.queue = self.rd / 'stages/_queue/terminal-x'; self.queue.mkdir(parents=True) def prepare(self, salty, timeout=60, **extra): cfg = dict(self.cfg, execution={'salty_remote': {'enabled': True, 'stages': ['terminal'] if salty else []}}) with patch.object(paths, 'config', return_value=cfg): return terminal.prepare(self.rd, self.queue, dict(target='development', command='python x.py', timeout=timeout, **extra)) def test_admitted_only_through_salty(self): self.assertEqual(self.prepare(True)['command'], 'python x.py') # A 1 s command still gets the execution's remaining time: waiting for a dev slot or a Salty container # is not the command's own (the container enforces the 1 s from its start). self.queue = self.rd / 'stages/_queue/terminal-y'; self.queue.mkdir(parents=True) self.assertEqual(self.prepare(True, timeout=1)['deadline_monotonic'], self.ex.snapshot()['deadline_monotonic']) with self.assertRaisesRegex(ValueError, 'natural-agent-v1'): self.prepare(False) def test_formal_evaluation_is_not_a_dev_command(self): with self.assertRaisesRegex(ValueError, 'commands only'): self.prepare(True, evaluate=True, predictions='p.jsonl') test_long_command_no_stage_and_private_filesystem = test_request_deadline_reaps_detached_child = None test_explicit_error_is_returned = test_valid_predictions_do_not_prove_model_execution = test_timing_out_in_the_queue_says_so = None if __name__ == '__main__': unittest.main()