data-gen-bpu/scripts/test_terminal.py
bench b3293851b9 bench dev:排队(dev 名额、Salty 容器)不算命令自己的时间
prepare 给走 Salty 的 dev 请求多 3 小时排队余量(仍不超过整条轨迹的时限),命令自己的 timeout 在容器启动后才开始算
(worker 用 spec timeout)。10-15:64Gi 的 dev 容器在 Salty 排队超过 agent 给的 300 秒,连 ls 都超时。
prepare 在 collector 里,下次重启后生效。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 04:59:46 +08:00

170 lines
9.7 KiB
Python

"""Real Linux namespace/worker regressions; no model/provider/board is contacted."""
import json
import hashlib
import io
import os
from pathlib import Path
import sys
import tempfile
import tarfile
import time
import unittest
from unittest.mock import patch
from harness import execution, paths, stageworker, terminal
class PredictionContract(unittest.TestCase):
def test_exact_ordered_coverage(self):
raw = b'{"sample_id":"b","top5":[2,1,3,4,5]}\n{"sample_id":"a","top5":[1,2,3,4,5]}\n'
self.assertEqual(terminal.score_predictions(raw, {'a': 1, 'b': 2})['top1'], 100)
for broken in (raw.splitlines()[0], raw + raw.splitlines()[0], raw.replace(b'"b"', b'"x"'),
raw.replace(b'[2,1,3,4,5]', b'[2,2,3,4,5]')):
with self.assertRaises(ValueError):
terminal.score_predictions(broken, {'a': 1, 'b': 2})
@unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required')
class TerminalWorker(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix='terminal-acceptance-')
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.rd = self.root / 'runs/r/j/rollouts/00'
for name in ('workspace', 'handles', 'stages'):
(self.rd / name).mkdir(parents=True)
(self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j', 'march': 'nash-p'}))
(self.rd / 'rollout.json').write_text(json.dumps({'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'}))
self.cfg = dict(paths.config())
self.cfg.update(terminal={'enabled': True}, pools={'cpu': {'workers': 1}})
self.cfg['paths'] = dict(self.cfg['paths'], harness_python=sys.executable)
self.config = self.root / 'config.yaml'
import yaml
self.config.write_text(yaml.safe_dump(self.cfg))
policy = dict(version='terminal-test', queue_deadline_s=20, execution_deadline_s=40,
suspect_after_s=5, cancel_grace_s=.1, cleanup_deadline_s=4,
ordinary_tool_deadline_s=.1)
policy['phases'] = {name: {k: policy[k] for k in execution.POLICY_FIELDS} for name in execution.PHASES}
self.ex = execution.Execution(self.rd); self.ex.start(policy)
for override in (patch.object(paths, 'STATE', self.root / 'state'),
patch.object(paths, 'config', return_value=self.cfg),
patch.dict(os.environ, {'BENCH_CONFIG': str(self.config), 'BENCH_STATE_DIR': str(self.root / 'state')})):
override.start(); self.addCleanup(override.stop)
def submit(self, command, timeout=10, **extra):
hid = 'terminal-test'
hd = self.rd / 'handles' / hid; hd.mkdir()
req = dict(stage='terminal', target='development', command=command, timeout=timeout, **extra)
(hd / 'request.json').write_text(json.dumps(req))
worker = stageworker.Worker(self.rd, execution=self.ex)
worker.scan()
result = json.loads((hd / 'status.json').read_text())
return result
def test_long_command_no_stage_and_private_filesystem(self):
marker = self.root / 'hidden-labels.txt'; marker.write_text('must-not-read')
result = self.submit(f'test ! -e {marker}; sleep .5; printf terminal-ok > result.txt; cat result.txt')
self.assertEqual(result['result'], 'OK', result)
self.assertEqual((self.rd / 'workspace/result.txt').read_text(), 'terminal-ok')
self.assertIn('terminal-ok', result['stdout_tail'])
self.assertEqual(result['cleanup_state'], 'VERIFIED')
self.assertEqual(result['model_execution_source'], 'UNVERIFIED')
scopes = self.ex.snapshot()['process_scopes']
self.assertTrue(scopes)
self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values()))
def test_request_deadline_reaps_detached_child(self):
result = self.submit('setsid sh -c "sleep 60" & wait', timeout=1)
self.assertNotEqual(result['result'], 'OK', result)
scopes = self.ex.snapshot()['process_scopes']
self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values()))
def test_explicit_error_is_returned(self):
result = self.submit('echo missing-private-package >&2; exit 9')
self.assertEqual(result['exit_code'], 9, result)
self.assertIn('missing-private-package', result['stderr_tail'])
def test_valid_predictions_do_not_prove_model_execution(self):
# Adversarial candidate: writes perfect predictions without loading a
# model. Scoring is real, but the formal model/HBM claim must stay unknown.
package = self.root / 'secret-evaluation.tar'
manifest = {'sha256': 'a' * 64, 'items': [
{'file': 'a.jpg', 'label': 1}, {'file': 'b.jpg', 'label': 1}]}
with tarfile.open(package, 'w') as tar:
for name, raw in [('manifest.json', json.dumps(manifest).encode()), ('a.jpg', b'fixture'), ('b.jpg', b'fixture')]:
info = tarfile.TarInfo(name); info.size = len(raw)
tar.addfile(info, io.BytesIO(raw))
jobpath = self.rd.parent.parent / 'job.json'
job = json.loads(jobpath.read_text())
job['data'] = {'t1': {'path': str(package), 'manifest_sha': hashlib.sha256(package.read_bytes()).hexdigest(),
'content_sha': 'a' * 64, 'n': 2}}
jobpath.write_text(json.dumps(job))
(self.rd / 'workspace/fake.py').write_text(
"import json,pathlib\n"
"items=json.loads(pathlib.Path('/input/samples.json').read_text())['items']\n"
"assert all('label' not in x for x in items)\n"
"pathlib.Path('/out/predictions.jsonl').write_text(''.join(json.dumps({'sample_id':x['sample_id'],'top5':[1,2,3,4,5]})+'\\n' for x in items))\n")
result = self.submit('/usr/bin/python3 /workspace/fake.py', evaluate=True, predictions='predictions.jsonl')
self.assertEqual(result['result'], 'OK', result)
record = json.loads((self.rd / 'terminal_result.json').read_text())
self.assertEqual(record['metrics']['top1'], 100)
self.assertEqual(record['measurement_terminal'], 'UNVERIFIED')
self.assertEqual(record['model_execution_source'], 'UNVERIFIED')
raw = Path(record['predictions']['path']).read_bytes()
self.assertEqual(hashlib.sha256(raw).hexdigest(), record['predictions']['sha256'])
class BoardCommand(unittest.TestCase):
"""bench dev --board: a marked development request becomes a board request only for v18+ tasks."""
def test_marked_request_goes_to_the_board_for_v18_only(self):
from types import SimpleNamespace
req = {'target': 'development', 'command': terminal.BOARD_MARK + 'python3 run.py', 'cwd': 'board',
'source': '.', 'evaluate': False, 'timeout': 3600, 'deadline_monotonic': time.monotonic() + 7200}
with tempfile.TemporaryDirectory() as d, \
patch('harness.salty_dev.download_dir', side_effect=lambda rd, rel, dest: dest.mkdir()) as fetch:
hd = Path(d)
ctx = SimpleNamespace(job={'march': 'nash-p'}, extra={'own_inputs_required': False})
with self.assertRaisesRegex(ValueError, 'BOARD_TERMINAL_UNAVAILABLE'):
terminal.board_request(ctx, hd, hd, req)
ctx = SimpleNamespace(job={'march': 'nash-p'}, extra={'own_inputs_required': True})
board, files = terminal.board_request(ctx, hd, hd, req)
self.assertEqual((board['target'], board['command'], board['source'], board['cwd']),
('board', 'python3 run.py', 'board', '.'))
self.assertLessEqual(board['deadline_monotonic'], time.monotonic() + terminal.BOARD_MAX_S)
self.assertEqual(fetch.call_args.args[1], 'board')
@unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required')
class SaltyDevAdmission(TerminalWorker):
"""bench dev on a current (non natural-agent) rollout is admitted only when it goes to Salty, whose
container sandbox is the isolation; the container run itself was exercised on Salty (10-14)."""
def setUp(self):
super().setUp()
(self.rd / 'rollout.json').write_text(json.dumps({'jail_policy': 'private-home'}))
self.queue = self.rd / 'stages/_queue/terminal-x'; self.queue.mkdir(parents=True)
def prepare(self, salty, timeout=60, **extra):
cfg = dict(self.cfg, execution={'salty_remote': {'enabled': True, 'stages': ['terminal'] if salty else []}})
with patch.object(paths, 'config', return_value=cfg):
return terminal.prepare(self.rd, self.queue, dict(target='development', command='python x.py', timeout=timeout, **extra))
def test_admitted_only_through_salty(self):
self.assertEqual(self.prepare(True)['command'], 'python x.py')
# A 1 s command still gets the execution's remaining time: waiting for a dev slot or a Salty container
# is not the command's own (the container enforces the 1 s from its start).
self.queue = self.rd / 'stages/_queue/terminal-y'; self.queue.mkdir(parents=True)
self.assertEqual(self.prepare(True, timeout=1)['deadline_monotonic'], self.ex.snapshot()['deadline_monotonic'])
with self.assertRaisesRegex(ValueError, 'natural-agent-v1'):
self.prepare(False)
def test_formal_evaluation_is_not_a_dev_command(self):
with self.assertRaisesRegex(ValueError, 'commands only'):
self.prepare(True, evaluate=True, predictions='p.jsonl')
test_long_command_no_stage_and_private_filesystem = test_request_deadline_reaps_detached_child = None
test_explicit_error_is_returned = test_valid_predictions_do_not_prove_model_execution = None
if __name__ == '__main__':
unittest.main()