data-gen-bpu/scripts/test_terminal.py
bench a92510dab8 智能体开发环境与自带图:bench dev 在 Salty 沙箱里跑命令,compile.yaml 可用 workspace 里自己的图和校准集
- bench dev:受管终端的 development 目标改走 Salty 容器(有完整工具链)。
  容器里用 user+mount 命名空间、uid nobody、去掉全部能力,只露工具链/缓存/数据(只读)和本轨迹 workspace;
  板卡密钥、其他轨迹、个人空间其余内容不可见(10-14 在 Salty 上实测,含 umount 逃逸被挡)。
  本地改动下次 bench dev 时上传,小文件拷回,大文件留在开发环境并列出。
- compile.yaml:model_parameters.onnx_model / calibration_parameters.cal_data_dir 可指向 workspace;
  04 在容器里核对输入输出与平台图一致(不一致直接拒、不占编译次数),记录图的 sha256 和校准来源。
- 配置白名单报错列出具体字段;SYSTEM.md 写明开发环境和自带图;agent/VERSION 15→16。
- 生效:stageworker/terminal/stageclient/SYSTEM.md 要重启;新配置要加 salty_remote.stages: terminal、
  terminal.enabled: true、pools.dev。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:49:14 +08:00

146 lines
8.0 KiB
Python

"""Real Linux namespace/worker regressions; no model/provider/board is contacted."""
import json
import hashlib
import io
import os
from pathlib import Path
import sys
import tempfile
import tarfile
import time
import unittest
from unittest.mock import patch
from harness import execution, paths, stageworker, terminal
class PredictionContract(unittest.TestCase):
def test_exact_ordered_coverage(self):
raw = b'{"sample_id":"b","top5":[2,1,3,4,5]}\n{"sample_id":"a","top5":[1,2,3,4,5]}\n'
self.assertEqual(terminal.score_predictions(raw, {'a': 1, 'b': 2})['top1'], 100)
for broken in (raw.splitlines()[0], raw + raw.splitlines()[0], raw.replace(b'"b"', b'"x"'),
raw.replace(b'[2,1,3,4,5]', b'[2,2,3,4,5]')):
with self.assertRaises(ValueError):
terminal.score_predictions(broken, {'a': 1, 'b': 2})
@unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required')
class TerminalWorker(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix='terminal-acceptance-')
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.rd = self.root / 'runs/r/j/rollouts/00'
for name in ('workspace', 'handles', 'stages'):
(self.rd / name).mkdir(parents=True)
(self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j', 'march': 'nash-p'}))
(self.rd / 'rollout.json').write_text(json.dumps({'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'}))
self.cfg = dict(paths.config())
self.cfg.update(terminal={'enabled': True}, pools={'cpu': {'workers': 1}})
self.cfg['paths'] = dict(self.cfg['paths'], harness_python=sys.executable)
self.config = self.root / 'config.yaml'
import yaml
self.config.write_text(yaml.safe_dump(self.cfg))
policy = dict(version='terminal-test', queue_deadline_s=20, execution_deadline_s=40,
suspect_after_s=5, cancel_grace_s=.1, cleanup_deadline_s=4,
ordinary_tool_deadline_s=.1)
policy['phases'] = {name: {k: policy[k] for k in execution.POLICY_FIELDS} for name in execution.PHASES}
self.ex = execution.Execution(self.rd); self.ex.start(policy)
for override in (patch.object(paths, 'STATE', self.root / 'state'),
patch.object(paths, 'config', return_value=self.cfg),
patch.dict(os.environ, {'BENCH_CONFIG': str(self.config), 'BENCH_STATE_DIR': str(self.root / 'state')})):
override.start(); self.addCleanup(override.stop)
def submit(self, command, timeout=10, **extra):
hid = 'terminal-test'
hd = self.rd / 'handles' / hid; hd.mkdir()
req = dict(stage='terminal', target='development', command=command, timeout=timeout, **extra)
(hd / 'request.json').write_text(json.dumps(req))
worker = stageworker.Worker(self.rd, execution=self.ex)
worker.scan()
result = json.loads((hd / 'status.json').read_text())
return result
def test_long_command_no_stage_and_private_filesystem(self):
marker = self.root / 'hidden-labels.txt'; marker.write_text('must-not-read')
result = self.submit(f'test ! -e {marker}; sleep .5; printf terminal-ok > result.txt; cat result.txt')
self.assertEqual(result['result'], 'OK', result)
self.assertEqual((self.rd / 'workspace/result.txt').read_text(), 'terminal-ok')
self.assertIn('terminal-ok', result['stdout_tail'])
self.assertEqual(result['cleanup_state'], 'VERIFIED')
self.assertEqual(result['model_execution_source'], 'UNVERIFIED')
scopes = self.ex.snapshot()['process_scopes']
self.assertTrue(scopes)
self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values()))
def test_request_deadline_reaps_detached_child(self):
result = self.submit('setsid sh -c "sleep 60" & wait', timeout=1)
self.assertNotEqual(result['result'], 'OK', result)
scopes = self.ex.snapshot()['process_scopes']
self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values()))
def test_explicit_error_is_returned(self):
result = self.submit('echo missing-private-package >&2; exit 9')
self.assertEqual(result['exit_code'], 9, result)
self.assertIn('missing-private-package', result['stderr_tail'])
def test_valid_predictions_do_not_prove_model_execution(self):
# Adversarial candidate: writes perfect predictions without loading a
# model. Scoring is real, but the formal model/HBM claim must stay unknown.
package = self.root / 'secret-evaluation.tar'
manifest = {'sha256': 'a' * 64, 'items': [
{'file': 'a.jpg', 'label': 1}, {'file': 'b.jpg', 'label': 1}]}
with tarfile.open(package, 'w') as tar:
for name, raw in [('manifest.json', json.dumps(manifest).encode()), ('a.jpg', b'fixture'), ('b.jpg', b'fixture')]:
info = tarfile.TarInfo(name); info.size = len(raw)
tar.addfile(info, io.BytesIO(raw))
jobpath = self.rd.parent.parent / 'job.json'
job = json.loads(jobpath.read_text())
job['data'] = {'t1': {'path': str(package), 'manifest_sha': hashlib.sha256(package.read_bytes()).hexdigest(),
'content_sha': 'a' * 64, 'n': 2}}
jobpath.write_text(json.dumps(job))
(self.rd / 'workspace/fake.py').write_text(
"import json,pathlib\n"
"items=json.loads(pathlib.Path('/input/samples.json').read_text())['items']\n"
"assert all('label' not in x for x in items)\n"
"pathlib.Path('/out/predictions.jsonl').write_text(''.join(json.dumps({'sample_id':x['sample_id'],'top5':[1,2,3,4,5]})+'\\n' for x in items))\n")
result = self.submit('/usr/bin/python3 /workspace/fake.py', evaluate=True, predictions='predictions.jsonl')
self.assertEqual(result['result'], 'OK', result)
record = json.loads((self.rd / 'terminal_result.json').read_text())
self.assertEqual(record['metrics']['top1'], 100)
self.assertEqual(record['measurement_terminal'], 'UNVERIFIED')
self.assertEqual(record['model_execution_source'], 'UNVERIFIED')
raw = Path(record['predictions']['path']).read_bytes()
self.assertEqual(hashlib.sha256(raw).hexdigest(), record['predictions']['sha256'])
@unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required')
class SaltyDevAdmission(TerminalWorker):
"""bench dev on a current (non natural-agent) rollout is admitted only when it goes to Salty, whose
container sandbox is the isolation; the container run itself was exercised on Salty (10-14)."""
def setUp(self):
super().setUp()
(self.rd / 'rollout.json').write_text(json.dumps({'jail_policy': 'private-home'}))
self.queue = self.rd / 'stages/_queue/terminal-x'; self.queue.mkdir(parents=True)
def prepare(self, salty, **extra):
cfg = dict(self.cfg, execution={'salty_remote': {'enabled': True, 'stages': ['terminal'] if salty else []}})
with patch.object(paths, 'config', return_value=cfg):
return terminal.prepare(self.rd, self.queue, dict(target='development', command='python x.py', timeout=60, **extra))
def test_admitted_only_through_salty(self):
self.assertEqual(self.prepare(True)['command'], 'python x.py')
with self.assertRaisesRegex(ValueError, 'natural-agent-v1'):
self.prepare(False)
def test_formal_evaluation_is_not_a_dev_command(self):
with self.assertRaisesRegex(ValueError, 'commands only'):
self.prepare(True, evaluate=True, predictions='p.jsonl')
test_long_command_no_stage_and_private_filesystem = test_request_deadline_reaps_detached_child = None
test_explicit_error_is_returned = test_valid_predictions_do_not_prove_model_execution = None
if __name__ == '__main__':
unittest.main()