prepare 给走 Salty 的 dev 请求多 3 小时排队余量(仍不超过整条轨迹的时限),命令自己的 timeout 在容器启动后才开始算 (worker 用 spec timeout)。10-15:64Gi 的 dev 容器在 Salty 排队超过 agent 给的 300 秒,连 ls 都超时。 prepare 在 collector 里,下次重启后生效。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
170 lines
9.7 KiB
Python
170 lines
9.7 KiB
Python
"""Real Linux namespace/worker regressions; no model/provider/board is contacted."""
|
|
import json
|
|
import hashlib
|
|
import io
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import tarfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import execution, paths, stageworker, terminal
|
|
|
|
|
|
class PredictionContract(unittest.TestCase):
|
|
def test_exact_ordered_coverage(self):
|
|
raw = b'{"sample_id":"b","top5":[2,1,3,4,5]}\n{"sample_id":"a","top5":[1,2,3,4,5]}\n'
|
|
self.assertEqual(terminal.score_predictions(raw, {'a': 1, 'b': 2})['top1'], 100)
|
|
for broken in (raw.splitlines()[0], raw + raw.splitlines()[0], raw.replace(b'"b"', b'"x"'),
|
|
raw.replace(b'[2,1,3,4,5]', b'[2,2,3,4,5]')):
|
|
with self.assertRaises(ValueError):
|
|
terminal.score_predictions(broken, {'a': 1, 'b': 2})
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required')
|
|
class TerminalWorker(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(prefix='terminal-acceptance-')
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.rd = self.root / 'runs/r/j/rollouts/00'
|
|
for name in ('workspace', 'handles', 'stages'):
|
|
(self.rd / name).mkdir(parents=True)
|
|
(self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j', 'march': 'nash-p'}))
|
|
(self.rd / 'rollout.json').write_text(json.dumps({'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'}))
|
|
self.cfg = dict(paths.config())
|
|
self.cfg.update(terminal={'enabled': True}, pools={'cpu': {'workers': 1}})
|
|
self.cfg['paths'] = dict(self.cfg['paths'], harness_python=sys.executable)
|
|
self.config = self.root / 'config.yaml'
|
|
import yaml
|
|
self.config.write_text(yaml.safe_dump(self.cfg))
|
|
policy = dict(version='terminal-test', queue_deadline_s=20, execution_deadline_s=40,
|
|
suspect_after_s=5, cancel_grace_s=.1, cleanup_deadline_s=4,
|
|
ordinary_tool_deadline_s=.1)
|
|
policy['phases'] = {name: {k: policy[k] for k in execution.POLICY_FIELDS} for name in execution.PHASES}
|
|
self.ex = execution.Execution(self.rd); self.ex.start(policy)
|
|
for override in (patch.object(paths, 'STATE', self.root / 'state'),
|
|
patch.object(paths, 'config', return_value=self.cfg),
|
|
patch.dict(os.environ, {'BENCH_CONFIG': str(self.config), 'BENCH_STATE_DIR': str(self.root / 'state')})):
|
|
override.start(); self.addCleanup(override.stop)
|
|
|
|
def submit(self, command, timeout=10, **extra):
|
|
hid = 'terminal-test'
|
|
hd = self.rd / 'handles' / hid; hd.mkdir()
|
|
req = dict(stage='terminal', target='development', command=command, timeout=timeout, **extra)
|
|
(hd / 'request.json').write_text(json.dumps(req))
|
|
worker = stageworker.Worker(self.rd, execution=self.ex)
|
|
worker.scan()
|
|
result = json.loads((hd / 'status.json').read_text())
|
|
return result
|
|
|
|
def test_long_command_no_stage_and_private_filesystem(self):
|
|
marker = self.root / 'hidden-labels.txt'; marker.write_text('must-not-read')
|
|
result = self.submit(f'test ! -e {marker}; sleep .5; printf terminal-ok > result.txt; cat result.txt')
|
|
self.assertEqual(result['result'], 'OK', result)
|
|
self.assertEqual((self.rd / 'workspace/result.txt').read_text(), 'terminal-ok')
|
|
self.assertIn('terminal-ok', result['stdout_tail'])
|
|
self.assertEqual(result['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(result['model_execution_source'], 'UNVERIFIED')
|
|
scopes = self.ex.snapshot()['process_scopes']
|
|
self.assertTrue(scopes)
|
|
self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values()))
|
|
|
|
def test_request_deadline_reaps_detached_child(self):
|
|
result = self.submit('setsid sh -c "sleep 60" & wait', timeout=1)
|
|
self.assertNotEqual(result['result'], 'OK', result)
|
|
scopes = self.ex.snapshot()['process_scopes']
|
|
self.assertTrue(all(execution.scope_receipt(s)['cleanup_state'] == 'VERIFIED' for s in scopes.values()))
|
|
|
|
def test_explicit_error_is_returned(self):
|
|
result = self.submit('echo missing-private-package >&2; exit 9')
|
|
self.assertEqual(result['exit_code'], 9, result)
|
|
self.assertIn('missing-private-package', result['stderr_tail'])
|
|
|
|
def test_valid_predictions_do_not_prove_model_execution(self):
|
|
# Adversarial candidate: writes perfect predictions without loading a
|
|
# model. Scoring is real, but the formal model/HBM claim must stay unknown.
|
|
package = self.root / 'secret-evaluation.tar'
|
|
manifest = {'sha256': 'a' * 64, 'items': [
|
|
{'file': 'a.jpg', 'label': 1}, {'file': 'b.jpg', 'label': 1}]}
|
|
with tarfile.open(package, 'w') as tar:
|
|
for name, raw in [('manifest.json', json.dumps(manifest).encode()), ('a.jpg', b'fixture'), ('b.jpg', b'fixture')]:
|
|
info = tarfile.TarInfo(name); info.size = len(raw)
|
|
tar.addfile(info, io.BytesIO(raw))
|
|
jobpath = self.rd.parent.parent / 'job.json'
|
|
job = json.loads(jobpath.read_text())
|
|
job['data'] = {'t1': {'path': str(package), 'manifest_sha': hashlib.sha256(package.read_bytes()).hexdigest(),
|
|
'content_sha': 'a' * 64, 'n': 2}}
|
|
jobpath.write_text(json.dumps(job))
|
|
(self.rd / 'workspace/fake.py').write_text(
|
|
"import json,pathlib\n"
|
|
"items=json.loads(pathlib.Path('/input/samples.json').read_text())['items']\n"
|
|
"assert all('label' not in x for x in items)\n"
|
|
"pathlib.Path('/out/predictions.jsonl').write_text(''.join(json.dumps({'sample_id':x['sample_id'],'top5':[1,2,3,4,5]})+'\\n' for x in items))\n")
|
|
result = self.submit('/usr/bin/python3 /workspace/fake.py', evaluate=True, predictions='predictions.jsonl')
|
|
self.assertEqual(result['result'], 'OK', result)
|
|
record = json.loads((self.rd / 'terminal_result.json').read_text())
|
|
self.assertEqual(record['metrics']['top1'], 100)
|
|
self.assertEqual(record['measurement_terminal'], 'UNVERIFIED')
|
|
self.assertEqual(record['model_execution_source'], 'UNVERIFIED')
|
|
raw = Path(record['predictions']['path']).read_bytes()
|
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), record['predictions']['sha256'])
|
|
|
|
|
|
class BoardCommand(unittest.TestCase):
|
|
"""bench dev --board: a marked development request becomes a board request only for v18+ tasks."""
|
|
def test_marked_request_goes_to_the_board_for_v18_only(self):
|
|
from types import SimpleNamespace
|
|
req = {'target': 'development', 'command': terminal.BOARD_MARK + 'python3 run.py', 'cwd': 'board',
|
|
'source': '.', 'evaluate': False, 'timeout': 3600, 'deadline_monotonic': time.monotonic() + 7200}
|
|
with tempfile.TemporaryDirectory() as d, \
|
|
patch('harness.salty_dev.download_dir', side_effect=lambda rd, rel, dest: dest.mkdir()) as fetch:
|
|
hd = Path(d)
|
|
ctx = SimpleNamespace(job={'march': 'nash-p'}, extra={'own_inputs_required': False})
|
|
with self.assertRaisesRegex(ValueError, 'BOARD_TERMINAL_UNAVAILABLE'):
|
|
terminal.board_request(ctx, hd, hd, req)
|
|
ctx = SimpleNamespace(job={'march': 'nash-p'}, extra={'own_inputs_required': True})
|
|
board, files = terminal.board_request(ctx, hd, hd, req)
|
|
self.assertEqual((board['target'], board['command'], board['source'], board['cwd']),
|
|
('board', 'python3 run.py', 'board', '.'))
|
|
self.assertLessEqual(board['deadline_monotonic'], time.monotonic() + terminal.BOARD_MAX_S)
|
|
self.assertEqual(fetch.call_args.args[1], 'board')
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'actual Linux namespaces required')
|
|
class SaltyDevAdmission(TerminalWorker):
|
|
"""bench dev on a current (non natural-agent) rollout is admitted only when it goes to Salty, whose
|
|
container sandbox is the isolation; the container run itself was exercised on Salty (10-14)."""
|
|
def setUp(self):
|
|
super().setUp()
|
|
(self.rd / 'rollout.json').write_text(json.dumps({'jail_policy': 'private-home'}))
|
|
self.queue = self.rd / 'stages/_queue/terminal-x'; self.queue.mkdir(parents=True)
|
|
|
|
def prepare(self, salty, timeout=60, **extra):
|
|
cfg = dict(self.cfg, execution={'salty_remote': {'enabled': True, 'stages': ['terminal'] if salty else []}})
|
|
with patch.object(paths, 'config', return_value=cfg):
|
|
return terminal.prepare(self.rd, self.queue, dict(target='development', command='python x.py', timeout=timeout, **extra))
|
|
|
|
def test_admitted_only_through_salty(self):
|
|
self.assertEqual(self.prepare(True)['command'], 'python x.py')
|
|
# A 1 s command still gets the execution's remaining time: waiting for a dev slot or a Salty container
|
|
# is not the command's own (the container enforces the 1 s from its start).
|
|
self.queue = self.rd / 'stages/_queue/terminal-y'; self.queue.mkdir(parents=True)
|
|
self.assertEqual(self.prepare(True, timeout=1)['deadline_monotonic'], self.ex.snapshot()['deadline_monotonic'])
|
|
with self.assertRaisesRegex(ValueError, 'natural-agent-v1'):
|
|
self.prepare(False)
|
|
|
|
def test_formal_evaluation_is_not_a_dev_command(self):
|
|
with self.assertRaisesRegex(ValueError, 'commands only'):
|
|
self.prepare(True, evaluate=True, predictions='p.jsonl')
|
|
|
|
test_long_command_no_stage_and_private_filesystem = test_request_deadline_reaps_detached_child = None
|
|
test_explicit_error_is_returned = test_valid_predictions_do_not_prove_model_execution = None
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|