- 04 对 v18 起的轨迹(rollout.json 版本 >= 18,stagerun 放进 ctx.extra)要求 compile.yaml 写自己的 onnx_model 和 cal_data_dir,缺了编译前拒绝、不占编译次数;更早的轨迹规则不变 - bench dev 对 v18 轨迹不挂平台 ONNX 和校准 npy,改挂 data/calib-raw(平台 128 张训练图, 反归一化成模型输入尺寸的 8 位图,归一化由 agent 自己做) - scripts/speed_ref.py:每题用默认配置在 dev 容器里编一次,hb_model_info 估计延迟写到 agent/speed_ref/<job_id>.json(jail 里只读可见、实时更新),path-guard 放行这个目录 - 提示词:删掉自己编默认配置做参考、平台图和校准集位置;VERSION 18 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
150 lines
7.8 KiB
Python
150 lines
7.8 KiB
Python
"""Agent-supplied graph/calibration (s04.own_inputs) and the bench dev sync helpers (salty_dev)."""
|
|
import io
|
|
import json
|
|
import hashlib
|
|
import os
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
import unittest.mock
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
from harness import salty_dev
|
|
from harness.stages import s04_calibrate
|
|
|
|
try:
|
|
import onnx
|
|
from onnx import helper, TensorProto
|
|
except ImportError:
|
|
onnx = None
|
|
|
|
|
|
def graph(path, out_shape=(1, 10), op='Relu'):
|
|
x = helper.make_tensor_value_info('in', TensorProto.FLOAT, [1, 3, 8, 8])
|
|
y = helper.make_tensor_value_info('out', TensorProto.FLOAT, list(out_shape))
|
|
nodes = [helper.make_node('Flatten', ['in'], ['f']), helper.make_node(op, ['f'], ['g']),
|
|
helper.make_node('Reshape', ['g', 'shape'], ['out'])]
|
|
shape = helper.make_tensor('shape', TensorProto.INT64, [len(out_shape)], list(out_shape))
|
|
onnx.save(helper.make_model(helper.make_graph(nodes, 'g', [x], [y], [shape])), str(path))
|
|
|
|
|
|
@unittest.skipUnless(onnx, 'onnx (bench-venv) required')
|
|
class OwnInputs(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
|
|
rd = Path(self.tmp.name) / 'rollouts/01'
|
|
self.ws = rd / 'workspace'; self.ws.mkdir(parents=True)
|
|
self.ctx = SimpleNamespace(stage_root=rd / 'stages', extra={})
|
|
self.platform = Path(self.tmp.name) / 'platform.onnx'
|
|
graph(self.platform)
|
|
|
|
def test_same_io_graph_and_calibration_accepted(self):
|
|
graph(self.ws / 'mine.onnx', op='Sigmoid') # different internals, same I/O
|
|
(self.ws / 'calib').mkdir(); (self.ws / 'calib/a.npy').write_bytes(b'x')
|
|
out = s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'mine.onnx'},
|
|
'calibration_parameters': {'cal_data_dir': 'calib'}}, self.platform)
|
|
self.assertEqual(out['onnx'], (self.ws / 'mine.onnx').resolve())
|
|
self.assertEqual(out['meta']['graph']['sha256'], hashlib.sha256((self.ws / 'mine.onnx').read_bytes()).hexdigest())
|
|
self.assertEqual((out['calib'], out['meta']['calib_n'], out['meta']['calib_source']),
|
|
((self.ws / 'calib').resolve(), 1, 'workspace'))
|
|
|
|
def test_changed_io_missing_file_and_escape_rejected(self):
|
|
graph(self.ws / 'bad.onnx', out_shape=(1, 12))
|
|
with self.assertRaisesRegex(ValueError, 'differ from the platform graph'):
|
|
s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'bad.onnx'}}, self.platform)
|
|
with self.assertRaises(FileNotFoundError):
|
|
s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'none.onnx'}}, self.platform)
|
|
(self.ws / 'out.onnx').symlink_to(self.platform)
|
|
with self.assertRaisesRegex(ValueError, 'outside the workspace'):
|
|
s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'out.onnx'}}, self.platform)
|
|
(self.ws / 'empty').mkdir()
|
|
with self.assertRaisesRegex(ValueError, 'no .npy'):
|
|
s04_calibrate.own_inputs(self.ctx, {'calibration_parameters': {'cal_data_dir': 'empty'}}, self.platform)
|
|
|
|
def test_from_v18_graph_and_calibration_are_required(self):
|
|
from harness.stagerun import own_inputs_required
|
|
rd = self.ws.parent
|
|
self.assertFalse(own_inputs_required(rd)) # no rollout.json
|
|
for version, required in (('17', False), ('18', True)):
|
|
(rd / 'rollout.json').write_text(json.dumps({'agent_config_version': version}))
|
|
self.assertIs(own_inputs_required(rd), required)
|
|
strict = SimpleNamespace(stage_root=self.ctx.stage_root, extra={'own_inputs_required': True})
|
|
with self.assertRaisesRegex(ValueError, 'model_parameters.onnx_model and calibration_parameters.cal_data_dir'):
|
|
s04_calibrate.own_inputs(strict, {}, self.platform)
|
|
graph(self.ws / 'mine.onnx')
|
|
with self.assertRaisesRegex(ValueError, 'own calibration_parameters.cal_data_dir'):
|
|
s04_calibrate.own_inputs(strict, {'model_parameters': {'onnx_model': 'mine.onnx'}}, self.platform)
|
|
|
|
|
|
class DevSync(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
|
|
self.rd = Path(self.tmp.name) / 'rollouts/01'
|
|
self.ws = self.rd / 'workspace'; self.ws.mkdir(parents=True)
|
|
|
|
def test_only_local_changes_since_last_sync_upload(self):
|
|
(self.ws / 'a.py').write_text('1'); (self.ws / '.pi').mkdir(); (self.ws / '.pi/s').write_text('x')
|
|
(self.ws / 'guard.log').write_text('x'); (self.ws / 'ln').symlink_to('/etc/passwd')
|
|
files = salty_dev.local_files(self.ws)
|
|
self.assertEqual(set(files), {'a.py'})
|
|
with salty_dev.synced(self.rd) as manifest:
|
|
manifest.update(files)
|
|
(self.ws / 'b.py').write_text('2')
|
|
with salty_dev.synced(self.rd) as manifest:
|
|
changed = [r for r, st in salty_dev.local_files(self.ws).items() if manifest.get(r) != st]
|
|
self.assertEqual(changed, ['b.py'])
|
|
|
|
def archive(self, members):
|
|
buf = io.BytesIO()
|
|
with tarfile.open(fileobj=buf, mode='w') as tar:
|
|
for name, body in members:
|
|
info = tarfile.TarInfo(name); info.size = len(body); tar.addfile(info, io.BytesIO(body))
|
|
data = buf.getvalue()
|
|
(self.rd / 'out.tar').write_bytes(data)
|
|
return {'archive': {'sha256': hashlib.sha256(data).hexdigest()},
|
|
'outputs': {n: {'sha256': hashlib.sha256(b).hexdigest()} for n, b in members}}
|
|
|
|
def test_install_checks_bytes_and_paths(self):
|
|
result = self.archive([('sub/r.txt', b'ok')])
|
|
self.assertEqual(salty_dev.install(self.ws, self.rd / 'out.tar', result), ['sub/r.txt'])
|
|
self.assertEqual((self.ws / 'sub/r.txt').read_bytes(), b'ok')
|
|
for bad in ([('../x', b'x')], [('.pi/x', b'x')]):
|
|
with self.assertRaises(ValueError):
|
|
salty_dev.install(self.ws, self.rd / 'out.tar', self.archive(bad))
|
|
(self.ws / 'lnk').symlink_to(self.tmp.name)
|
|
with self.assertRaisesRegex(ValueError, 'symlink'):
|
|
salty_dev.install(self.ws, self.rd / 'out.tar', self.archive([('lnk/x', b'x')]))
|
|
result = self.archive([('r.txt', b'ok')]); result['outputs']['r.txt']['sha256'] = '0' * 64
|
|
with self.assertRaises(ValueError):
|
|
salty_dev.install(self.ws, self.rd / 'out.tar', result)
|
|
|
|
|
|
class DevClient(unittest.TestCase):
|
|
def test_bench_dev_writes_a_terminal_ticket(self):
|
|
import importlib.util, contextlib
|
|
spec = importlib.util.spec_from_file_location('stageclient', Path(__file__).parents[1] / 'bin/stageclient.py')
|
|
client = importlib.util.module_from_spec(spec); spec.loader.exec_module(client)
|
|
with tempfile.TemporaryDirectory() as d:
|
|
rd = Path(d); (rd / 'workspace/sub').mkdir(parents=True); (rd / 'handles').mkdir()
|
|
cwd = os.getcwd()
|
|
os.chdir(rd / 'workspace/sub')
|
|
try:
|
|
with unittest.mock.patch.dict(os.environ, {'BENCH_ROLLOUT_DIR': str(rd)}), \
|
|
unittest.mock.patch.object(sys, 'argv', ['stageclient', 'dev', '--timeout', '60', 'python', 'a b.py']), \
|
|
contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(client.main(), 0)
|
|
finally:
|
|
os.chdir(cwd)
|
|
(hd,) = (rd / 'handles').iterdir()
|
|
req = json.loads((hd / 'request.json').read_text())
|
|
self.assertTrue(hd.name.startswith('terminal-'))
|
|
self.assertEqual(req, {'stage': 'terminal', 'target': 'development', 'command': "python 'a b.py'",
|
|
'timeout': 60, 'cwd': 'sub'})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import unittest.mock
|
|
unittest.main()
|