data-gen-bpu/scripts/test_own_graph.py
bench 1f8a7e20d1 v18:图和校准由 agent 自己做(平台不兜底),速度参考由平台预算
- 04 对 v18 起的轨迹(rollout.json 版本 >= 18,stagerun 放进 ctx.extra)要求 compile.yaml 写自己的
  onnx_model 和 cal_data_dir,缺了编译前拒绝、不占编译次数;更早的轨迹规则不变
- bench dev 对 v18 轨迹不挂平台 ONNX 和校准 npy,改挂 data/calib-raw(平台 128 张训练图,
  反归一化成模型输入尺寸的 8 位图,归一化由 agent 自己做)
- scripts/speed_ref.py:每题用默认配置在 dev 容器里编一次,hb_model_info 估计延迟写到
  agent/speed_ref/<job_id>.json(jail 里只读可见、实时更新),path-guard 放行这个目录
- 提示词:删掉自己编默认配置做参考、平台图和校准集位置;VERSION 18

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 20:46:51 +08:00

150 lines
7.8 KiB
Python

"""Agent-supplied graph/calibration (s04.own_inputs) and the bench dev sync helpers (salty_dev)."""
import io
import json
import hashlib
import os
import sys
import tarfile
import tempfile
import unittest
import unittest.mock
from pathlib import Path
from types import SimpleNamespace
from harness import salty_dev
from harness.stages import s04_calibrate
try:
import onnx
from onnx import helper, TensorProto
except ImportError:
onnx = None
def graph(path, out_shape=(1, 10), op='Relu'):
x = helper.make_tensor_value_info('in', TensorProto.FLOAT, [1, 3, 8, 8])
y = helper.make_tensor_value_info('out', TensorProto.FLOAT, list(out_shape))
nodes = [helper.make_node('Flatten', ['in'], ['f']), helper.make_node(op, ['f'], ['g']),
helper.make_node('Reshape', ['g', 'shape'], ['out'])]
shape = helper.make_tensor('shape', TensorProto.INT64, [len(out_shape)], list(out_shape))
onnx.save(helper.make_model(helper.make_graph(nodes, 'g', [x], [y], [shape])), str(path))
@unittest.skipUnless(onnx, 'onnx (bench-venv) required')
class OwnInputs(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
rd = Path(self.tmp.name) / 'rollouts/01'
self.ws = rd / 'workspace'; self.ws.mkdir(parents=True)
self.ctx = SimpleNamespace(stage_root=rd / 'stages', extra={})
self.platform = Path(self.tmp.name) / 'platform.onnx'
graph(self.platform)
def test_same_io_graph_and_calibration_accepted(self):
graph(self.ws / 'mine.onnx', op='Sigmoid') # different internals, same I/O
(self.ws / 'calib').mkdir(); (self.ws / 'calib/a.npy').write_bytes(b'x')
out = s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'mine.onnx'},
'calibration_parameters': {'cal_data_dir': 'calib'}}, self.platform)
self.assertEqual(out['onnx'], (self.ws / 'mine.onnx').resolve())
self.assertEqual(out['meta']['graph']['sha256'], hashlib.sha256((self.ws / 'mine.onnx').read_bytes()).hexdigest())
self.assertEqual((out['calib'], out['meta']['calib_n'], out['meta']['calib_source']),
((self.ws / 'calib').resolve(), 1, 'workspace'))
def test_changed_io_missing_file_and_escape_rejected(self):
graph(self.ws / 'bad.onnx', out_shape=(1, 12))
with self.assertRaisesRegex(ValueError, 'differ from the platform graph'):
s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'bad.onnx'}}, self.platform)
with self.assertRaises(FileNotFoundError):
s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'none.onnx'}}, self.platform)
(self.ws / 'out.onnx').symlink_to(self.platform)
with self.assertRaisesRegex(ValueError, 'outside the workspace'):
s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'out.onnx'}}, self.platform)
(self.ws / 'empty').mkdir()
with self.assertRaisesRegex(ValueError, 'no .npy'):
s04_calibrate.own_inputs(self.ctx, {'calibration_parameters': {'cal_data_dir': 'empty'}}, self.platform)
def test_from_v18_graph_and_calibration_are_required(self):
from harness.stagerun import own_inputs_required
rd = self.ws.parent
self.assertFalse(own_inputs_required(rd)) # no rollout.json
for version, required in (('17', False), ('18', True)):
(rd / 'rollout.json').write_text(json.dumps({'agent_config_version': version}))
self.assertIs(own_inputs_required(rd), required)
strict = SimpleNamespace(stage_root=self.ctx.stage_root, extra={'own_inputs_required': True})
with self.assertRaisesRegex(ValueError, 'model_parameters.onnx_model and calibration_parameters.cal_data_dir'):
s04_calibrate.own_inputs(strict, {}, self.platform)
graph(self.ws / 'mine.onnx')
with self.assertRaisesRegex(ValueError, 'own calibration_parameters.cal_data_dir'):
s04_calibrate.own_inputs(strict, {'model_parameters': {'onnx_model': 'mine.onnx'}}, self.platform)
class DevSync(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
self.rd = Path(self.tmp.name) / 'rollouts/01'
self.ws = self.rd / 'workspace'; self.ws.mkdir(parents=True)
def test_only_local_changes_since_last_sync_upload(self):
(self.ws / 'a.py').write_text('1'); (self.ws / '.pi').mkdir(); (self.ws / '.pi/s').write_text('x')
(self.ws / 'guard.log').write_text('x'); (self.ws / 'ln').symlink_to('/etc/passwd')
files = salty_dev.local_files(self.ws)
self.assertEqual(set(files), {'a.py'})
with salty_dev.synced(self.rd) as manifest:
manifest.update(files)
(self.ws / 'b.py').write_text('2')
with salty_dev.synced(self.rd) as manifest:
changed = [r for r, st in salty_dev.local_files(self.ws).items() if manifest.get(r) != st]
self.assertEqual(changed, ['b.py'])
def archive(self, members):
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode='w') as tar:
for name, body in members:
info = tarfile.TarInfo(name); info.size = len(body); tar.addfile(info, io.BytesIO(body))
data = buf.getvalue()
(self.rd / 'out.tar').write_bytes(data)
return {'archive': {'sha256': hashlib.sha256(data).hexdigest()},
'outputs': {n: {'sha256': hashlib.sha256(b).hexdigest()} for n, b in members}}
def test_install_checks_bytes_and_paths(self):
result = self.archive([('sub/r.txt', b'ok')])
self.assertEqual(salty_dev.install(self.ws, self.rd / 'out.tar', result), ['sub/r.txt'])
self.assertEqual((self.ws / 'sub/r.txt').read_bytes(), b'ok')
for bad in ([('../x', b'x')], [('.pi/x', b'x')]):
with self.assertRaises(ValueError):
salty_dev.install(self.ws, self.rd / 'out.tar', self.archive(bad))
(self.ws / 'lnk').symlink_to(self.tmp.name)
with self.assertRaisesRegex(ValueError, 'symlink'):
salty_dev.install(self.ws, self.rd / 'out.tar', self.archive([('lnk/x', b'x')]))
result = self.archive([('r.txt', b'ok')]); result['outputs']['r.txt']['sha256'] = '0' * 64
with self.assertRaises(ValueError):
salty_dev.install(self.ws, self.rd / 'out.tar', result)
class DevClient(unittest.TestCase):
def test_bench_dev_writes_a_terminal_ticket(self):
import importlib.util, contextlib
spec = importlib.util.spec_from_file_location('stageclient', Path(__file__).parents[1] / 'bin/stageclient.py')
client = importlib.util.module_from_spec(spec); spec.loader.exec_module(client)
with tempfile.TemporaryDirectory() as d:
rd = Path(d); (rd / 'workspace/sub').mkdir(parents=True); (rd / 'handles').mkdir()
cwd = os.getcwd()
os.chdir(rd / 'workspace/sub')
try:
with unittest.mock.patch.dict(os.environ, {'BENCH_ROLLOUT_DIR': str(rd)}), \
unittest.mock.patch.object(sys, 'argv', ['stageclient', 'dev', '--timeout', '60', 'python', 'a b.py']), \
contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(client.main(), 0)
finally:
os.chdir(cwd)
(hd,) = (rd / 'handles').iterdir()
req = json.loads((hd / 'request.json').read_text())
self.assertTrue(hd.name.startswith('terminal-'))
self.assertEqual(req, {'stage': 'terminal', 'target': 'development', 'command': "python 'a b.py'",
'timeout': 60, 'cwd': 'sub'})
if __name__ == '__main__':
import unittest.mock
unittest.main()