"""Agent-supplied graph/calibration (s04.own_inputs) and the bench dev sync helpers (salty_dev).""" import io import json import hashlib import os import sys import tarfile import tempfile import unittest import unittest.mock from pathlib import Path from types import SimpleNamespace from harness import salty_dev from harness.stages import s04_calibrate try: import onnx from onnx import helper, TensorProto except ImportError: onnx = None def graph(path, out_shape=(1, 10), op='Relu'): x = helper.make_tensor_value_info('in', TensorProto.FLOAT, [1, 3, 8, 8]) y = helper.make_tensor_value_info('out', TensorProto.FLOAT, list(out_shape)) nodes = [helper.make_node('Flatten', ['in'], ['f']), helper.make_node(op, ['f'], ['g']), helper.make_node('Reshape', ['g', 'shape'], ['out'])] shape = helper.make_tensor('shape', TensorProto.INT64, [len(out_shape)], list(out_shape)) onnx.save(helper.make_model(helper.make_graph(nodes, 'g', [x], [y], [shape])), str(path)) @unittest.skipUnless(onnx, 'onnx (bench-venv) required') class OwnInputs(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) rd = Path(self.tmp.name) / 'rollouts/01' self.ws = rd / 'workspace'; self.ws.mkdir(parents=True) self.ctx = SimpleNamespace(stage_root=rd / 'stages', extra={}) self.platform = Path(self.tmp.name) / 'platform.onnx' graph(self.platform) def test_same_io_graph_and_calibration_accepted(self): graph(self.ws / 'mine.onnx', op='Sigmoid') # different internals, same I/O (self.ws / 'calib').mkdir(); (self.ws / 'calib/a.npy').write_bytes(b'x') out = s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'mine.onnx'}, 'calibration_parameters': {'cal_data_dir': 'calib'}}, self.platform) self.assertEqual(out['onnx'], (self.ws / 'mine.onnx').resolve()) self.assertEqual(out['meta']['graph']['sha256'], hashlib.sha256((self.ws / 'mine.onnx').read_bytes()).hexdigest()) self.assertEqual((out['calib'], out['meta']['calib_n'], out['meta']['calib_source']), ((self.ws / 'calib').resolve(), 1, 'workspace')) def test_changed_io_missing_file_and_escape_rejected(self): graph(self.ws / 'bad.onnx', out_shape=(1, 12)) with self.assertRaisesRegex(ValueError, 'differ from the platform graph'): s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'bad.onnx'}}, self.platform) with self.assertRaises(FileNotFoundError): s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'none.onnx'}}, self.platform) (self.ws / 'out.onnx').symlink_to(self.platform) with self.assertRaisesRegex(ValueError, 'outside the workspace'): s04_calibrate.own_inputs(self.ctx, {'model_parameters': {'onnx_model': 'out.onnx'}}, self.platform) (self.ws / 'empty').mkdir() with self.assertRaisesRegex(ValueError, 'no .npy'): s04_calibrate.own_inputs(self.ctx, {'calibration_parameters': {'cal_data_dir': 'empty'}}, self.platform) def test_from_v18_graph_and_calibration_are_required(self): from harness.stagerun import own_inputs_required rd = self.ws.parent self.assertFalse(own_inputs_required(rd)) # no rollout.json for version, required in (('17', False), ('18', True)): (rd / 'rollout.json').write_text(json.dumps({'agent_config_version': version})) self.assertIs(own_inputs_required(rd), required) strict = SimpleNamespace(stage_root=self.ctx.stage_root, extra={'own_inputs_required': True}) with self.assertRaisesRegex(ValueError, 'model_parameters.onnx_model and calibration_parameters.cal_data_dir'): s04_calibrate.own_inputs(strict, {}, self.platform) graph(self.ws / 'mine.onnx') with self.assertRaisesRegex(ValueError, 'own calibration_parameters.cal_data_dir'): s04_calibrate.own_inputs(strict, {'model_parameters': {'onnx_model': 'mine.onnx'}}, self.platform) class DevSync(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) self.rd = Path(self.tmp.name) / 'rollouts/01' self.ws = self.rd / 'workspace'; self.ws.mkdir(parents=True) def test_only_local_changes_since_last_sync_upload(self): (self.ws / 'a.py').write_text('1'); (self.ws / '.pi').mkdir(); (self.ws / '.pi/s').write_text('x') (self.ws / 'guard.log').write_text('x'); (self.ws / 'ln').symlink_to('/etc/passwd') files = salty_dev.local_files(self.ws) self.assertEqual(set(files), {'a.py'}) with salty_dev.synced(self.rd) as manifest: manifest.update(files) (self.ws / 'b.py').write_text('2') with salty_dev.synced(self.rd) as manifest: changed = [r for r, st in salty_dev.local_files(self.ws).items() if manifest.get(r) != st] self.assertEqual(changed, ['b.py']) def archive(self, members): buf = io.BytesIO() with tarfile.open(fileobj=buf, mode='w') as tar: for name, body in members: info = tarfile.TarInfo(name); info.size = len(body); tar.addfile(info, io.BytesIO(body)) data = buf.getvalue() (self.rd / 'out.tar').write_bytes(data) return {'archive': {'sha256': hashlib.sha256(data).hexdigest()}, 'outputs': {n: {'sha256': hashlib.sha256(b).hexdigest()} for n, b in members}} def test_install_checks_bytes_and_paths(self): result = self.archive([('sub/r.txt', b'ok')]) self.assertEqual(salty_dev.install(self.ws, self.rd / 'out.tar', result), ['sub/r.txt']) self.assertEqual((self.ws / 'sub/r.txt').read_bytes(), b'ok') for bad in ([('../x', b'x')], [('.pi/x', b'x')]): with self.assertRaises(ValueError): salty_dev.install(self.ws, self.rd / 'out.tar', self.archive(bad)) (self.ws / 'lnk').symlink_to(self.tmp.name) with self.assertRaisesRegex(ValueError, 'symlink'): salty_dev.install(self.ws, self.rd / 'out.tar', self.archive([('lnk/x', b'x')])) result = self.archive([('r.txt', b'ok')]); result['outputs']['r.txt']['sha256'] = '0' * 64 with self.assertRaises(ValueError): salty_dev.install(self.ws, self.rd / 'out.tar', result) class DevClient(unittest.TestCase): def test_bench_dev_writes_a_terminal_ticket(self): import importlib.util, contextlib spec = importlib.util.spec_from_file_location('stageclient', Path(__file__).parents[1] / 'bin/stageclient.py') client = importlib.util.module_from_spec(spec); spec.loader.exec_module(client) with tempfile.TemporaryDirectory() as d: rd = Path(d); (rd / 'workspace/sub').mkdir(parents=True); (rd / 'handles').mkdir() cwd = os.getcwd() os.chdir(rd / 'workspace/sub') try: with unittest.mock.patch.dict(os.environ, {'BENCH_ROLLOUT_DIR': str(rd)}), \ unittest.mock.patch.object(sys, 'argv', ['stageclient', 'dev', '--timeout', '60', 'python', 'a b.py']), \ contextlib.redirect_stdout(io.StringIO()): self.assertEqual(client.main(), 0) finally: os.chdir(cwd) (hd,) = (rd / 'handles').iterdir() req = json.loads((hd / 'request.json').read_text()) self.assertTrue(hd.name.startswith('terminal-')) self.assertEqual(req, {'stage': 'terminal', 'target': 'development', 'command': "python 'a b.py'", 'timeout': 60, 'cwd': 'sub'}) if __name__ == '__main__': import unittest.mock unittest.main()