data-gen-bpu/scripts/test_recipe_pipeline.py

672 lines
39 KiB
Python

"""Independent wiring checks; namespace/model checks require an explicit Linux run.
The portable tests exercise actual tickets, snapshots and numeric proxy messages.
They do not claim model, namespace or board acceptance. Enable
BENCH_TEST_RECIPE_NAMESPACE=1 in an existing Torch/ONNX runtime for those tests.
"""
from contextlib import ExitStack
import hashlib
import io
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import tarfile
import tempfile
import textwrap
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import numpy as np
from harness import adapters, paths, recipes, registry, stageworker, stagerun
from harness.eval.common import RecipeRunner, T1
BENCH = Path(__file__).resolve().parents[1]
NAMESPACE = os.environ.get("BENCH_TEST_RECIPE_NAMESPACE") == "1"
def sha(value):
return hashlib.sha256(value).hexdigest()
class Files(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(prefix="recipe-pipeline-")
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name).resolve()
self.rd = self.root / "job/rollouts/00"
self.workspace = self.rd / "workspace"
self.workspace.mkdir(parents=True)
(self.rd / "handles").mkdir()
self.reference = b"raise RuntimeError('reference must never be imported by control process')\n"
self.candidate = b"raise RuntimeError('candidate must never be imported by control process')\n"
self.weights = b"pinned weights"
self.job = {"run_id": "test-run", "job_id": "recipe-fixture", "task": "seg", "model": "new.seg",
"backend": "ptq", "march": "nash-m", "model_spec": {
"source": "recipe", "input_spec": "1x3x2x2", "preprocess": {}},
"model_recipe": {"reference": {"entrypoint": "reference.py",
"files": {"reference.py": sha(self.reference)}, "weights_sha256": sha(self.weights)}}}
self.manifest = {"schema": recipes.SCHEMA, "files": ["candidate.py", "reference.py", "weights.bin"],
"entrypoint": "candidate.py", "weights": "weights.bin", "input_spec": "1x3x2x2",
"preprocess": {}}
self.save()
def save(self):
for name, data in {"candidate.py": self.candidate, "reference.py": self.reference,
"weights.bin": self.weights}.items():
(self.workspace / name).write_bytes(data)
(self.workspace / "recipe.json").write_text(json.dumps(self.manifest))
(self.rd.parent.parent / "job.json").write_text(json.dumps(self.job))
def frozen(self, name="snapshot"):
return recipes.snapshot(self.workspace, "recipe.json", self.root / name, self.job)
def client(self, *args):
result = subprocess.run([sys.executable, str(BENCH / "bin/stageclient.py"), "stage", "run", *args],
cwd=self.workspace, env={**os.environ, "BENCH_ROLLOUT_DIR": str(self.rd)},
capture_output=True, text=True, timeout=15)
self.assertEqual(result.returncode, 0, result.stderr)
return result.stdout.splitlines()[0]
class JobPolicy(Files):
def build(self, *, recipe):
row = {"model_id": "new.seg", "task": "seg", "source": "custom", "input_spec": "1x3x2x2",
"dataset": "cityscapes", "preprocess": {"resize": 2}, "runtime_python": "/platform/bin/python"}
spec = registry.JobSpec("seg--new.seg--nash-m--ptq--int8", "seg", "new.seg", "nash-m", "ptq",
"int8", "P1", "e2e", overrides={"model_recipe": self.job["model_recipe"]} if recipe else {})
with ExitStack() as stack:
for name, value in (("models", [row]), ("board_type_for_march", "s100"), ("board_for_march", {}),
("gate_snapshot", {"required": ["G1", "G3"]}), ("registry_fingerprint", "registry"),
("datasets", {"datasets": {"cityscapes": {"version": "fixed", "t1": {"n": 2}}}})):
stack.enter_context(patch.object(registry, name, return_value=value))
stack.enter_context(patch.object(paths, "DATA", self.root / "data"))
stack.enter_context(patch.object(paths, "config", return_value={"paths": {}, "model_runtimes": {
"new.seg": "/configured/bin/python"}}))
stack.enter_context(patch("harness.toolchains.resolve", return_value={"name": "fixed-toolchain"}))
return registry.build_job_json(spec, "run", "test-evaluator")
def test_recipe_job_never_imports_same_name_public_adapter(self):
with patch.object(adapters, "get", side_effect=AssertionError("public model adapter imported")):
job = self.build(recipe=True)
self.assertEqual(job["model_spec"]["input_spec"], "1x3x2x2")
self.assertEqual(job["model_spec"]["preprocess"], {"resize": 2})
self.assertEqual(job["model_runtime"]["python"], "/configured/bin/python")
self.assertEqual(job["cache_contract"]["exporter_version"], "task-recipe-v1")
self.assertEqual(job["model_recipe"]["reference"]["preprocess"], {"resize": 2})
self.assertEqual(job["data"]["dataset"], "cityscapes")
def test_legacy_job_keeps_public_adapter_resolution(self):
adapter = SimpleNamespace(MODEL_SPEC={"input_spec": "1x3x8x8"}, PREPROCESS={"resize": 8},
RUNTIME_PYTHON="/old/bin/python", EXPORTER_VERSION="old-exporter")
with patch.object(adapters, "get", return_value=adapter) as get:
job = self.build(recipe=False)
self.assertGreater(get.call_count, 0)
self.assertNotIn("model_recipe", job)
self.assertEqual(job["model_spec"]["input_spec"], "1x3x8x8")
self.assertEqual(job["model_spec"]["preprocess"], {"resize": 8})
self.assertEqual(job["model_spec"]["runtime_python"], "/old/bin/python")
self.assertEqual(job["cache_contract"]["exporter_version"], "old-exporter")
def test_e2e_expansion_publishes_recipe_without_legacy_model_lookup(self):
from harness import db, pipeline
job = self.build(recipe=True)
spec = registry.JobSpec(job["job_id"], "seg", "new.seg", "nash-m", "ptq", "int8", "P1", "e2e",
policy="teacher", overrides={"model_recipe": self.job["model_recipe"]})
database = self.root / "jobs.sqlite"
original_connect = db.connect
with ExitStack() as stack:
stack.enter_context(patch.object(db, "STATE", self.root))
stack.enter_context(patch.object(db, "connect", side_effect=lambda **kw: original_connect(database, **kw)))
stack.enter_context(patch.object(paths, "RUNS", self.root / "runs"))
stack.enter_context(patch.object(paths, "DATA", self.root / "data"))
stack.enter_context(patch.object(pipeline, "model_row", side_effect=AssertionError("legacy model imported in expand")))
stack.enter_context(patch.object(registry, "models", return_value=[dict(job["model_spec"], source="custom")]))
stack.enter_context(patch.object(adapters, "get", side_effect=AssertionError("public adapter imported while publishing recipe job")))
stack.enter_context(patch.object(pipeline, "expand_matrix", return_value=[spec]))
stack.enter_context(patch.object(pipeline, "build_job_json", return_value=job))
with db.connect() as con:
db.create_run(con, "run", {})
result = pipeline.expand("run", modes=["e2e"], model_ids=["new.seg"])
saved = json.loads((paths.RUNS / "run" / job["job_id"] / "job.json").read_text())
with db.connect() as con:
count = con.execute("SELECT COUNT(*) FROM jobs WHERE run_id='run'").fetchone()[0]
self.assertEqual(result["jobs_new"], 1)
self.assertEqual(count, 1)
self.assertEqual(saved["model_recipe"], job["model_recipe"])
class ClientWorker(Files):
def execute(self, worker):
captured = []
def accept(command, **kwargs):
# Worker now crosses a process boundary. Observe the protected
# executor command, not an in-process _exec call that must not occur.
self.assertEqual(command[1:4], ['-m', 'harness.stagerun', '--exec-protected'])
queue = Path(command[4])
self.assertEqual(Path(command[5]), self.rd)
request = json.loads((queue / "request.json").read_text())
captured.append((request, recipes.load(Path(request["recipe"]))))
kwargs['on_start'](SimpleNamespace(pid=os.getpid()))
(queue / 'status.json').write_text('{"state":"DONE","result":"OK"}')
return stageworker.watchdog.RunResult('OK', 0, 0, kwargs['stdout'], kwargs['stderr'])
with patch.object(stageworker.watchdog, "run", side_effect=accept):
worker.scan()
return captured
def test_real_cli_ticket_freezes_and_submission_ignores_agent_status(self):
first = self.client("fetch", "--recipe", "workspace/recipe.json")
worker = stageworker.Worker(self.rd)
initial = self.execute(worker)
self.assertEqual(len(initial), 1, (self.rd / "handles" / first / "status.json").read_text())
request, frozen = initial[0]
self.assertTrue(Path(request["recipe"]).is_relative_to(self.rd / "stages/_queue"))
(self.workspace / "candidate.py").write_text("changed after submission")
(self.rd / "handles" / first / "status.json").write_text(json.dumps({
"state": "DONE", "result": "OK", "recipe": "/other/rollout/recipe.json"}))
(self.rd / "handles" / first / "request.json").write_text('{"recipe":"/forged"}')
self.client("export", "--submission", first)
reused = self.execute(worker)
self.assertEqual(len(reused), 1)
self.assertEqual(reused[0][1]["submission_sha256"], frozen["submission_sha256"])
self.assertEqual((Path(reused[0][1]["root"]) / "candidate.py").read_bytes(), self.candidate)
def test_worker_rejects_cross_rollout_and_symlinked_recipe(self):
(self.workspace / "linked.json").symlink_to(self.workspace / "recipe.json")
linked = self.client("fetch", "--recipe", "linked.json")
other = self.root / "other"
other.mkdir()
(other / "recipe.json").write_text((self.workspace / "recipe.json").read_text())
# A malicious Agent may bypass the helpful client validation entirely.
outside = "01-fetch-outside"
outside_handle = self.rd / "handles" / outside
outside_handle.mkdir()
(outside_handle / "request.json").write_text(json.dumps({
"stage": "01-fetch", "recipe": str(other / "recipe.json")}))
worker = stageworker.Worker(self.rd)
with patch.object(stageworker.watchdog, "run", side_effect=AssertionError("untrusted recipe executed")) as dispatch:
worker.scan()
dispatch.assert_not_called()
for handle in (linked, outside):
status = json.loads((self.rd / "handles" / handle / "status.json").read_text())
self.assertEqual(status["result"], "FAIL")
self.assertEqual(status["error_class"], "invalid_request")
queue = self.rd / "stages/_queue" / handle
self.assertIn("invalid_request", (queue / "runner.log").read_text())
self.assertIn("request:", (queue / "runner.log").read_text())
self.assertFalse((queue / "request.json").exists()) # Invalid input is evidence, not accepted authority.
self.assertEqual(json.loads((queue / 'operation.json').read_text())['state'], 'DONE')
with patch.object(stageworker.watchdog, "run") as dispatch:
stageworker.Worker(self.rd).scan()
dispatch.assert_not_called()
self.assertEqual(json.loads((self.rd/'handles'/linked/'status.json').read_text())['error_class'], 'invalid_request')
def test_forged_submission_handle_cannot_point_to_other_rollout(self):
handle = "03-export-forged"
hd = self.rd / "handles" / handle
hd.mkdir()
(hd / "request.json").write_text(json.dumps({"stage": "03-export", "submission": "../01/foreign"}))
with patch.object(stageworker.watchdog, "run", side_effect=AssertionError("cross rollout executed")) as dispatch:
stageworker.Worker(self.rd).scan()
dispatch.assert_not_called()
status = json.loads((hd / "status.json").read_text())
self.assertEqual(status["error_class"], "invalid_request")
@unittest.skipUnless(sys.platform == 'linux', 'Linux managed process identities')
def test_prior_preparing_operation_still_quarantines_unknown_side_effects(self):
from harness.execution import Execution
from scripts.test_execution import policy
handle = self.client('fetch', '--recipe', 'recipe.json')
owner = Execution(self.rd); owner.start(policy())
request = json.loads((self.rd/'handles'/handle/'request.json').read_text())
digest = sha(json.dumps(request, sort_keys=True, separators=(',', ':')).encode())
queue = self.rd/'stages/_queue'/handle; queue.mkdir(parents=True)
(queue/'operation.json').write_text(json.dumps({'operation_id':handle, 'payload_sha256':digest,
'state':'PREPARING', 'execution_id':owner.snapshot()['execution_id'], 'epoch':owner.epoch}))
with patch.object(stageworker.watchdog, 'run') as dispatch:
stageworker.Worker(self.rd, execution=owner).scan()
dispatch.assert_not_called()
status = json.loads((self.rd/'handles'/handle/'status.json').read_text())
self.assertEqual(status['error_class'], 'SIDE_EFFECT_UNKNOWN')
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
@unittest.skipUnless(sys.platform == 'linux', 'Linux managed process identities')
def test_invalid_new_ticket_has_durable_rejection_without_quarantining_execution(self):
from harness.execution import Execution
from scripts.test_execution import policy
owner = Execution(self.rd); owner.start(policy())
handle = '03-export-invalid'; hd = self.rd/'handles'/handle; hd.mkdir()
(hd/'request.json').write_text('{"stage":"03-export","submission":"../foreign"}')
with patch.object(stageworker.watchdog, 'run') as dispatch:
stageworker.Worker(self.rd, execution=owner).scan()
stageworker.Worker(self.rd, execution=owner).scan()
dispatch.assert_not_called()
self.assertEqual(json.loads((hd/'status.json').read_text())['error_class'], 'invalid_request')
self.assertEqual(owner.snapshot()['operations'][handle]['state'], 'DONE')
self.assertNotEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
@unittest.skipUnless(sys.platform == 'linux', 'Linux kernel cleanup receipts')
def test_receipt_recovery_requires_identity_and_kernel_cleanup(self):
from harness import execution, watchdog
from scripts.test_execution import policy
for fault in (None, 'operation_id', 'payload_sha256', 'epoch', 'ledger_payload',
'scope_missing', 'scope_receipt_missing', 'board_receipt_missing'):
with self.subTest(fault=fault):
rd = self.root / ('receipt-' + str(fault))
hd = rd / 'handles/h1'; hd.mkdir(parents=True)
queue = rd / 'stages/_queue/h1'; queue.mkdir(parents=True)
name = '07-board' if fault == 'board_receipt_missing' else '01-fetch'
req = {'stage': name}
(hd / 'request.json').write_text(json.dumps(req))
owner = execution.Execution(rd); owner.start(policy())
previous = dict(operation_id='h1', payload_sha256=sha(json.dumps(req, sort_keys=True,
separators=(',', ':')).encode()), execution_id=owner.snapshot()['execution_id'],
epoch=owner.epoch, state='PREPARING')
def started(proc):
previous.update(state='DISPATCHED', executor=execution.process_identity(proc.pid))
execution.atomic_json(queue / 'operation.json', previous)
with owner.mutation() as state:
state['operations']['h1'] = dict(previous)
owner._save(state, 'TEST_DISPATCHED')
code = ('import json,os,sys; from pathlib import Path; from harness.execution import process_identity; '
'p=Path(sys.argv[1]); s=json.loads((p/"operation.json").read_text()); '
's.update(state="DONE",result="OK",process=process_identity(os.getpid())); '
'(p/"status.json").write_text(json.dumps(s))')
with owner.scope(name, 'h1'):
run = watchdog.run([sys.executable, '-c', code, str(queue)], cwd=BENCH,
stdout=queue/'out', stderr=queue/'err', execution=owner, role='stage',
on_start=started, poll_s=.01)
self.assertEqual(run.state, 'OK', (queue/'err').read_text())
receipt_path = queue/'status.json'
receipt = json.loads(receipt_path.read_text())
if fault in ('operation_id', 'payload_sha256', 'epoch'):
receipt[fault] = 'foreign'
receipt_path.write_text(json.dumps(receipt))
if fault in ('ledger_payload', 'scope_missing', 'scope_receipt_missing'):
with owner.mutation() as state:
if fault == 'ledger_payload':
state['operations']['h1']['payload_sha256'] = 'foreign'
elif fault == 'scope_missing':
state['process_scopes'] = {}
else:
Path(next(iter(state['process_scopes'].values()))['receipt_path']).unlink()
owner._save(state, 'TEST_FAULT')
before = receipt_path.read_bytes()
with patch.object(stageworker.watchdog, 'run') as dispatch:
stageworker.Worker(rd, execution=owner).scan()
dispatch.assert_not_called()
self.assertEqual(receipt_path.read_bytes(), before)
if fault:
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertNotEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
self.assertEqual(owner.snapshot()['stop_reason'], 'OPERATION_STATE_UNKNOWN')
if (hd/'status.json').exists():
self.assertNotEqual(json.loads((hd/'status.json').read_text()).get('state'), 'DONE')
else:
self.assertEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
self.assertEqual(owner.finish(0, protocol_complete=True)['execution_outcome'], 'COMPLETED')
def test_old_unprotected_exec_rejects_even_valid_recipe(self):
frozen = self.frozen()
handle = self.rd / "handles/unprotected"
handle.mkdir()
(handle / "request.json").write_text(json.dumps({"stage": "03-export", "recipe": str(Path(frozen["root"]) / "recipe.json")}))
with patch("harness.recipe_runtime.runtime_identity", side_effect=AssertionError("runtime reached")):
with self.assertRaisesRegex(ValueError, "protected outer Worker"):
stagerun._exec(handle)
class NumericProxy(Files):
def test_control_plane_runtime_identity_needs_no_model_packages(self):
code = textwrap.dedent('''
import json, pathlib, sys
from harness.recipe_runtime import runtime_identity
identity = runtime_identity(pathlib.Path(sys.executable))
print(json.dumps({"sha256": identity["sha256"],
"loaded": sorted(set(sys.modules) & {"numpy", "torch", "onnx", "PIL"})}))
''')
result = subprocess.run([sys.executable, "-S", "-c", code], cwd=BENCH,
env={"PATH": "/usr/bin:/bin"}, capture_output=True, text=True, timeout=60)
self.assertEqual(result.returncode, 0, result.stderr)
evidence = json.loads(result.stdout)
self.assertEqual(evidence["loaded"], [])
self.assertEqual(len(evidence["sha256"]), 64)
def runner(self, **kwargs):
return RecipeRunner(self.frozen(), sys.executable, self.root / "calls", "cityscapes", **kwargs)
def test_proxy_uses_numeric_operations_without_importing_candidate(self):
runner = self.runner()
calls = []
def call(python, recipe, operation, values, work, **kwargs):
calls.append((operation, values, kwargs))
if operation == "preprocess":
return {key: value.transpose(2, 0, 1)[None].astype(np.float32) for key, value in values.items()}
if operation == "predict":
return {key + "/out0": np.ones((1, 19, 2, 2), np.float32) for key in values}
return values
with patch("harness.recipe_runtime.call", side_effect=call), \
patch.object(adapters, "get", side_effect=AssertionError("candidate/public import")):
outputs = runner.batch(runner.prepare([np.zeros((2, 2, 3), np.uint8)]))
self.assertEqual([x[0] for x in calls], ["preprocess", "predict", "decode"])
self.assertEqual(len(outputs), 1)
self.assertEqual(outputs[0][0].shape, (1, 19, 2, 2))
self.assertTrue(all(isinstance(value, np.ndarray) for _, values, _ in calls for value in values.values()))
def test_missing_extra_noncontiguous_sample_outputs_fail(self):
runner = self.runner()
array = np.zeros((1, 19, 2, 2), np.float32)
for values in ({}, {"s0000/out1": array}, {"s0000/out0": array, "s0001/out0": array}):
with self.subTest(keys=list(values)), patch("harness.recipe_runtime.call", return_value=values):
with self.assertRaisesRegex(ValueError, "identities"):
runner.batch([np.zeros((1, 3, 2, 2), np.float32)])
def test_preprocess_cannot_change_sample_count_shape_or_dtype(self):
runner = self.runner()
for values in ({}, {"s0000": np.zeros((1, 3, 3, 3), np.float32)},
{"s0000": np.zeros((1, 3, 2, 2), np.float64)}):
with self.subTest(keys=list(values)), patch("harness.recipe_runtime.call", return_value=values):
with self.assertRaisesRegex(ValueError, "preprocessing"):
runner.prepare([np.zeros((2, 2, 3), np.uint8)])
def test_g2_uses_current_deploy_preprocessing_not_old_reference_tensors(self):
from harness.eval.run import numeric_diagnostic
self.manifest["input_spec"] = "1x3x4x4"
self.save()
runner = self.runner()
baseline = self.root / "baseline"
baseline.mkdir()
np.savez(baseline / "g2_inputs.npz", s000=np.zeros((1, 3, 2, 2), np.float32))
np.savez(baseline / "g2_images.npz", s000=np.ones((2, 2, 3), np.uint8))
np.savez(baseline / "g2_fp32.npz", **{"s000/out0": np.ones((1, 19, 2, 2), np.float32)})
seen = []
def call(python, recipe, operation, values, work, **kwargs):
seen.append(operation)
if operation == "preprocess":
return {key: np.ones((1, 3, 4, 4), np.float32) for key in values}
if operation == "predict":
for value in values.values():
self.assertEqual(value.shape, (1, 3, 4, 4), "old reference input leaked into deploy runtime")
return {key + "/out0": np.ones((1, 19, 2, 2), np.float32) for key in values}
return values
with patch("harness.recipe_runtime.call", side_effect=call):
result = numeric_diagnostic(runner, baseline, "t0")
self.assertEqual(seen, ["preprocess", "predict", "decode"])
self.assertAlmostEqual(result["cos_min"], 1.0)
@unittest.skipUnless(NAMESPACE, "not run: requires BENCH_TEST_RECIPE_NAMESPACE=1 and Linux model runtime")
class SegmentationAndGraph(Files):
def setUp(self):
super().setUp()
import importlib.util
self.assertEqual(sys.platform, "linux", "namespace acceptance must run on Linux")
for name in ("torch", "onnx", "onnxruntime", "PIL"):
self.assertIsNotNone(importlib.util.find_spec(name), f"missing required model package: {name}")
def t1(self):
from PIL import Image
archive = self.root / "t1.tar"
labels = [np.array([[7, 8], [7, 8]], np.uint8), np.array([[8, 7], [8, 7]], np.uint8)]
items = []
with tarfile.open(archive, "w") as stream:
def add(name, data):
info = tarfile.TarInfo(name)
info.size = len(data)
stream.addfile(info, io.BytesIO(data))
for i, label in enumerate(labels):
item = {"id": f"image-{i}", "file": f"image-{i}.png", "label_file": f"label-{i}.png",
"label_format": "cityscapes_labelIds"}
items.append(item)
image = np.repeat((label == 8)[..., None], 3, axis=-1).astype(np.uint8)
for name, value in ((item["file"], image), (item["label_file"], label)):
data = io.BytesIO()
Image.fromarray(value).save(data, format="PNG")
add(name, data.getvalue())
add("manifest.json", json.dumps({"dataset": "cityscapes", "sha256": "fixed-t1", "items": items}).encode())
result = T1.open(archive)
self.addCleanup(result.tar.close)
return result
def test_fixed_t1_platform_scores_numeric_proxy_predictions(self):
from harness.eval import seg
t1 = self.t1()
runner = RecipeRunner(self.frozen(), sys.executable, self.root / "calls", "cityscapes")
def call(python, recipe, operation, values, work, **kwargs):
if operation == "preprocess":
return {key: value.transpose(2, 0, 1)[None].astype(np.float32) for key, value in values.items()}
if operation == "predict":
result = {}
for key, value in values.items():
logits = np.zeros((1, 19, 2, 2), np.float32)
logits[0, 0] = 1 - value[0, 0]
logits[0, 1] = value[0, 0]
result[key + "/out0"] = logits
return result
return values
baseline = {"model": "new.seg", "t1_manifest_sha": "fixed-t1"}
with patch("harness.recipe_runtime.call", side_effect=call), \
patch.object(seg, "get", side_effect=AssertionError("public adapter imported")):
seg.run_fp32(runner, t1, "1x3x2x2", baseline)
quant = {"model": "new.seg", "t1_manifest_sha": "fixed-t1"}
seg.run_quant(runner, t1, "1x3x2x2", baseline, quant)
self.assertEqual(quant["G3"]["quant"], 100.0)
self.assertEqual([item["id"] for item in quant["_seg_samples"]], ["image-0", "image-1"])
self.assertNotIn("source", quant["G3"])
def real_model(self, variant="correct", direct=False):
# No weights are deserialized in this fixture; real source files still
# cross the normal digest/snapshot/process boundaries.
module = textwrap.dedent('''
import numpy as np
import torch
class Model(torch.nn.Module):
def forward(self, x):
zero = x[:, :1] * 0
return torch.cat((1-x[:, :1], x[:, :1], *([zero]*17)), dim=1)
def build(weights, input_spec):
return Model()
def preprocess(image, input_spec):
return np.asarray(image, dtype=np.float32).transpose(2,0,1)[None]
''').encode()
self.reference = module
self.candidate = module
self.job["model_recipe"]["reference"]["files"] = {"reference.py": sha(module)}
if variant == "missing_output":
self.candidate += b"\ndef decode(outputs, input_spec): return []\n"
if direct:
import onnx
from onnx import helper, TensorProto, numpy_helper
channels = 3 if variant == "removed_head" else 19
output = np.zeros((1, channels, 2, 2), np.float32)
output[:, 0] = 1
if variant == "wrong_prediction":
output[:, 0] = 0
output[:, 1] = 1
graph = helper.make_graph([helper.make_node("Constant", [], ["logits"], value=numpy_helper.from_array(output))],
"fixture", [helper.make_tensor_value_info("input", TensorProto.FLOAT, [1, 3, 2, 2])],
[helper.make_tensor_value_info("logits", TensorProto.FLOAT, list(output.shape))])
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 11)])
model.ir_version = 7
onnx.save(model, self.workspace / "candidate.onnx")
self.manifest["files"].append("candidate.onnx")
self.manifest["onnx"] = "candidate.onnx"
self.save()
frozen = self.frozen()
baseline = self.root / "baseline"
baseline.mkdir()
image = np.zeros((2, 2, 3), np.uint8)
logits = np.zeros((1, 19, 2, 2), np.float32)
logits[:, 0] = 1
np.savez(baseline / "g2_images.npz", s000=image)
np.savez(baseline / "g2_fp32.npz", **{"s000/out0": logits})
(baseline / "metrics.json").write_text(json.dumps({"seg_protocol": {"dataset": "cityscapes", "classes": 19}}))
return SimpleNamespace(recipe=str(Path(frozen["root"]) / "recipe.json"), out=str(self.root / "stage/graph/model.onnx"),
baseline=str(baseline), dataset="cityscapes")
def test_export_script_checks_graph_without_quality_precondition(self):
from harness.tools.export_onnx import export_recipe
args = self.real_model()
self.assertEqual(export_recipe(args), 0)
acceptance = json.loads((Path(args.out).parent / "acceptance.json").read_text())
self.assertIn("onnx", acceptance)
self.assertNotIn("comparisons", acceptance)
def test_direct_graph_runs_without_reexport(self):
from harness.tools.export_onnx import export_recipe
args = self.real_model(direct=True)
with patch("harness.recipe_runtime.export", side_effect=AssertionError("direct graph was needlessly re-exported")):
self.assertEqual(export_recipe(args), 0)
self.assertTrue((Path(args.out).parent / "acceptance.json").is_file())
def score_export(self, args):
from harness.eval import seg
from harness.gates import g3
frozen = recipes.load(Path(args.recipe))
t1 = self.t1()
baseline = {"model": "new.seg", "t1_manifest_sha": "fixed-t1"}
reference = RecipeRunner(frozen, sys.executable, self.root / "official-calls", "cityscapes", reference=True)
seg.run_fp32(reference, t1, "1x3x2x2", baseline)
quant = {"model": "new.seg", "t1_manifest_sha": "fixed-t1"}
candidate = RecipeRunner(frozen, sys.executable, self.root / "candidate-calls", "cityscapes", graph=Path(args.out))
seg.run_quant(candidate, t1, "1x3x2x2", baseline, quant)
return g3(quant["G3"], {"metric": "miou", "direction": "higher", "kind": "abs_pp", "tol": -1})
def test_removed_prediction_head_can_be_exported_but_incomplete_predictions_fail_scoring(self):
from harness.tools.export_onnx import export_recipe
args = self.real_model("removed_head", direct=True)
self.assertEqual(export_recipe(args), 0)
with self.assertRaisesRegex(ValueError, "invalid segmentation logits"):
self.score_export(args)
self.assertTrue(Path(args.out).exists())
def test_wrong_prediction_is_recorded_and_independent_g3_fails(self):
from harness.tools.export_onnx import export_recipe
args = self.real_model("wrong_prediction", direct=True)
self.assertEqual(export_recipe(args), 0)
result = self.score_export(args)
self.assertEqual(result["verdict"], "fail")
self.assertEqual(result["fp32"], 100.0)
self.assertLess(result["quant"], 100.0)
def test_missing_decoded_output_fails_scoring_not_graph_submission(self):
from harness.tools.export_onnx import export_recipe
args = self.real_model("missing_output", direct=True)
self.assertEqual(export_recipe(args), 0)
with self.assertRaisesRegex(ValueError, "output identities"):
self.score_export(args)
self.assertTrue(Path(args.out).exists())
def test_graph_only_export_needs_no_source_reference_or_weights(self):
from harness.tools.export_onnx import export_recipe
from harness import verify
self.real_model(direct=True)
self.manifest = {"schema": recipes.SCHEMA, "files": ["candidate.onnx"], "onnx": "candidate.onnx"}
self.save()
frozen = self.frozen("graph-only")
args = SimpleNamespace(recipe=str(Path(frozen["root"]) / "recipe.json"), out=str(self.root / "only-graph/model.onnx"))
self.assertEqual(export_recipe(args), 0)
runner = RecipeRunner(frozen, sys.executable, self.root / "missing-ref", "cityscapes", reference=True)
with self.assertRaisesRegex(ValueError, "reference"):
runner.prepare([np.zeros((2, 2, 3), np.uint8)])
self.assertNotIn("G3", verify.collect_measurements(self.rd)[0])
def test_real_worker_fetch_baseline_export_calibrate_share_original_stages_and_receipts(self):
"""Actual CLI, Worker, SQLite, flock pools, watchdog and model processes.
No Stage.run, runtime operation, snapshot or scheduler is mocked. Only
installation paths/config are redirected into this temporary fixture.
This stops before proprietary compilation and physical board work.
"""
from harness import db
from harness.eval import EVALUATOR_VERSION
from PIL import Image
self.real_model()
t1 = self.t1()
package = Path(t1.tar.name)
training = self.root / "training"
training.mkdir()
Image.fromarray(np.full((2, 2, 3), 2, np.uint8)).save(training / "calibration.png")
self.job.update(qcfg="int8", priority="A", mode="e2e", evaluator_version=EVALUATOR_VERSION,
model_runtime={"configured": sys.executable, "python": sys.executable},
data={"dataset": "cityscapes", "t1": {
"path": str(package), "manifest_sha": sha(package.read_bytes()), "content_sha": "fixed-t1"},
"calib": {"path": str(self.root / "shared-calibration"), "n": 1, "recipe_source": {
"dataset": "cityscapes", "local_root": str(self.root), "pool": str(self.root / "pool"),
"seed": 7, "dataset_config": {"sources": [], "calib": {"train_dir": "training"}}}}},
gates={"G2": {"n_samples": 2}})
self.save()
cfg = {"paths": {"work_root": str(self.root / "work")},
"pools": {"cpu": {"workers": 1}, "gpu": {"workers": 0, "devices": []}},
"watchdog": {"stall_s": 120, "cpu_idle_pct": 1, "poll_s": 0.1, "hard_ceiling_s": 300,
"io_stall_s": 180, "no_progress_ceiling_s": 240}}
# Worker dispatches a fresh Python process. In-process patches alone
# cannot redirect its DB/config and must not inherit the host's state.
state = self.root / "state"
state.mkdir()
database = state / "jobs.sqlite"
config_file = self.root / "worker-config.json"
config_file.write_text(json.dumps(cfg))
original_connect = db.connect
with ExitStack() as stack:
stack.enter_context(patch.dict(os.environ, {"BENCH_STATE_DIR": str(state),
"BENCH_CONFIG": str(config_file)}))
stack.enter_context(patch.object(db, "connect", side_effect=lambda **kw: original_connect(database, **kw)))
stack.enter_context(patch.object(paths, "STATE", state))
stack.enter_context(patch.object(paths, "config", return_value=cfg))
with db.connect() as con:
db.create_run(con, self.job["run_id"], {})
db.upsert_job(con, self.job["run_id"], self.job, 1)
db.rollout_upsert(con, self.job["run_id"], self.job["job_id"], 0, "e2e", "teacher")
worker = stageworker.Worker(self.rd)
submitted = self.client("fetch", "--recipe", "recipe.json")
def stalled_stage(signum, frame):
raise TimeoutError("fixture stage stalled; CPU recipe must not wait for a GPU slot")
previous_alarm = signal.signal(signal.SIGALRM, stalled_stage)
stack.callback(signal.signal, signal.SIGALRM, previous_alarm)
stack.callback(signal.alarm, 0)
for stage in ("fetch", "export", "calibrate", "baseline"):
handle = submitted if stage == "fetch" else self.client(stage, "--submission", submitted)
signal.alarm(180)
try:
worker.scan()
finally:
signal.alarm(0)
status = json.loads((self.rd / "handles" / handle / "status.json").read_text())
detail = Path(status["stderr"]).read_text() if status.get("stderr") else status.get("detail", "")
self.assertEqual(status.get("result"), "OK", f"{stage}: {detail}")
with db.connect() as con:
rows = con.execute("SELECT stage, state FROM stages ORDER BY stage").fetchall()
self.assertEqual([(row["stage"], row["state"]) for row in rows],
[("01-fetch", "OK"), ("02-baseline", "OK"), ("03-export", "OK"), ("04-calibrate", "OK")])
baseline = json.loads((self.rd / "stages/02-baseline/a00/meta.json").read_text())
exported = json.loads((self.rd / "stages/03-export/a00/meta.json").read_text())
self.assertEqual(baseline["pool"], "cpu")
self.assertEqual(exported["recipe"]["submission"], baseline["recipe"]["submission"])
self.assertNotIn("02-baseline", exported["recipe_parents"])
self.assertEqual(baseline["measurements"]["baseline"]["miou"], 100.0)
graph = Path(exported["recipe_outputs"]["onnx"]["path"])
self.assertTrue(graph.is_relative_to(self.rd / "stages/03-export/a00"))
accepted = json.loads((graph.parent / "acceptance.json").read_text())
self.assertNotIn("comparisons", accepted)
calibration = json.loads((self.rd / "stages/04-calibrate/a00/meta.json").read_text())
self.assertEqual(calibration["recipe"]["submission"], exported["recipe"]["submission"])
self.assertEqual(calibration["recipe_parents"]["03-export"]["attempt"], 0)
directory = Path(calibration["recipe_outputs"]["calibration"]["path"])
self.assertTrue(directory.is_relative_to(self.rd / "stages/04-calibrate/a00"))
manifest = json.loads((directory / "manifest.json").read_text())
self.assertEqual(manifest["source"], "train-dir")
self.assertEqual(manifest["count"], 1)
self.assertEqual(manifest["preprocessing_identity"]["submission_sha256"], exported["recipe"]["submission"])
np.testing.assert_array_equal(np.load(directory / "npy_f32/00000.npy", allow_pickle=False),
np.full((1, 3, 2, 2), 2, np.float32))
self.assertFalse((self.root / "shared-calibration").exists())
if __name__ == "__main__":
unittest.main()