672 lines
39 KiB
Python
672 lines
39 KiB
Python
"""Independent wiring checks; namespace/model checks require an explicit Linux run.
|
|
|
|
The portable tests exercise actual tickets, snapshots and numeric proxy messages.
|
|
They do not claim model, namespace or board acceptance. Enable
|
|
BENCH_TEST_RECIPE_NAMESPACE=1 in an existing Torch/ONNX runtime for those tests.
|
|
"""
|
|
from contextlib import ExitStack
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import textwrap
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
|
|
from harness import adapters, paths, recipes, registry, stageworker, stagerun
|
|
from harness.eval.common import RecipeRunner, T1
|
|
|
|
BENCH = Path(__file__).resolve().parents[1]
|
|
NAMESPACE = os.environ.get("BENCH_TEST_RECIPE_NAMESPACE") == "1"
|
|
|
|
|
|
def sha(value):
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
class Files(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory(prefix="recipe-pipeline-")
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.root = Path(self.temporary.name).resolve()
|
|
self.rd = self.root / "job/rollouts/00"
|
|
self.workspace = self.rd / "workspace"
|
|
self.workspace.mkdir(parents=True)
|
|
(self.rd / "handles").mkdir()
|
|
self.reference = b"raise RuntimeError('reference must never be imported by control process')\n"
|
|
self.candidate = b"raise RuntimeError('candidate must never be imported by control process')\n"
|
|
self.weights = b"pinned weights"
|
|
self.job = {"run_id": "test-run", "job_id": "recipe-fixture", "task": "seg", "model": "new.seg",
|
|
"backend": "ptq", "march": "nash-m", "model_spec": {
|
|
"source": "recipe", "input_spec": "1x3x2x2", "preprocess": {}},
|
|
"model_recipe": {"reference": {"entrypoint": "reference.py",
|
|
"files": {"reference.py": sha(self.reference)}, "weights_sha256": sha(self.weights)}}}
|
|
self.manifest = {"schema": recipes.SCHEMA, "files": ["candidate.py", "reference.py", "weights.bin"],
|
|
"entrypoint": "candidate.py", "weights": "weights.bin", "input_spec": "1x3x2x2",
|
|
"preprocess": {}}
|
|
self.save()
|
|
|
|
def save(self):
|
|
for name, data in {"candidate.py": self.candidate, "reference.py": self.reference,
|
|
"weights.bin": self.weights}.items():
|
|
(self.workspace / name).write_bytes(data)
|
|
(self.workspace / "recipe.json").write_text(json.dumps(self.manifest))
|
|
(self.rd.parent.parent / "job.json").write_text(json.dumps(self.job))
|
|
|
|
def frozen(self, name="snapshot"):
|
|
return recipes.snapshot(self.workspace, "recipe.json", self.root / name, self.job)
|
|
|
|
def client(self, *args):
|
|
result = subprocess.run([sys.executable, str(BENCH / "bin/stageclient.py"), "stage", "run", *args],
|
|
cwd=self.workspace, env={**os.environ, "BENCH_ROLLOUT_DIR": str(self.rd)},
|
|
capture_output=True, text=True, timeout=15)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
return result.stdout.splitlines()[0]
|
|
|
|
|
|
class JobPolicy(Files):
|
|
def build(self, *, recipe):
|
|
row = {"model_id": "new.seg", "task": "seg", "source": "custom", "input_spec": "1x3x2x2",
|
|
"dataset": "cityscapes", "preprocess": {"resize": 2}, "runtime_python": "/platform/bin/python"}
|
|
spec = registry.JobSpec("seg--new.seg--nash-m--ptq--int8", "seg", "new.seg", "nash-m", "ptq",
|
|
"int8", "P1", "e2e", overrides={"model_recipe": self.job["model_recipe"]} if recipe else {})
|
|
with ExitStack() as stack:
|
|
for name, value in (("models", [row]), ("board_type_for_march", "s100"), ("board_for_march", {}),
|
|
("gate_snapshot", {"required": ["G1", "G3"]}), ("registry_fingerprint", "registry"),
|
|
("datasets", {"datasets": {"cityscapes": {"version": "fixed", "t1": {"n": 2}}}})):
|
|
stack.enter_context(patch.object(registry, name, return_value=value))
|
|
stack.enter_context(patch.object(paths, "DATA", self.root / "data"))
|
|
stack.enter_context(patch.object(paths, "config", return_value={"paths": {}, "model_runtimes": {
|
|
"new.seg": "/configured/bin/python"}}))
|
|
stack.enter_context(patch("harness.toolchains.resolve", return_value={"name": "fixed-toolchain"}))
|
|
return registry.build_job_json(spec, "run", "test-evaluator")
|
|
|
|
def test_recipe_job_never_imports_same_name_public_adapter(self):
|
|
with patch.object(adapters, "get", side_effect=AssertionError("public model adapter imported")):
|
|
job = self.build(recipe=True)
|
|
self.assertEqual(job["model_spec"]["input_spec"], "1x3x2x2")
|
|
self.assertEqual(job["model_spec"]["preprocess"], {"resize": 2})
|
|
self.assertEqual(job["model_runtime"]["python"], "/configured/bin/python")
|
|
self.assertEqual(job["cache_contract"]["exporter_version"], "task-recipe-v1")
|
|
self.assertEqual(job["model_recipe"]["reference"]["preprocess"], {"resize": 2})
|
|
self.assertEqual(job["data"]["dataset"], "cityscapes")
|
|
|
|
def test_legacy_job_keeps_public_adapter_resolution(self):
|
|
adapter = SimpleNamespace(MODEL_SPEC={"input_spec": "1x3x8x8"}, PREPROCESS={"resize": 8},
|
|
RUNTIME_PYTHON="/old/bin/python", EXPORTER_VERSION="old-exporter")
|
|
with patch.object(adapters, "get", return_value=adapter) as get:
|
|
job = self.build(recipe=False)
|
|
self.assertGreater(get.call_count, 0)
|
|
self.assertNotIn("model_recipe", job)
|
|
self.assertEqual(job["model_spec"]["input_spec"], "1x3x8x8")
|
|
self.assertEqual(job["model_spec"]["preprocess"], {"resize": 8})
|
|
self.assertEqual(job["model_spec"]["runtime_python"], "/old/bin/python")
|
|
self.assertEqual(job["cache_contract"]["exporter_version"], "old-exporter")
|
|
|
|
def test_e2e_expansion_publishes_recipe_without_legacy_model_lookup(self):
|
|
from harness import db, pipeline
|
|
job = self.build(recipe=True)
|
|
spec = registry.JobSpec(job["job_id"], "seg", "new.seg", "nash-m", "ptq", "int8", "P1", "e2e",
|
|
policy="teacher", overrides={"model_recipe": self.job["model_recipe"]})
|
|
database = self.root / "jobs.sqlite"
|
|
original_connect = db.connect
|
|
with ExitStack() as stack:
|
|
stack.enter_context(patch.object(db, "STATE", self.root))
|
|
stack.enter_context(patch.object(db, "connect", side_effect=lambda **kw: original_connect(database, **kw)))
|
|
stack.enter_context(patch.object(paths, "RUNS", self.root / "runs"))
|
|
stack.enter_context(patch.object(paths, "DATA", self.root / "data"))
|
|
stack.enter_context(patch.object(pipeline, "model_row", side_effect=AssertionError("legacy model imported in expand")))
|
|
stack.enter_context(patch.object(registry, "models", return_value=[dict(job["model_spec"], source="custom")]))
|
|
stack.enter_context(patch.object(adapters, "get", side_effect=AssertionError("public adapter imported while publishing recipe job")))
|
|
stack.enter_context(patch.object(pipeline, "expand_matrix", return_value=[spec]))
|
|
stack.enter_context(patch.object(pipeline, "build_job_json", return_value=job))
|
|
with db.connect() as con:
|
|
db.create_run(con, "run", {})
|
|
result = pipeline.expand("run", modes=["e2e"], model_ids=["new.seg"])
|
|
saved = json.loads((paths.RUNS / "run" / job["job_id"] / "job.json").read_text())
|
|
with db.connect() as con:
|
|
count = con.execute("SELECT COUNT(*) FROM jobs WHERE run_id='run'").fetchone()[0]
|
|
self.assertEqual(result["jobs_new"], 1)
|
|
self.assertEqual(count, 1)
|
|
self.assertEqual(saved["model_recipe"], job["model_recipe"])
|
|
|
|
|
|
class ClientWorker(Files):
|
|
def execute(self, worker):
|
|
captured = []
|
|
def accept(command, **kwargs):
|
|
# Worker now crosses a process boundary. Observe the protected
|
|
# executor command, not an in-process _exec call that must not occur.
|
|
self.assertEqual(command[1:4], ['-m', 'harness.stagerun', '--exec-protected'])
|
|
queue = Path(command[4])
|
|
self.assertEqual(Path(command[5]), self.rd)
|
|
request = json.loads((queue / "request.json").read_text())
|
|
captured.append((request, recipes.load(Path(request["recipe"]))))
|
|
kwargs['on_start'](SimpleNamespace(pid=os.getpid()))
|
|
(queue / 'status.json').write_text('{"state":"DONE","result":"OK"}')
|
|
return stageworker.watchdog.RunResult('OK', 0, 0, kwargs['stdout'], kwargs['stderr'])
|
|
with patch.object(stageworker.watchdog, "run", side_effect=accept):
|
|
worker.scan()
|
|
return captured
|
|
|
|
def test_real_cli_ticket_freezes_and_submission_ignores_agent_status(self):
|
|
first = self.client("fetch", "--recipe", "workspace/recipe.json")
|
|
worker = stageworker.Worker(self.rd)
|
|
initial = self.execute(worker)
|
|
self.assertEqual(len(initial), 1, (self.rd / "handles" / first / "status.json").read_text())
|
|
request, frozen = initial[0]
|
|
self.assertTrue(Path(request["recipe"]).is_relative_to(self.rd / "stages/_queue"))
|
|
(self.workspace / "candidate.py").write_text("changed after submission")
|
|
(self.rd / "handles" / first / "status.json").write_text(json.dumps({
|
|
"state": "DONE", "result": "OK", "recipe": "/other/rollout/recipe.json"}))
|
|
(self.rd / "handles" / first / "request.json").write_text('{"recipe":"/forged"}')
|
|
self.client("export", "--submission", first)
|
|
reused = self.execute(worker)
|
|
self.assertEqual(len(reused), 1)
|
|
self.assertEqual(reused[0][1]["submission_sha256"], frozen["submission_sha256"])
|
|
self.assertEqual((Path(reused[0][1]["root"]) / "candidate.py").read_bytes(), self.candidate)
|
|
|
|
def test_worker_rejects_cross_rollout_and_symlinked_recipe(self):
|
|
(self.workspace / "linked.json").symlink_to(self.workspace / "recipe.json")
|
|
linked = self.client("fetch", "--recipe", "linked.json")
|
|
other = self.root / "other"
|
|
other.mkdir()
|
|
(other / "recipe.json").write_text((self.workspace / "recipe.json").read_text())
|
|
# A malicious Agent may bypass the helpful client validation entirely.
|
|
outside = "01-fetch-outside"
|
|
outside_handle = self.rd / "handles" / outside
|
|
outside_handle.mkdir()
|
|
(outside_handle / "request.json").write_text(json.dumps({
|
|
"stage": "01-fetch", "recipe": str(other / "recipe.json")}))
|
|
worker = stageworker.Worker(self.rd)
|
|
with patch.object(stageworker.watchdog, "run", side_effect=AssertionError("untrusted recipe executed")) as dispatch:
|
|
worker.scan()
|
|
dispatch.assert_not_called()
|
|
for handle in (linked, outside):
|
|
status = json.loads((self.rd / "handles" / handle / "status.json").read_text())
|
|
self.assertEqual(status["result"], "FAIL")
|
|
self.assertEqual(status["error_class"], "invalid_request")
|
|
queue = self.rd / "stages/_queue" / handle
|
|
self.assertIn("invalid_request", (queue / "runner.log").read_text())
|
|
self.assertIn("request:", (queue / "runner.log").read_text())
|
|
self.assertFalse((queue / "request.json").exists()) # Invalid input is evidence, not accepted authority.
|
|
self.assertEqual(json.loads((queue / 'operation.json').read_text())['state'], 'DONE')
|
|
with patch.object(stageworker.watchdog, "run") as dispatch:
|
|
stageworker.Worker(self.rd).scan()
|
|
dispatch.assert_not_called()
|
|
self.assertEqual(json.loads((self.rd/'handles'/linked/'status.json').read_text())['error_class'], 'invalid_request')
|
|
|
|
def test_forged_submission_handle_cannot_point_to_other_rollout(self):
|
|
handle = "03-export-forged"
|
|
hd = self.rd / "handles" / handle
|
|
hd.mkdir()
|
|
(hd / "request.json").write_text(json.dumps({"stage": "03-export", "submission": "../01/foreign"}))
|
|
with patch.object(stageworker.watchdog, "run", side_effect=AssertionError("cross rollout executed")) as dispatch:
|
|
stageworker.Worker(self.rd).scan()
|
|
dispatch.assert_not_called()
|
|
status = json.loads((hd / "status.json").read_text())
|
|
self.assertEqual(status["error_class"], "invalid_request")
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux managed process identities')
|
|
def test_prior_preparing_operation_still_quarantines_unknown_side_effects(self):
|
|
from harness.execution import Execution
|
|
from scripts.test_execution import policy
|
|
handle = self.client('fetch', '--recipe', 'recipe.json')
|
|
owner = Execution(self.rd); owner.start(policy())
|
|
request = json.loads((self.rd/'handles'/handle/'request.json').read_text())
|
|
digest = sha(json.dumps(request, sort_keys=True, separators=(',', ':')).encode())
|
|
queue = self.rd/'stages/_queue'/handle; queue.mkdir(parents=True)
|
|
(queue/'operation.json').write_text(json.dumps({'operation_id':handle, 'payload_sha256':digest,
|
|
'state':'PREPARING', 'execution_id':owner.snapshot()['execution_id'], 'epoch':owner.epoch}))
|
|
with patch.object(stageworker.watchdog, 'run') as dispatch:
|
|
stageworker.Worker(self.rd, execution=owner).scan()
|
|
dispatch.assert_not_called()
|
|
status = json.loads((self.rd/'handles'/handle/'status.json').read_text())
|
|
self.assertEqual(status['error_class'], 'SIDE_EFFECT_UNKNOWN')
|
|
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux managed process identities')
|
|
def test_invalid_new_ticket_has_durable_rejection_without_quarantining_execution(self):
|
|
from harness.execution import Execution
|
|
from scripts.test_execution import policy
|
|
owner = Execution(self.rd); owner.start(policy())
|
|
handle = '03-export-invalid'; hd = self.rd/'handles'/handle; hd.mkdir()
|
|
(hd/'request.json').write_text('{"stage":"03-export","submission":"../foreign"}')
|
|
with patch.object(stageworker.watchdog, 'run') as dispatch:
|
|
stageworker.Worker(self.rd, execution=owner).scan()
|
|
stageworker.Worker(self.rd, execution=owner).scan()
|
|
dispatch.assert_not_called()
|
|
self.assertEqual(json.loads((hd/'status.json').read_text())['error_class'], 'invalid_request')
|
|
self.assertEqual(owner.snapshot()['operations'][handle]['state'], 'DONE')
|
|
self.assertNotEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux kernel cleanup receipts')
|
|
def test_receipt_recovery_requires_identity_and_kernel_cleanup(self):
|
|
from harness import execution, watchdog
|
|
from scripts.test_execution import policy
|
|
for fault in (None, 'operation_id', 'payload_sha256', 'epoch', 'ledger_payload',
|
|
'scope_missing', 'scope_receipt_missing', 'board_receipt_missing'):
|
|
with self.subTest(fault=fault):
|
|
rd = self.root / ('receipt-' + str(fault))
|
|
hd = rd / 'handles/h1'; hd.mkdir(parents=True)
|
|
queue = rd / 'stages/_queue/h1'; queue.mkdir(parents=True)
|
|
name = '07-board' if fault == 'board_receipt_missing' else '01-fetch'
|
|
req = {'stage': name}
|
|
(hd / 'request.json').write_text(json.dumps(req))
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
previous = dict(operation_id='h1', payload_sha256=sha(json.dumps(req, sort_keys=True,
|
|
separators=(',', ':')).encode()), execution_id=owner.snapshot()['execution_id'],
|
|
epoch=owner.epoch, state='PREPARING')
|
|
def started(proc):
|
|
previous.update(state='DISPATCHED', executor=execution.process_identity(proc.pid))
|
|
execution.atomic_json(queue / 'operation.json', previous)
|
|
with owner.mutation() as state:
|
|
state['operations']['h1'] = dict(previous)
|
|
owner._save(state, 'TEST_DISPATCHED')
|
|
code = ('import json,os,sys; from pathlib import Path; from harness.execution import process_identity; '
|
|
'p=Path(sys.argv[1]); s=json.loads((p/"operation.json").read_text()); '
|
|
's.update(state="DONE",result="OK",process=process_identity(os.getpid())); '
|
|
'(p/"status.json").write_text(json.dumps(s))')
|
|
with owner.scope(name, 'h1'):
|
|
run = watchdog.run([sys.executable, '-c', code, str(queue)], cwd=BENCH,
|
|
stdout=queue/'out', stderr=queue/'err', execution=owner, role='stage',
|
|
on_start=started, poll_s=.01)
|
|
self.assertEqual(run.state, 'OK', (queue/'err').read_text())
|
|
receipt_path = queue/'status.json'
|
|
receipt = json.loads(receipt_path.read_text())
|
|
if fault in ('operation_id', 'payload_sha256', 'epoch'):
|
|
receipt[fault] = 'foreign'
|
|
receipt_path.write_text(json.dumps(receipt))
|
|
if fault in ('ledger_payload', 'scope_missing', 'scope_receipt_missing'):
|
|
with owner.mutation() as state:
|
|
if fault == 'ledger_payload':
|
|
state['operations']['h1']['payload_sha256'] = 'foreign'
|
|
elif fault == 'scope_missing':
|
|
state['process_scopes'] = {}
|
|
else:
|
|
Path(next(iter(state['process_scopes'].values()))['receipt_path']).unlink()
|
|
owner._save(state, 'TEST_FAULT')
|
|
before = receipt_path.read_bytes()
|
|
with patch.object(stageworker.watchdog, 'run') as dispatch:
|
|
stageworker.Worker(rd, execution=owner).scan()
|
|
dispatch.assert_not_called()
|
|
self.assertEqual(receipt_path.read_bytes(), before)
|
|
if fault:
|
|
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
|
|
self.assertNotEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
|
|
self.assertEqual(owner.snapshot()['stop_reason'], 'OPERATION_STATE_UNKNOWN')
|
|
if (hd/'status.json').exists():
|
|
self.assertNotEqual(json.loads((hd/'status.json').read_text()).get('state'), 'DONE')
|
|
else:
|
|
self.assertEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
|
|
self.assertEqual(owner.finish(0, protocol_complete=True)['execution_outcome'], 'COMPLETED')
|
|
|
|
def test_old_unprotected_exec_rejects_even_valid_recipe(self):
|
|
frozen = self.frozen()
|
|
handle = self.rd / "handles/unprotected"
|
|
handle.mkdir()
|
|
(handle / "request.json").write_text(json.dumps({"stage": "03-export", "recipe": str(Path(frozen["root"]) / "recipe.json")}))
|
|
with patch("harness.recipe_runtime.runtime_identity", side_effect=AssertionError("runtime reached")):
|
|
with self.assertRaisesRegex(ValueError, "protected outer Worker"):
|
|
stagerun._exec(handle)
|
|
|
|
|
|
class NumericProxy(Files):
|
|
def test_control_plane_runtime_identity_needs_no_model_packages(self):
|
|
code = textwrap.dedent('''
|
|
import json, pathlib, sys
|
|
from harness.recipe_runtime import runtime_identity
|
|
identity = runtime_identity(pathlib.Path(sys.executable))
|
|
print(json.dumps({"sha256": identity["sha256"],
|
|
"loaded": sorted(set(sys.modules) & {"numpy", "torch", "onnx", "PIL"})}))
|
|
''')
|
|
result = subprocess.run([sys.executable, "-S", "-c", code], cwd=BENCH,
|
|
env={"PATH": "/usr/bin:/bin"}, capture_output=True, text=True, timeout=60)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
evidence = json.loads(result.stdout)
|
|
self.assertEqual(evidence["loaded"], [])
|
|
self.assertEqual(len(evidence["sha256"]), 64)
|
|
|
|
def runner(self, **kwargs):
|
|
return RecipeRunner(self.frozen(), sys.executable, self.root / "calls", "cityscapes", **kwargs)
|
|
|
|
def test_proxy_uses_numeric_operations_without_importing_candidate(self):
|
|
runner = self.runner()
|
|
calls = []
|
|
def call(python, recipe, operation, values, work, **kwargs):
|
|
calls.append((operation, values, kwargs))
|
|
if operation == "preprocess":
|
|
return {key: value.transpose(2, 0, 1)[None].astype(np.float32) for key, value in values.items()}
|
|
if operation == "predict":
|
|
return {key + "/out0": np.ones((1, 19, 2, 2), np.float32) for key in values}
|
|
return values
|
|
with patch("harness.recipe_runtime.call", side_effect=call), \
|
|
patch.object(adapters, "get", side_effect=AssertionError("candidate/public import")):
|
|
outputs = runner.batch(runner.prepare([np.zeros((2, 2, 3), np.uint8)]))
|
|
self.assertEqual([x[0] for x in calls], ["preprocess", "predict", "decode"])
|
|
self.assertEqual(len(outputs), 1)
|
|
self.assertEqual(outputs[0][0].shape, (1, 19, 2, 2))
|
|
self.assertTrue(all(isinstance(value, np.ndarray) for _, values, _ in calls for value in values.values()))
|
|
|
|
def test_missing_extra_noncontiguous_sample_outputs_fail(self):
|
|
runner = self.runner()
|
|
array = np.zeros((1, 19, 2, 2), np.float32)
|
|
for values in ({}, {"s0000/out1": array}, {"s0000/out0": array, "s0001/out0": array}):
|
|
with self.subTest(keys=list(values)), patch("harness.recipe_runtime.call", return_value=values):
|
|
with self.assertRaisesRegex(ValueError, "identities"):
|
|
runner.batch([np.zeros((1, 3, 2, 2), np.float32)])
|
|
|
|
def test_preprocess_cannot_change_sample_count_shape_or_dtype(self):
|
|
runner = self.runner()
|
|
for values in ({}, {"s0000": np.zeros((1, 3, 3, 3), np.float32)},
|
|
{"s0000": np.zeros((1, 3, 2, 2), np.float64)}):
|
|
with self.subTest(keys=list(values)), patch("harness.recipe_runtime.call", return_value=values):
|
|
with self.assertRaisesRegex(ValueError, "preprocessing"):
|
|
runner.prepare([np.zeros((2, 2, 3), np.uint8)])
|
|
|
|
def test_g2_uses_current_deploy_preprocessing_not_old_reference_tensors(self):
|
|
from harness.eval.run import numeric_diagnostic
|
|
self.manifest["input_spec"] = "1x3x4x4"
|
|
self.save()
|
|
runner = self.runner()
|
|
baseline = self.root / "baseline"
|
|
baseline.mkdir()
|
|
np.savez(baseline / "g2_inputs.npz", s000=np.zeros((1, 3, 2, 2), np.float32))
|
|
np.savez(baseline / "g2_images.npz", s000=np.ones((2, 2, 3), np.uint8))
|
|
np.savez(baseline / "g2_fp32.npz", **{"s000/out0": np.ones((1, 19, 2, 2), np.float32)})
|
|
seen = []
|
|
def call(python, recipe, operation, values, work, **kwargs):
|
|
seen.append(operation)
|
|
if operation == "preprocess":
|
|
return {key: np.ones((1, 3, 4, 4), np.float32) for key in values}
|
|
if operation == "predict":
|
|
for value in values.values():
|
|
self.assertEqual(value.shape, (1, 3, 4, 4), "old reference input leaked into deploy runtime")
|
|
return {key + "/out0": np.ones((1, 19, 2, 2), np.float32) for key in values}
|
|
return values
|
|
with patch("harness.recipe_runtime.call", side_effect=call):
|
|
result = numeric_diagnostic(runner, baseline, "t0")
|
|
self.assertEqual(seen, ["preprocess", "predict", "decode"])
|
|
self.assertAlmostEqual(result["cos_min"], 1.0)
|
|
|
|
|
|
@unittest.skipUnless(NAMESPACE, "not run: requires BENCH_TEST_RECIPE_NAMESPACE=1 and Linux model runtime")
|
|
class SegmentationAndGraph(Files):
|
|
def setUp(self):
|
|
super().setUp()
|
|
import importlib.util
|
|
self.assertEqual(sys.platform, "linux", "namespace acceptance must run on Linux")
|
|
for name in ("torch", "onnx", "onnxruntime", "PIL"):
|
|
self.assertIsNotNone(importlib.util.find_spec(name), f"missing required model package: {name}")
|
|
|
|
def t1(self):
|
|
from PIL import Image
|
|
archive = self.root / "t1.tar"
|
|
labels = [np.array([[7, 8], [7, 8]], np.uint8), np.array([[8, 7], [8, 7]], np.uint8)]
|
|
items = []
|
|
with tarfile.open(archive, "w") as stream:
|
|
def add(name, data):
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(data)
|
|
stream.addfile(info, io.BytesIO(data))
|
|
for i, label in enumerate(labels):
|
|
item = {"id": f"image-{i}", "file": f"image-{i}.png", "label_file": f"label-{i}.png",
|
|
"label_format": "cityscapes_labelIds"}
|
|
items.append(item)
|
|
image = np.repeat((label == 8)[..., None], 3, axis=-1).astype(np.uint8)
|
|
for name, value in ((item["file"], image), (item["label_file"], label)):
|
|
data = io.BytesIO()
|
|
Image.fromarray(value).save(data, format="PNG")
|
|
add(name, data.getvalue())
|
|
add("manifest.json", json.dumps({"dataset": "cityscapes", "sha256": "fixed-t1", "items": items}).encode())
|
|
result = T1.open(archive)
|
|
self.addCleanup(result.tar.close)
|
|
return result
|
|
|
|
def test_fixed_t1_platform_scores_numeric_proxy_predictions(self):
|
|
from harness.eval import seg
|
|
t1 = self.t1()
|
|
runner = RecipeRunner(self.frozen(), sys.executable, self.root / "calls", "cityscapes")
|
|
def call(python, recipe, operation, values, work, **kwargs):
|
|
if operation == "preprocess":
|
|
return {key: value.transpose(2, 0, 1)[None].astype(np.float32) for key, value in values.items()}
|
|
if operation == "predict":
|
|
result = {}
|
|
for key, value in values.items():
|
|
logits = np.zeros((1, 19, 2, 2), np.float32)
|
|
logits[0, 0] = 1 - value[0, 0]
|
|
logits[0, 1] = value[0, 0]
|
|
result[key + "/out0"] = logits
|
|
return result
|
|
return values
|
|
baseline = {"model": "new.seg", "t1_manifest_sha": "fixed-t1"}
|
|
with patch("harness.recipe_runtime.call", side_effect=call), \
|
|
patch.object(seg, "get", side_effect=AssertionError("public adapter imported")):
|
|
seg.run_fp32(runner, t1, "1x3x2x2", baseline)
|
|
quant = {"model": "new.seg", "t1_manifest_sha": "fixed-t1"}
|
|
seg.run_quant(runner, t1, "1x3x2x2", baseline, quant)
|
|
self.assertEqual(quant["G3"]["quant"], 100.0)
|
|
self.assertEqual([item["id"] for item in quant["_seg_samples"]], ["image-0", "image-1"])
|
|
self.assertNotIn("source", quant["G3"])
|
|
|
|
def real_model(self, variant="correct", direct=False):
|
|
# No weights are deserialized in this fixture; real source files still
|
|
# cross the normal digest/snapshot/process boundaries.
|
|
module = textwrap.dedent('''
|
|
import numpy as np
|
|
import torch
|
|
class Model(torch.nn.Module):
|
|
def forward(self, x):
|
|
zero = x[:, :1] * 0
|
|
return torch.cat((1-x[:, :1], x[:, :1], *([zero]*17)), dim=1)
|
|
def build(weights, input_spec):
|
|
return Model()
|
|
def preprocess(image, input_spec):
|
|
return np.asarray(image, dtype=np.float32).transpose(2,0,1)[None]
|
|
''').encode()
|
|
self.reference = module
|
|
self.candidate = module
|
|
self.job["model_recipe"]["reference"]["files"] = {"reference.py": sha(module)}
|
|
if variant == "missing_output":
|
|
self.candidate += b"\ndef decode(outputs, input_spec): return []\n"
|
|
if direct:
|
|
import onnx
|
|
from onnx import helper, TensorProto, numpy_helper
|
|
channels = 3 if variant == "removed_head" else 19
|
|
output = np.zeros((1, channels, 2, 2), np.float32)
|
|
output[:, 0] = 1
|
|
if variant == "wrong_prediction":
|
|
output[:, 0] = 0
|
|
output[:, 1] = 1
|
|
graph = helper.make_graph([helper.make_node("Constant", [], ["logits"], value=numpy_helper.from_array(output))],
|
|
"fixture", [helper.make_tensor_value_info("input", TensorProto.FLOAT, [1, 3, 2, 2])],
|
|
[helper.make_tensor_value_info("logits", TensorProto.FLOAT, list(output.shape))])
|
|
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 11)])
|
|
model.ir_version = 7
|
|
onnx.save(model, self.workspace / "candidate.onnx")
|
|
self.manifest["files"].append("candidate.onnx")
|
|
self.manifest["onnx"] = "candidate.onnx"
|
|
self.save()
|
|
frozen = self.frozen()
|
|
baseline = self.root / "baseline"
|
|
baseline.mkdir()
|
|
image = np.zeros((2, 2, 3), np.uint8)
|
|
logits = np.zeros((1, 19, 2, 2), np.float32)
|
|
logits[:, 0] = 1
|
|
np.savez(baseline / "g2_images.npz", s000=image)
|
|
np.savez(baseline / "g2_fp32.npz", **{"s000/out0": logits})
|
|
(baseline / "metrics.json").write_text(json.dumps({"seg_protocol": {"dataset": "cityscapes", "classes": 19}}))
|
|
return SimpleNamespace(recipe=str(Path(frozen["root"]) / "recipe.json"), out=str(self.root / "stage/graph/model.onnx"),
|
|
baseline=str(baseline), dataset="cityscapes")
|
|
|
|
def test_export_script_checks_graph_without_quality_precondition(self):
|
|
from harness.tools.export_onnx import export_recipe
|
|
args = self.real_model()
|
|
self.assertEqual(export_recipe(args), 0)
|
|
acceptance = json.loads((Path(args.out).parent / "acceptance.json").read_text())
|
|
self.assertIn("onnx", acceptance)
|
|
self.assertNotIn("comparisons", acceptance)
|
|
|
|
def test_direct_graph_runs_without_reexport(self):
|
|
from harness.tools.export_onnx import export_recipe
|
|
args = self.real_model(direct=True)
|
|
with patch("harness.recipe_runtime.export", side_effect=AssertionError("direct graph was needlessly re-exported")):
|
|
self.assertEqual(export_recipe(args), 0)
|
|
self.assertTrue((Path(args.out).parent / "acceptance.json").is_file())
|
|
|
|
def score_export(self, args):
|
|
from harness.eval import seg
|
|
from harness.gates import g3
|
|
frozen = recipes.load(Path(args.recipe))
|
|
t1 = self.t1()
|
|
baseline = {"model": "new.seg", "t1_manifest_sha": "fixed-t1"}
|
|
reference = RecipeRunner(frozen, sys.executable, self.root / "official-calls", "cityscapes", reference=True)
|
|
seg.run_fp32(reference, t1, "1x3x2x2", baseline)
|
|
quant = {"model": "new.seg", "t1_manifest_sha": "fixed-t1"}
|
|
candidate = RecipeRunner(frozen, sys.executable, self.root / "candidate-calls", "cityscapes", graph=Path(args.out))
|
|
seg.run_quant(candidate, t1, "1x3x2x2", baseline, quant)
|
|
return g3(quant["G3"], {"metric": "miou", "direction": "higher", "kind": "abs_pp", "tol": -1})
|
|
|
|
def test_removed_prediction_head_can_be_exported_but_incomplete_predictions_fail_scoring(self):
|
|
from harness.tools.export_onnx import export_recipe
|
|
args = self.real_model("removed_head", direct=True)
|
|
self.assertEqual(export_recipe(args), 0)
|
|
with self.assertRaisesRegex(ValueError, "invalid segmentation logits"):
|
|
self.score_export(args)
|
|
self.assertTrue(Path(args.out).exists())
|
|
|
|
def test_wrong_prediction_is_recorded_and_independent_g3_fails(self):
|
|
from harness.tools.export_onnx import export_recipe
|
|
args = self.real_model("wrong_prediction", direct=True)
|
|
self.assertEqual(export_recipe(args), 0)
|
|
result = self.score_export(args)
|
|
self.assertEqual(result["verdict"], "fail")
|
|
self.assertEqual(result["fp32"], 100.0)
|
|
self.assertLess(result["quant"], 100.0)
|
|
|
|
def test_missing_decoded_output_fails_scoring_not_graph_submission(self):
|
|
from harness.tools.export_onnx import export_recipe
|
|
args = self.real_model("missing_output", direct=True)
|
|
self.assertEqual(export_recipe(args), 0)
|
|
with self.assertRaisesRegex(ValueError, "output identities"):
|
|
self.score_export(args)
|
|
self.assertTrue(Path(args.out).exists())
|
|
|
|
def test_graph_only_export_needs_no_source_reference_or_weights(self):
|
|
from harness.tools.export_onnx import export_recipe
|
|
from harness import verify
|
|
self.real_model(direct=True)
|
|
self.manifest = {"schema": recipes.SCHEMA, "files": ["candidate.onnx"], "onnx": "candidate.onnx"}
|
|
self.save()
|
|
frozen = self.frozen("graph-only")
|
|
args = SimpleNamespace(recipe=str(Path(frozen["root"]) / "recipe.json"), out=str(self.root / "only-graph/model.onnx"))
|
|
self.assertEqual(export_recipe(args), 0)
|
|
runner = RecipeRunner(frozen, sys.executable, self.root / "missing-ref", "cityscapes", reference=True)
|
|
with self.assertRaisesRegex(ValueError, "reference"):
|
|
runner.prepare([np.zeros((2, 2, 3), np.uint8)])
|
|
self.assertNotIn("G3", verify.collect_measurements(self.rd)[0])
|
|
|
|
def test_real_worker_fetch_baseline_export_calibrate_share_original_stages_and_receipts(self):
|
|
"""Actual CLI, Worker, SQLite, flock pools, watchdog and model processes.
|
|
|
|
No Stage.run, runtime operation, snapshot or scheduler is mocked. Only
|
|
installation paths/config are redirected into this temporary fixture.
|
|
This stops before proprietary compilation and physical board work.
|
|
"""
|
|
from harness import db
|
|
from harness.eval import EVALUATOR_VERSION
|
|
from PIL import Image
|
|
self.real_model()
|
|
t1 = self.t1()
|
|
package = Path(t1.tar.name)
|
|
training = self.root / "training"
|
|
training.mkdir()
|
|
Image.fromarray(np.full((2, 2, 3), 2, np.uint8)).save(training / "calibration.png")
|
|
self.job.update(qcfg="int8", priority="A", mode="e2e", evaluator_version=EVALUATOR_VERSION,
|
|
model_runtime={"configured": sys.executable, "python": sys.executable},
|
|
data={"dataset": "cityscapes", "t1": {
|
|
"path": str(package), "manifest_sha": sha(package.read_bytes()), "content_sha": "fixed-t1"},
|
|
"calib": {"path": str(self.root / "shared-calibration"), "n": 1, "recipe_source": {
|
|
"dataset": "cityscapes", "local_root": str(self.root), "pool": str(self.root / "pool"),
|
|
"seed": 7, "dataset_config": {"sources": [], "calib": {"train_dir": "training"}}}}},
|
|
gates={"G2": {"n_samples": 2}})
|
|
self.save()
|
|
cfg = {"paths": {"work_root": str(self.root / "work")},
|
|
"pools": {"cpu": {"workers": 1}, "gpu": {"workers": 0, "devices": []}},
|
|
"watchdog": {"stall_s": 120, "cpu_idle_pct": 1, "poll_s": 0.1, "hard_ceiling_s": 300,
|
|
"io_stall_s": 180, "no_progress_ceiling_s": 240}}
|
|
# Worker dispatches a fresh Python process. In-process patches alone
|
|
# cannot redirect its DB/config and must not inherit the host's state.
|
|
state = self.root / "state"
|
|
state.mkdir()
|
|
database = state / "jobs.sqlite"
|
|
config_file = self.root / "worker-config.json"
|
|
config_file.write_text(json.dumps(cfg))
|
|
original_connect = db.connect
|
|
with ExitStack() as stack:
|
|
stack.enter_context(patch.dict(os.environ, {"BENCH_STATE_DIR": str(state),
|
|
"BENCH_CONFIG": str(config_file)}))
|
|
stack.enter_context(patch.object(db, "connect", side_effect=lambda **kw: original_connect(database, **kw)))
|
|
stack.enter_context(patch.object(paths, "STATE", state))
|
|
stack.enter_context(patch.object(paths, "config", return_value=cfg))
|
|
with db.connect() as con:
|
|
db.create_run(con, self.job["run_id"], {})
|
|
db.upsert_job(con, self.job["run_id"], self.job, 1)
|
|
db.rollout_upsert(con, self.job["run_id"], self.job["job_id"], 0, "e2e", "teacher")
|
|
worker = stageworker.Worker(self.rd)
|
|
submitted = self.client("fetch", "--recipe", "recipe.json")
|
|
def stalled_stage(signum, frame):
|
|
raise TimeoutError("fixture stage stalled; CPU recipe must not wait for a GPU slot")
|
|
previous_alarm = signal.signal(signal.SIGALRM, stalled_stage)
|
|
stack.callback(signal.signal, signal.SIGALRM, previous_alarm)
|
|
stack.callback(signal.alarm, 0)
|
|
for stage in ("fetch", "export", "calibrate", "baseline"):
|
|
handle = submitted if stage == "fetch" else self.client(stage, "--submission", submitted)
|
|
signal.alarm(180)
|
|
try:
|
|
worker.scan()
|
|
finally:
|
|
signal.alarm(0)
|
|
status = json.loads((self.rd / "handles" / handle / "status.json").read_text())
|
|
detail = Path(status["stderr"]).read_text() if status.get("stderr") else status.get("detail", "")
|
|
self.assertEqual(status.get("result"), "OK", f"{stage}: {detail}")
|
|
with db.connect() as con:
|
|
rows = con.execute("SELECT stage, state FROM stages ORDER BY stage").fetchall()
|
|
self.assertEqual([(row["stage"], row["state"]) for row in rows],
|
|
[("01-fetch", "OK"), ("02-baseline", "OK"), ("03-export", "OK"), ("04-calibrate", "OK")])
|
|
baseline = json.loads((self.rd / "stages/02-baseline/a00/meta.json").read_text())
|
|
exported = json.loads((self.rd / "stages/03-export/a00/meta.json").read_text())
|
|
self.assertEqual(baseline["pool"], "cpu")
|
|
self.assertEqual(exported["recipe"]["submission"], baseline["recipe"]["submission"])
|
|
self.assertNotIn("02-baseline", exported["recipe_parents"])
|
|
self.assertEqual(baseline["measurements"]["baseline"]["miou"], 100.0)
|
|
graph = Path(exported["recipe_outputs"]["onnx"]["path"])
|
|
self.assertTrue(graph.is_relative_to(self.rd / "stages/03-export/a00"))
|
|
accepted = json.loads((graph.parent / "acceptance.json").read_text())
|
|
self.assertNotIn("comparisons", accepted)
|
|
calibration = json.loads((self.rd / "stages/04-calibrate/a00/meta.json").read_text())
|
|
self.assertEqual(calibration["recipe"]["submission"], exported["recipe"]["submission"])
|
|
self.assertEqual(calibration["recipe_parents"]["03-export"]["attempt"], 0)
|
|
directory = Path(calibration["recipe_outputs"]["calibration"]["path"])
|
|
self.assertTrue(directory.is_relative_to(self.rd / "stages/04-calibrate/a00"))
|
|
manifest = json.loads((directory / "manifest.json").read_text())
|
|
self.assertEqual(manifest["source"], "train-dir")
|
|
self.assertEqual(manifest["count"], 1)
|
|
self.assertEqual(manifest["preprocessing_identity"]["submission_sha256"], exported["recipe"]["submission"])
|
|
np.testing.assert_array_equal(np.load(directory / "npy_f32/00000.npy", allow_pickle=False),
|
|
np.full((1, 3, 2, 2), 2, np.float32))
|
|
self.assertFalse((self.root / "shared-calibration").exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|