droboticflow/scripts/poc_operator_full_acceptance_job.py

1673 lines
68 KiB
Python
Executable File

#!/usr/bin/env python3
from __future__ import annotations
import datetime as dt
import hashlib
import importlib.util
import json
import os
import pathlib
import shutil
import sqlite3
import subprocess
import sys
import time
from typing import Any, Callable
from urllib import request
RESULT_PREFIX = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="
DEFAULT_REQUIRED_EXTERNALS = {"mongo", "bos"}
FIXTURE_MANIFEST_ENV_NAME = "POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH"
FIXTURE_ENV_NAMES = (
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
)
MANIFEST_CONFIG_PATHS = {
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH": (
("fixtures", "xplatform"),
("fixtures", "xplatformSourcePath"),
("xplatform", "sourcePath"),
),
"POC_ACCEPTANCE_CSV_INPUT_PATH": (("fixtures", "csv"), ("fixtures", "csvInputPath")),
"POC_ACCEPTANCE_JSON_INPUT_PATH": (("fixtures", "json"), ("fixtures", "jsonInputPath")),
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH": (("fixtures", "lerobot"), ("fixtures", "lerobotSourcePath")),
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH": (("fixtures", "rosbag"), ("fixtures", "rosbagSourcePath")),
"POC_ACCEPTANCE_MODEL_JSON_PATH": (("fixtures", "model"), ("fixtures", "modelJsonPath")),
"POC_ACCEPTANCE_SHELL_COMMAND": (("shellCommand",), ("shell", "command")),
"POC_ACCEPTANCE_JSON_FILTER_FIELD": (("json", "filterField"),),
"POC_ACCEPTANCE_JSON_FILTER_EXPECTED": (("json", "filterExpected"),),
"POC_ACCEPTANCE_JSON_EXTRACT_FIELD": (("json", "extractField"),),
"POC_ACCEPTANCE_JSON_FIELD_MAP_JSON": (("json", "fieldMap"),),
"POC_ACCEPTANCE_TEMPLATE_TEXT": (("json", "templateText"), ("templateText",)),
"POC_ACCEPTANCE_MIN_CLARITY": (("video", "minClarity"), ("xplatform", "minClarity")),
"POC_ACCEPTANCE_MAX_SHAKE": (("video", "maxShake"), ("xplatform", "maxShake")),
"POC_ACCEPTANCE_MAX_HAND_OCCLUSION": (("video", "maxHandOcclusion"), ("xplatform", "maxHandOcclusion")),
"POC_ACCEPTANCE_CLARITY_SCALE": (("video", "clarityScale"),),
"POC_ACCEPTANCE_XPLATFORM_DATE_START": (("xplatform", "dateStart"),),
"POC_ACCEPTANCE_XPLATFORM_DATE_END": (("xplatform", "dateEnd"),),
"POC_ACCEPTANCE_XPLATFORM_PLACE": (("xplatform", "place"),),
"POC_ACCEPTANCE_XPLATFORM_TYPE": (("xplatform", "type"),),
"POC_ACCEPTANCE_SEGMENT_DURATION_SECONDS": (("video", "segmentDurationSeconds"),),
"POC_ACCEPTANCE_FRAME_INTERVAL_SECONDS": (("video", "frameIntervalSeconds"),),
"POC_ACCEPTANCE_AUDIT_RULES_JSON": (("audit", "rules"),),
}
_FIXTURE_MANIFEST: dict[str, Any] | None = None
def load_runner_module():
runner_path = pathlib.Path(os.getenv("OPERATOR_RUNNER_MAIN", "/app/backend/operator-runner/main.py"))
if not runner_path.exists():
runner_path = pathlib.Path(__file__).resolve().parents[1] / "backend" / "operator-runner" / "main.py"
if not runner_path.exists():
raise RuntimeError(f"operator-runner main.py not found: {runner_path}")
project_root = runner_path.parents[2]
for candidate in (project_root, runner_path.parent):
if str(candidate) not in sys.path:
sys.path.insert(0, str(candidate))
spec = importlib.util.spec_from_file_location("operator_runner_main", runner_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"failed to import operator-runner from {runner_path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
MODULE = load_runner_module()
class AcceptanceFailure(RuntimeError):
pass
def builtin_operator_root() -> pathlib.Path:
candidates = [pathlib.Path("/app/backend/operators/builtin")]
runner_file = pathlib.Path(str(getattr(MODULE, "__file__", "")))
if str(runner_file):
candidates.append(runner_file.resolve().parents[1] / "operators" / "builtin")
candidates.append(pathlib.Path(__file__).resolve().parents[1] / "backend" / "operators" / "builtin")
for candidate in candidates:
if candidate.exists():
return candidate
return candidates[0]
def load_builtin_operator_metadata(slug: str) -> dict[str, Any]:
metadata_path = builtin_operator_root() / slug / "operator.json"
if not metadata_path.exists():
raise AcceptanceFailure(f"builtin operator metadata is missing: {metadata_path}")
try:
payload = json.loads(metadata_path.read_text(encoding="utf-8"))
except json.JSONDecodeError as error:
raise AcceptanceFailure(f"builtin operator metadata is not valid JSON: {metadata_path}") from error
if not isinstance(payload, dict):
raise AcceptanceFailure(f"builtin operator metadata must be a JSON object: {metadata_path}")
return payload
def now_id() -> str:
return dt.datetime.now(dt.timezone.utc).strftime("%Y%m%d%H%M%S")
def now_iso() -> str:
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
def source_commit() -> str:
for env_name in ("POC_ACCEPTANCE_SOURCE_COMMIT", "DROBOTICFLOW_BUILD_COMMIT"):
value = os.getenv(env_name, "").strip()
if value:
return value
project_root = pathlib.Path(__file__).resolve().parents[1]
try:
return subprocess.run(
["git", "-C", str(project_root), "rev-parse", "HEAD"],
check=True,
capture_output=True,
text=True,
).stdout.strip() or "unknown"
except (OSError, subprocess.CalledProcessError):
return "unknown"
def builtin_operator_catalog_metadata() -> tuple[str, int]:
injected_hash = os.getenv("POC_ACCEPTANCE_CATALOG_HASH", "").strip().lower()
injected_count = os.getenv("POC_ACCEPTANCE_CATALOG_COUNT", "").strip()
if injected_hash and injected_count:
if len(injected_hash) != 64 or any(char not in "0123456789abcdef" for char in injected_hash):
raise AcceptanceFailure("POC_ACCEPTANCE_CATALOG_HASH must be a lowercase SHA-256 digest")
try:
catalog_count = int(injected_count)
except ValueError as error:
raise AcceptanceFailure("POC_ACCEPTANCE_CATALOG_COUNT must be an integer") from error
if catalog_count <= 0:
raise AcceptanceFailure("POC_ACCEPTANCE_CATALOG_COUNT must be positive")
return injected_hash, catalog_count
candidates = [
pathlib.Path(os.getenv("POC_ACCEPTANCE_CATALOG_PATH", "")),
pathlib.Path("/app/internal/api/builtin_operators_catalog.json"),
pathlib.Path(__file__).resolve().parents[1] / "internal" / "api" / "builtin_operators_catalog.json",
]
catalog_path = next((path for path in candidates if str(path) and path.is_file()), None)
if catalog_path is None:
raise AcceptanceFailure("builtin operator catalog is not available for acceptance provenance")
try:
catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise AcceptanceFailure(f"builtin operator catalog is not readable: {catalog_path}") from error
if not isinstance(catalog, list) or not catalog:
raise AcceptanceFailure("builtin operator catalog must be a non-empty JSON list")
canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
# Go's encoding/json keeps UTF-8 but escapes HTML-sensitive and line-separator runes.
for raw, escaped in (
("&", "\\u0026"),
("<", "\\u003c"),
(">", "\\u003e"),
("\u2028", "\\u2028"),
("\u2029", "\\u2029"),
):
canonical_text = canonical_text.replace(raw, escaped)
canonical = canonical_text.encode("utf-8")
return hashlib.sha256(canonical).hexdigest(), len(catalog)
def acceptance_provenance(completed_at: str | None = None, *, tolerate_errors: bool = False) -> dict[str, Any]:
try:
catalog_hash, catalog_count = builtin_operator_catalog_metadata()
except Exception as error:
if not tolerate_errors:
raise
catalog_hash = os.getenv("POC_ACCEPTANCE_CATALOG_HASH", "").strip().lower()
try:
catalog_count = int(os.getenv("POC_ACCEPTANCE_CATALOG_COUNT", "0"))
except ValueError:
catalog_count = 0
provenance_error = f"{type(error).__name__}: {error}"
else:
provenance_error = ""
payload = {
"sourceCommit": source_commit(),
"catalogHash": catalog_hash,
"catalogCount": catalog_count,
"completedAt": completed_at or now_iso(),
}
if provenance_error:
payload["provenanceError"] = provenance_error
return payload
def json_value(value: Any) -> str:
if isinstance(value, str):
return value
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
def hyperparams(values: dict[str, Any]) -> list[dict[str, str]]:
return [{"key": key, "value": json_value(value)} for key, value in values.items()]
def node(label: str, params: dict[str, Any] | None = None, outputs: list[dict[str, Any]] | None = None) -> dict[str, Any]:
payload: dict[str, Any] = {"label": label, "hyperparams": hyperparams(params or {})}
if outputs is not None:
payload["outputs"] = outputs
return payload
def ensure(condition: bool, message: str) -> None:
if not condition:
raise AcceptanceFailure(message)
def write_json(path: pathlib.Path, payload: Any) -> pathlib.Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def json_artifact(run_id: str, node_id: str, output_dir: pathlib.Path, name: str, payload: dict[str, Any]) -> dict[str, Any]:
path = write_json(output_dir / "_inputs" / name, payload)
return MODULE.build_artifact(run_id, node_id, path, "json", data=payload)
def load_payload(artifact: dict[str, Any]) -> dict[str, Any]:
payload = MODULE.load_artifact_payload(artifact)
if not isinstance(payload, dict):
raise AcceptanceFailure(f"artifact {artifact.get('name')} did not contain a JSON object payload")
return payload
def first_artifact(artifacts: list[dict[str, Any]], name: str) -> dict[str, Any]:
for artifact in artifacts:
if artifact.get("name") == name:
return artifact
raise AcceptanceFailure(f"missing artifact {name}; got {[item.get('name') for item in artifacts]}")
def model_inference_evidence(artifacts: list[dict[str, Any]]) -> dict[str, Any]:
payload = load_payload(first_artifact(artifacts, "inference-result.json"))
model_source = str(payload.get("modelSource", "")).strip()
model_sha256 = str(payload.get("modelSha256", "")).strip()
ensure(model_source == "MODEL_JSON" or model_source.startswith("MODEL_PATH:"), "model inference must record model source")
ensure(
len(model_sha256) == 64 and all(char in "0123456789abcdef" for char in model_sha256),
"model inference must record a sha256 model digest",
)
ensure(str(payload.get("modelType", "")).strip() in {"rule", "linear", "nearest_centroid"}, "model type must be real")
return {
"inputCount": payload.get("inputCount"),
"modelType": payload.get("modelType"),
"modelSource": model_source,
"modelSha256": model_sha256,
}
def video_target_clip_evidence(artifacts: list[dict[str, Any]]) -> dict[str, Any]:
payload = load_payload(first_artifact(artifacts, "target-clips-manifest.json"))
detection_count = int(payload.get("detectionCount") or 0)
clip_count = int(payload.get("clipCount") or 0)
ensure(detection_count > 0, "video_target_clip must detect at least one target in POC video fixture")
ensure(clip_count > 0, "video_target_clip must output at least one real clip in POC video fixture")
return {
"detectionCount": detection_count,
"clipCount": clip_count,
}
def run_operator(
run_id: str,
slug: str,
handler: str,
base_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
validate: Callable[[list[dict[str, Any]]], dict[str, Any]],
records: list[dict[str, Any]],
) -> list[dict[str, Any]]:
output_dir = base_dir / "outputs" / slug
output_dir.mkdir(parents=True, exist_ok=True)
started = time.time()
try:
artifacts = MODULE.BUILTIN_HANDLERS[handler](run_id, slug, output_dir, node_data, inputs)
evidence = validate(artifacts)
except Exception as error:
records.append(
{
"operator": slug,
"handler": handler,
"status": "failed",
"error": f"{type(error).__name__}: {error}",
"durationSeconds": round(time.time() - started, 3),
}
)
raise
records.append(
{
"operator": slug,
"handler": handler,
"status": "passed",
"artifactCount": len(artifacts),
"artifacts": [str(item.get("name", "")) for item in artifacts],
"evidence": MODULE.json_compatible(evidence),
"durationSeconds": round(time.time() - started, 3),
}
)
return artifacts
def record_skipped(records: list[dict[str, Any]], slug: str, handler: str, reason: str, required: bool) -> None:
records.append(
{
"operator": slug,
"handler": handler,
"status": "skipped-required" if required else "skipped",
"reason": reason,
}
)
def required_externals() -> set[str]:
raw = os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS", ",".join(sorted(DEFAULT_REQUIRED_EXTERNALS))).strip()
if not raw:
return set()
return {item.strip().lower() for item in raw.split(",") if item.strip()}
def acceptance_scope(required: set[str]) -> str:
if DEFAULT_REQUIRED_EXTERNALS.issubset(required):
return "full"
return "dependency-isolation"
def has_passed_record(records: list[dict[str, Any]], operator: str) -> bool:
return any(item.get("operator") == operator and item.get("status") == "passed" for item in records)
def external_verification_summary(records: list[dict[str, Any]], required: set[str]) -> dict[str, Any]:
required_external_systems = sorted(DEFAULT_REQUIRED_EXTERNALS.intersection(required))
passed: list[str] = []
if "bos" in required and has_passed_record(records, "upload_bos_external") and has_passed_record(records, "bos_incremental_scan"):
passed.append("bos")
if "mongo" in required and has_passed_record(records, "mongodb_write_external_verification"):
passed.append("mongo")
return {
"externalVerificationPassed": sorted(passed),
"externalVerificationMissing": sorted(set(required_external_systems) - set(passed)),
"externalVerificationPassedCount": len(passed),
"externalVerificationRequiredCount": len(required_external_systems),
}
def env_bool(name: str, default: bool = False) -> bool:
raw = os.getenv(name)
if raw is None or not raw.strip():
return default
normalized = raw.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
raise AcceptanceFailure(f"{name} must be a boolean value")
def env_text(name: str, default: str = "") -> str:
raw = os.getenv(name)
if raw is None or not raw.strip():
manifest_value = acceptance_manifest_config_text(name)
return manifest_value if manifest_value else default
return raw.strip()
def acceptance_fixture_manifest_path() -> pathlib.Path | None:
raw = os.getenv(FIXTURE_MANIFEST_ENV_NAME, "").strip()
if not raw:
return None
return pathlib.Path(raw)
def load_acceptance_fixture_manifest() -> dict[str, Any]:
global _FIXTURE_MANIFEST
if _FIXTURE_MANIFEST is not None:
return _FIXTURE_MANIFEST
path = acceptance_fixture_manifest_path()
if path is None:
_FIXTURE_MANIFEST = {}
return _FIXTURE_MANIFEST
if not path.exists():
raise AcceptanceFailure(f"POC acceptance fixture manifest does not exist inside the acceptance Job: {path}")
if not path.is_file():
raise AcceptanceFailure(f"POC acceptance fixture manifest must be a file: {path}")
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as error:
raise AcceptanceFailure(f"POC acceptance fixture manifest is not valid JSON: {path}") from error
if not isinstance(payload, dict):
raise AcceptanceFailure(f"POC acceptance fixture manifest must be a JSON object: {path}")
_FIXTURE_MANIFEST = payload
return _FIXTURE_MANIFEST
def manifest_nested_value(payload: dict[str, Any], path: tuple[str, ...]) -> Any:
current: Any = payload
for key in path:
if not isinstance(current, dict) or key not in current:
return None
current = current[key]
return current
def acceptance_manifest_config_text(name: str) -> str:
paths = MANIFEST_CONFIG_PATHS.get(name)
if not paths:
return ""
manifest = load_acceptance_fixture_manifest()
if not manifest:
return ""
for path in paths:
value = manifest_nested_value(manifest, path)
if value is None:
continue
if isinstance(value, str):
return value.strip()
if isinstance(value, (dict, list, int, float, bool)):
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
raise AcceptanceFailure(f"POC acceptance fixture manifest field {'.'.join(path)} for {name} must be a string, number, boolean, object, or array")
return ""
def create_video(path: pathlib.Path, seconds: int = 3) -> dict[str, Any]:
path.parent.mkdir(parents=True, exist_ok=True)
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-y",
"-f",
"lavfi",
"-i",
"testsrc=size=128x96:rate=12",
"-t",
str(seconds),
"-pix_fmt",
"yuv420p",
str(path),
]
subprocess.run(cmd, check=True)
metrics = MODULE.inspect_video_file(path)
ensure(path.exists() and path.stat().st_size > 0, f"failed to generate video fixture: {path}")
ensure(float(metrics.get("duration") or 0) > 0, "generated video did not produce readable duration")
return metrics
def create_xplatform_dataset(base_dir: pathlib.Path) -> tuple[pathlib.Path, pathlib.Path, dict[str, Any]]:
root = base_dir / "inputs" / "xplatform" / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
video_dir = root / "DJI_001"
video_path = video_dir / "DJI_20260318112834_0001_D.MP4"
metrics = create_video(video_path)
write_json(
root / "meta.json",
{
"type": "家庭",
"place": "蓟门里小区",
"date": "20260319",
"project": "poc-operator-acceptance",
},
)
write_json(
root / "video_meta.json",
[
{
"filename": video_path.name,
"codec": "mpeg4",
"resolution": metrics.get("resolution", "128x96"),
"frameRate": metrics.get("frameRate", 12.0),
"duration": metrics.get("duration", 3.0),
}
],
)
return root, video_path, metrics
def create_csv(base_dir: pathlib.Path) -> pathlib.Path:
path = base_dir / "inputs" / "records.csv"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("id,label,score\nclip-a,keep,0.91\nclip-b,reject,0.42\n", encoding="utf-8")
return path
def create_lerobot_dataset(base_dir: pathlib.Path) -> pathlib.Path:
root = base_dir / "inputs" / "lerobot"
meta_dir = root / "meta"
meta_dir.mkdir(parents=True, exist_ok=True)
write_json(meta_dir / "info.json", {"robot_type": "droboticflow-poc-arm", "fps": 10, "features": {"observation.image": {}}})
write_json(meta_dir / "stats.json", {"episodes": 1, "frames": 2})
(meta_dir / "tasks.jsonl").write_text('{"task_index":0,"task":"pick object"}\n', encoding="utf-8")
(meta_dir / "episodes.jsonl").write_text('{"episode_index":0,"tasks":[0],"length":2}\n', encoding="utf-8")
parquet_path = root / "data" / "chunk-000" / "episode_000000.parquet"
parquet_path.parent.mkdir(parents=True, exist_ok=True)
parquet_path.write_bytes(b"PAR1acceptance")
create_video(root / "videos" / "chunk-000" / "observation.image" / "episode_000000.mp4", seconds=1)
return root
def create_rosbag2_dataset(base_dir: pathlib.Path) -> pathlib.Path:
root = base_dir / "inputs" / "rosbag2"
root.mkdir(parents=True, exist_ok=True)
(root / "metadata.yaml").write_text(
"rosbag2_bagfile_information:\n"
" version: 5\n"
" storage_identifier: sqlite3\n"
" relative_file_paths:\n"
" - poc_0.db3\n",
encoding="utf-8",
)
db_path = root / "poc_0.db3"
connection = sqlite3.connect(db_path)
try:
connection.execute(
"create table topics(id integer primary key, name text, type text, serialization_format text, offered_qos_profiles text)"
)
connection.execute("create table messages(id integer primary key, topic_id integer, timestamp integer, data blob)")
connection.execute(
"insert into topics(id, name, type, serialization_format, offered_qos_profiles) values(1, '/camera/color/image_raw', 'sensor_msgs/msg/Image', 'cdr', '')"
)
connection.execute("insert into messages(topic_id, timestamp, data) values(1, 1, ?)", (b"frame",))
connection.commit()
finally:
connection.close()
return root
def fixture_missing_message(env_name: str, description: str) -> str:
return (
f"{description} requires {env_name}=<path> or {FIXTURE_MANIFEST_ENV_NAME}=<json> pointing to a real file or directory inside the acceptance Job. "
"Set POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES=true only for explicitly labeled diagnostic runs; "
"generated fixtures are not valid final POC acceptance evidence."
)
def require_existing_path(env_name: str, description: str, *, kind: str) -> pathlib.Path:
raw = env_text(env_name)
if not raw:
raise AcceptanceFailure(fixture_missing_message(env_name, description))
path = pathlib.Path(raw)
if not path.exists():
raise AcceptanceFailure(f"{description} path from {env_name} does not exist inside the acceptance Job: {path}")
if kind == "file" and not path.is_file():
raise AcceptanceFailure(f"{description} path from {env_name} must be a file: {path}")
if kind == "dir" and not path.is_dir():
raise AcceptanceFailure(f"{description} path from {env_name} must be a directory: {path}")
if kind == "any" and not (path.is_file() or path.is_dir()):
raise AcceptanceFailure(f"{description} path from {env_name} must be a file or directory: {path}")
return path
def resolve_fixture_path(
env_name: str,
description: str,
*,
kind: str,
allow_generated: bool,
generated_factory: Callable[[], pathlib.Path],
) -> tuple[pathlib.Path, bool]:
if env_text(env_name):
return require_existing_path(env_name, description, kind=kind), False
if allow_generated:
return generated_factory(), True
raise AcceptanceFailure(fixture_missing_message(env_name, description))
def first_video_file(root: pathlib.Path) -> pathlib.Path | None:
suffixes = set(getattr(MODULE, "VIDEO_FILE_SUFFIXES", {".mp4", ".mov", ".avi", ".mkv"}))
candidates = root.rglob("*") if root.is_dir() else [root]
for candidate in sorted(candidates):
if candidate.is_file() and candidate.suffix.lower() in suffixes:
return candidate
return None
def load_acceptance_json_payload(path: pathlib.Path) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as error:
raise AcceptanceFailure(f"POC acceptance JSON input is not valid JSON: {path}") from error
if isinstance(payload, list):
if not payload:
raise AcceptanceFailure(f"POC acceptance JSON input list must not be empty: {path}")
if not all(isinstance(item, dict) for item in payload):
raise AcceptanceFailure(f"POC acceptance JSON input list must contain JSON objects: {path}")
return {"items": payload}
if not isinstance(payload, dict):
raise AcceptanceFailure(f"POC acceptance JSON input must be an object or object array: {path}")
items = payload.get("items")
if items is not None and (not isinstance(items, list) or not items):
raise AcceptanceFailure(f"POC acceptance JSON input items must be a non-empty array when present: {path}")
return payload
def load_acceptance_model(path: pathlib.Path) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as error:
raise AcceptanceFailure(f"POC acceptance model JSON is not valid JSON: {path}") from error
if not isinstance(payload, dict):
raise AcceptanceFailure(f"POC acceptance model JSON must be an object: {path}")
model_type = str(payload.get("type", "")).strip()
if not model_type:
raise AcceptanceFailure(f"POC acceptance model JSON must include a non-empty type: {path}")
return payload
def default_generated_json_payload() -> dict[str, Any]:
return {
"items": [
{"id": "clip-a", "label": "keep", "score": 0.91, "quality": {"clarity": 0.93, "shake": 0.02}},
{"id": "clip-b", "label": "reject", "score": 0.42, "quality": {"clarity": 0.45, "shake": 0.18}},
]
}
def default_generated_model_payload() -> dict[str, Any]:
return {
"type": "linear",
"name": "quality-gate",
"features": ["quality.clarity", "quality.shake"],
"weights": [1.0, -2.0],
"bias": 0.0,
"threshold": 0.5,
"positiveLabel": "pass",
"negativeLabel": "fail",
}
def acceptance_fixture_sources(paths: dict[str, pathlib.Path], generated: dict[str, bool]) -> dict[str, Any]:
manifest_path = acceptance_fixture_manifest_path()
return {
"allowGeneratedFixtures": env_bool("POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES", False),
"fixtureManifestPath": str(manifest_path) if manifest_path is not None else "",
"items": {
name: {
"path": str(path),
"generated": generated.get(name, False),
}
for name, path in sorted(paths.items())
},
}
def fixture_source_counts(fixture_sources: dict[str, Any]) -> dict[str, Any]:
items = fixture_sources.get("items") if isinstance(fixture_sources.get("items"), dict) else {}
generated_count = 0
real_count = 0
for item in items.values():
if not isinstance(item, dict):
continue
if item.get("generated") is True:
generated_count += 1
else:
real_count += 1
return {
"generatedFixtureCount": generated_count,
"realFixtureCount": real_count,
"allFixturesReal": bool(items) and generated_count == 0 and fixture_sources.get("allowGeneratedFixtures") is not True,
}
def prepare_acceptance_inputs(base_dir: pathlib.Path) -> dict[str, Any]:
allow_generated = env_bool("POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES", False)
paths: dict[str, pathlib.Path] = {}
generated: dict[str, bool] = {}
xplatform_root, xplatform_generated = resolve_fixture_path(
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"X-platform acceptance dataset",
kind="dir",
allow_generated=allow_generated,
generated_factory=lambda: create_xplatform_dataset(base_dir)[0],
)
paths["xplatform"] = xplatform_root
generated["xplatform"] = xplatform_generated
csv_path, csv_generated = resolve_fixture_path(
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"CSV acceptance input",
kind="file",
allow_generated=allow_generated,
generated_factory=lambda: create_csv(base_dir),
)
suffixes = set(getattr(MODULE, "TABULAR_FILE_SUFFIXES", {".csv", ".tsv"}))
if csv_path.suffix.lower() not in suffixes:
raise AcceptanceFailure(f"CSV acceptance input must be a csv/tsv file: {csv_path}")
paths["csv"] = csv_path
generated["csv"] = csv_generated
lerobot_root, lerobot_generated = resolve_fixture_path(
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"LeRobot acceptance dataset",
kind="dir",
allow_generated=allow_generated,
generated_factory=lambda: create_lerobot_dataset(base_dir),
)
paths["lerobot"] = lerobot_root
generated["lerobot"] = lerobot_generated
rosbag_root, rosbag_generated = resolve_fixture_path(
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"ROSBag acceptance dataset",
kind="any",
allow_generated=allow_generated,
generated_factory=lambda: create_rosbag2_dataset(base_dir),
)
paths["rosbag"] = rosbag_root
generated["rosbag"] = rosbag_generated
json_generated = False
if env_text("POC_ACCEPTANCE_JSON_INPUT_PATH"):
json_path = require_existing_path("POC_ACCEPTANCE_JSON_INPUT_PATH", "JSON acceptance input", kind="file")
json_payload = load_acceptance_json_payload(json_path)
paths["json"] = json_path
elif allow_generated:
json_payload = default_generated_json_payload()
json_path = base_dir / "inputs" / "json-input.json"
write_json(json_path, json_payload)
paths["json"] = json_path
json_generated = True
else:
raise AcceptanceFailure(fixture_missing_message("POC_ACCEPTANCE_JSON_INPUT_PATH", "JSON acceptance input"))
generated["json"] = json_generated
model_generated = False
if env_text("POC_ACCEPTANCE_MODEL_JSON_PATH"):
model_path = require_existing_path("POC_ACCEPTANCE_MODEL_JSON_PATH", "Model acceptance JSON", kind="file")
model_payload = load_acceptance_model(model_path)
paths["model"] = model_path
elif allow_generated:
model_payload = default_generated_model_payload()
model_path = base_dir / "inputs" / "model.json"
write_json(model_path, model_payload)
paths["model"] = model_path
model_generated = True
else:
raise AcceptanceFailure(fixture_missing_message("POC_ACCEPTANCE_MODEL_JSON_PATH", "Model acceptance JSON"))
generated["model"] = model_generated
video_path = first_video_file(xplatform_root)
video_metrics: dict[str, Any] = {}
if video_path is not None:
try:
video_metrics = MODULE.inspect_video_file(video_path)
except Exception as error:
video_metrics = {"inspectionError": f"{type(error).__name__}: {error}"}
return {
"xplatformRoot": xplatform_root,
"csvPath": csv_path,
"jsonPath": paths["json"],
"jsonPayload": json_payload,
"lerobotRoot": lerobot_root,
"rosbagRoot": rosbag_root,
"modelPath": paths["model"],
"model": model_payload,
"videoPath": video_path,
"videoMetrics": video_metrics,
"fixtureSources": acceptance_fixture_sources(paths, generated),
}
def validate_static_operator_layout(base_dir: pathlib.Path) -> dict[str, Any]:
operator_root = builtin_operator_root()
# Preserve this directory for historical runs but do not count it as a
# current catalog builtin: it has no release migration and must not make
# strict acceptance demand an artifact for a retired operator id.
unmigrated_historical = {"temporal_causal_eval"}
metadata_paths = sorted(path for path in operator_root.glob("*/operator.json") if path.parent.name not in unmigrated_historical)
sources = []
repository_manifests = []
repository_manifest_errors = []
legacy = []
builtin_helper_calls = []
hardcoded_external_api_sources = []
seen_operator_ids: dict[str, str] = {}
duplicate_operator_ids = []
seen_operator_names: dict[str, str] = {}
duplicate_operator_names = []
for metadata_path in metadata_paths:
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
slug = metadata_path.parent.name
operator_path = metadata_path.with_name("operator.py")
repository_path = metadata_path.with_name("repository.json")
source = operator_path.read_text(encoding="utf-8") if operator_path.exists() else ""
operator_relative_source = str(operator_path.relative_to(operator_root.parent.parent.parent))
sources.append(operator_relative_source)
operator_id = str(metadata.get("id", "")).strip()
operator_name = str(metadata.get("name", "")).strip()
if not operator_id:
repository_manifest_errors.append(f"{slug}: operator.json id is required")
elif operator_id in seen_operator_ids:
duplicate_operator_ids.append(operator_id)
else:
seen_operator_ids[operator_id] = slug
if not operator_name:
repository_manifest_errors.append(f"{slug}: operator.json name is required")
elif operator_name in seen_operator_names:
duplicate_operator_names.append(operator_name)
else:
seen_operator_names[operator_name] = slug
if slug == "human_review" and metadata.get("runtimeHandler") != "builtin_human_review":
repository_manifest_errors.append("human_review: runtimeHandler must be builtin_human_review")
if "LEGACY_HANDLER" in source:
legacy.append(slug)
if 'helpers["builtin_' in source or "helpers['builtin_" in source:
builtin_helper_calls.append(slug)
if not repository_path.exists():
repository_manifest_errors.append(f"{slug}: missing repository.json")
continue
try:
repository_manifest = json.loads(repository_path.read_text(encoding="utf-8"))
except Exception as error:
repository_manifest_errors.append(f"{slug}: invalid repository.json ({type(error).__name__}: {error})")
continue
expected_fields = {
"schemaVersion": "droboticflow.operator.repository/v1",
"repositoryId": f"builtin/{slug}",
"operatorId": metadata.get("id"),
"operatorSlug": slug,
"sourceRoot": f"backend/operators/builtin/{slug}",
"entrypoint": "operator.py",
"metadata": "operator.json",
"runtimeSource": metadata.get("runtimeSource"),
"runtimeHandler": metadata.get("runtimeHandler"),
"containerEntrypoint": "backend/operator-runner/main.py",
}
for key, expected in expected_fields.items():
if repository_manifest.get(key) != expected:
repository_manifest_errors.append(
f"{slug}: repository.json {key}={repository_manifest.get(key)!r}, expected {expected!r}"
)
if "externalApi" in metadata and repository_manifest.get("externalApi") != metadata.get("externalApi"):
repository_manifest_errors.append(
f"{slug}: repository.json externalApi must match operator.json"
)
acceptance = repository_manifest.get("acceptance") if isinstance(repository_manifest.get("acceptance"), dict) else {}
if not acceptance.get("requiresRealInputs"):
repository_manifest_errors.append(f"{slug}: repository.json must require real inputs")
for key in ("mockOnlyAccepted", "simulationOnlyAccepted", "hardcodedOutputAccepted", "shallowOnlyAccepted"):
if acceptance.get(key) is not False:
repository_manifest_errors.append(f"{slug}: repository.json acceptance.{key} must be false")
if acceptance.get("requiresPersistentWorkspaceEvidence") is not True:
repository_manifest_errors.append(
f"{slug}: repository.json must require persistent workspace evidence"
)
dependencies = repository_manifest.get("runtimeDependencies")
if not isinstance(dependencies, list) or not dependencies:
repository_manifest_errors.append(f"{slug}: repository.json must list runtime dependencies")
repository_manifests.append(str(repository_path.relative_to(operator_root.parent.parent.parent)))
helper_builtin_keys = [key for key in MODULE.OPERATOR_HELPERS if str(key).startswith("builtin_")]
payload = {
"operatorFileCount": len(metadata_paths),
"repositoryManifestCount": len(repository_manifests),
"repositoryManifestErrors": repository_manifest_errors,
"duplicateOperatorIds": sorted(duplicate_operator_ids),
"duplicateOperatorNames": sorted(duplicate_operator_names),
"legacyOperators": legacy,
"builtinHelperCallOperators": builtin_helper_calls,
"hardcodedExternalApiSources": hardcoded_external_api_sources,
"helperBuiltinKeys": helper_builtin_keys,
"runtimeSources": sources,
"repositoryManifests": repository_manifests,
}
write_json(base_dir / "static-operator-layout.json", payload)
ensure(metadata_paths, "expected at least one builtin operator metadata file")
ensure(
len(repository_manifests) == len(metadata_paths),
f"expected one repository manifest per builtin operator, got {len(repository_manifests)}/{len(metadata_paths)}",
)
ensure(not repository_manifest_errors, f"operator repository manifest errors: {repository_manifest_errors}")
ensure(not duplicate_operator_ids, f"duplicate builtin operator ids: {duplicate_operator_ids}")
ensure(not duplicate_operator_names, f"duplicate builtin operator names: {duplicate_operator_names}")
ensure(not legacy, f"legacy handlers still referenced: {legacy}")
ensure(not builtin_helper_calls, f"builtin helper delegation still referenced: {builtin_helper_calls}")
ensure(not hardcoded_external_api_sources, f"external API URLs still hardcoded in runtime sources: {hardcoded_external_api_sources}")
ensure(not helper_builtin_keys, f"legacy builtin helpers still exposed: {helper_builtin_keys}")
return payload
def operator_repository_summary(static_payload: dict[str, Any]) -> dict[str, Any]:
policy_violation_keys = (
"repositoryManifestErrors",
"duplicateOperatorIds",
"duplicateOperatorNames",
"legacyOperators",
"builtinHelperCallOperators",
"hardcodedExternalApiSources",
"helperBuiltinKeys",
)
violations: list[str] = []
for key in policy_violation_keys:
for item in static_payload.get(key, []) or []:
text = str(item).strip()
if text:
violations.append(f"{key}:{text}")
required_count = int(static_payload.get("operatorFileCount") or 0)
repository_count = int(static_payload.get("repositoryManifestCount") or 0)
return {
"operatorRepositoryRequiredCount": required_count,
"operatorRepositoryCount": repository_count,
"operatorRepositoryCoverageComplete": required_count > 0 and repository_count == required_count,
"operatorRepositoryPolicyClean": len(violations) == 0,
"operatorRepositoryPolicyViolations": violations,
}
def strict_full_external_acceptance(
scope: str,
all_operator_coverage_complete: bool,
failures: list[str],
external_summary: dict[str, Any],
fixture_counts: dict[str, Any],
repository_summary: dict[str, Any],
) -> bool:
return (
scope == "full"
and all_operator_coverage_complete
and not failures
and not external_summary.get("externalVerificationMissing")
and int(external_summary.get("externalVerificationPassedCount") or 0)
>= int(external_summary.get("externalVerificationRequiredCount") or 0)
and int(external_summary.get("externalVerificationRequiredCount") or 0) >= len(DEFAULT_REQUIRED_EXTERNALS)
and fixture_counts.get("allFixturesReal") is True
and int(fixture_counts.get("generatedFixtureCount") or 0) == 0
and int(fixture_counts.get("realFixtureCount") or 0) > 0
and repository_summary.get("operatorRepositoryCoverageComplete") is True
and repository_summary.get("operatorRepositoryPolicyClean") is True
and not repository_summary.get("operatorRepositoryPolicyViolations")
)
def network_reachable(url: str) -> bool:
try:
with request.urlopen(url, timeout=10) as response:
return 200 <= int(response.status) < 500
except Exception:
return False
def bos_config() -> dict[str, str]:
return {
"bucket": os.getenv("POC_ACCEPTANCE_BOS_BUCKET", "").strip() or os.getenv("WORKSPACE_BOS_BUCKET", "").strip(),
"endpoint": os.getenv("POC_ACCEPTANCE_BOS_ENDPOINT", "").strip() or os.getenv("BOS_ENDPOINT", "").strip(),
"ak": os.getenv("BOS_ACCESS_KEY_ID", "").strip(),
"sk": os.getenv("BOS_SECRET_ACCESS_KEY", "").strip(),
}
def run_acceptance() -> dict[str, Any]:
run_id = os.getenv("POC_ACCEPTANCE_RUN_ID", f"poc-operator-acceptance-{now_id()}")
workspace_root = pathlib.Path(os.getenv("WORKSPACE_MOUNT_PATH", "/workspace"))
base_dir = pathlib.Path(os.getenv("POC_ACCEPTANCE_WORKDIR", str(workspace_root / "acceptance" / run_id)))
allow_tmp_fallback = env_bool("POC_ACCEPTANCE_ALLOW_TMP_FALLBACK", False)
workspace_fallback_reason = ""
try:
if base_dir.exists():
shutil.rmtree(base_dir)
base_dir.mkdir(parents=True)
except OSError as error:
workspace_fallback_reason = f"{type(error).__name__}: {error}"
if not allow_tmp_fallback:
raise AcceptanceFailure(
f"workspace evidence directory is not writable: {base_dir}; {workspace_fallback_reason}. "
"Set POC_ACCEPTANCE_ALLOW_TMP_FALLBACK=true only for diagnostic runs."
) from error
base_dir = pathlib.Path("/tmp/droboticflow-acceptance") / run_id
if base_dir.exists():
shutil.rmtree(base_dir)
base_dir.mkdir(parents=True)
records: list[dict[str, Any]] = []
failures: list[str] = []
static_payload = validate_static_operator_layout(base_dir)
inputs = prepare_acceptance_inputs(base_dir)
xplatform_root = inputs["xplatformRoot"]
csv_path = inputs["csvPath"]
json_payload = inputs["jsonPayload"]
lerobot_root = inputs["lerobotRoot"]
rosbag_root = inputs["rosbagRoot"]
model = inputs["model"]
video_path = inputs["videoPath"]
video_metrics = inputs["videoMetrics"]
fixture_sources = inputs["fixtureSources"]
fixture_counts = fixture_source_counts(fixture_sources)
required = required_externals()
def safe_run(callable_: Callable[[], Any]) -> Any:
try:
return callable_()
except Exception as error:
failures.append(f"{type(error).__name__}: {error}")
return None
source_artifacts = safe_run(
lambda: run_operator(
run_id,
"real_source",
"source",
base_dir,
node("真实数据源", {"DATA_SOURCE": "LOCAL", "SOURCE_PATH": str(xplatform_root), "SCAN_MODE": "FULL"}),
[],
lambda artifacts: (
lambda payload: {
"entryCount": len(payload.get("entries") or []),
"sourcePath": payload.get("sourcePath"),
}
)(load_payload(first_artifact(artifacts, "source-manifest.json"))),
records,
)
)
source_artifacts = source_artifacts or []
directory_artifacts = safe_run(
lambda: run_operator(
run_id,
"directory_scan",
"directory_scan",
base_dir,
node("目录扫描", {"SOURCE_PATH": str(xplatform_root)}),
[],
lambda artifacts: {"summary": load_payload(first_artifact(artifacts, "directory-scan.json")).get("summary", {})},
records,
)
)
directory_artifacts = directory_artifacts or []
csv_artifacts = safe_run(
lambda: run_operator(
run_id,
"csv_to_json",
"csv_to_json",
base_dir,
node("CSV 转 JSON", {"SOURCE_PATH": str(csv_path)}),
[],
lambda artifacts: {"rowCount": load_payload(first_artifact(artifacts, "table.json")).get("rowCount")},
records,
)
)
csv_artifacts = csv_artifacts or []
json_input = json_artifact(
run_id,
"json-input",
base_dir,
"json-input.json",
json_payload,
)
json_filter_field = env_text("POC_ACCEPTANCE_JSON_FILTER_FIELD", "label")
json_filter_expected = env_text("POC_ACCEPTANCE_JSON_FILTER_EXPECTED", "keep")
json_extract_field = env_text("POC_ACCEPTANCE_JSON_EXTRACT_FIELD", "items.0.decision")
raw_field_map = env_text("POC_ACCEPTANCE_JSON_FIELD_MAP_JSON")
if raw_field_map:
try:
json_field_map = json.loads(raw_field_map)
except json.JSONDecodeError as error:
raise AcceptanceFailure("POC_ACCEPTANCE_JSON_FIELD_MAP_JSON must be valid JSON") from error
if not isinstance(json_field_map, dict) or not all(
isinstance(key, str) and isinstance(value, str) for key, value in json_field_map.items()
):
raise AcceptanceFailure("POC_ACCEPTANCE_JSON_FIELD_MAP_JSON must be an object with string source and destination fields")
else:
json_field_map = {"label": "decision", "score": "confidence"}
filtered_json_artifacts = safe_run(
lambda: run_operator(
run_id,
"json_filter",
"json_filter",
base_dir,
node("JSON 过滤", {"FIELD_PATH": json_filter_field, "OPERATOR": "eq", "EXPECTED": json_filter_expected}),
[json_input],
lambda artifacts: {"count": load_payload(first_artifact(artifacts, "filtered.json")).get("count")},
records,
)
)
filtered_json_artifacts = filtered_json_artifacts or []
mapped_json_artifacts = safe_run(
lambda: run_operator(
run_id,
"json_map_rename",
"json_map",
base_dir,
node("JSON 映射", {"FIELD_MAP_JSON": json_field_map}),
filtered_json_artifacts,
lambda artifacts: {"count": load_payload(first_artifact(artifacts, "mapped.json")).get("count")},
records,
)
)
mapped_json_artifacts = mapped_json_artifacts or []
safe_run(
lambda: run_operator(
run_id,
"json_field_extract",
"json_extract",
base_dir,
node("字段提取", {"FIELD_PATH": json_extract_field}),
mapped_json_artifacts,
lambda artifacts: {"value": load_payload(first_artifact(artifacts, "extracted.json")).get("value")},
records,
)
)
json_to_csv_artifacts = safe_run(
lambda: run_operator(
run_id,
"json_to_csv",
"json_to_csv",
base_dir,
node("JSON 转 CSV"),
mapped_json_artifacts,
lambda artifacts: {"names": [item.get("name") for item in artifacts]},
records,
)
)
json_to_csv_artifacts = json_to_csv_artifacts or []
safe_run(
lambda: run_operator(
run_id,
"conditional_route",
"condition_router",
base_dir,
node(
"条件分流",
{"FIELD_PATH": json_filter_field, "OPERATOR": "eq", "EXPECTED": json_filter_expected},
outputs=[{"id": "matched", "name": "route-match", "type": "json"}, {"id": "unmatched", "name": "route-miss", "type": "json"}],
),
[json_input],
lambda artifacts: {
"matched": load_payload(first_artifact(artifacts, "route-match.json")).get("count"),
"unmatched": load_payload(first_artifact(artifacts, "route-miss.json")).get("count"),
},
records,
)
)
safe_run(
lambda: run_operator(
run_id,
"template_render",
"text_template",
base_dir,
node(
"模板渲染",
{
"TEMPLATE": env_text("POC_ACCEPTANCE_TEMPLATE_TEXT", "accepted={items.0.id}; decision={items.0.decision}"),
"OUTPUT_NAME": "decision.txt",
},
),
mapped_json_artifacts,
lambda artifacts: {"names": [item.get("name") for item in artifacts]},
records,
)
)
safe_run(
lambda: run_operator(
run_id,
"merge",
"merge",
base_dir,
node("数据合并"),
[json_input, *mapped_json_artifacts],
lambda artifacts: {"keys": sorted(load_payload(first_artifact(artifacts, "merged.json")).keys())},
records,
)
)
safe_run(
lambda: run_operator(
run_id,
"human_review",
"human_review",
base_dir,
node("人工审核", {"REVIEW_TYPE": "task", "REVIEWER_ROLE": "qa_reviewer", "INSTRUCTIONS": "POC acceptance"}),
mapped_json_artifacts,
lambda artifacts: {"taskCount": load_payload(first_artifact(artifacts, "human-review-tasks.json")).get("taskCount")},
records,
)
)
xplatform_index_artifacts = safe_run(
lambda: run_operator(
run_id,
"xplatform_dataset_index",
"xplatform_index",
base_dir,
node("X 平台数据集索引", {"SOURCE_PATH": str(xplatform_root)}),
[],
lambda artifacts: {
"collectionCount": load_payload(first_artifact(artifacts, "dataset-index.json")).get("collectionCount"),
"videoCount": load_payload(first_artifact(artifacts, "dataset-index.json")).get("videoCount"),
},
records,
)
)
xplatform_index_artifacts = xplatform_index_artifacts or []
safe_run(
lambda: run_operator(
run_id,
"xplatform_dataset_validate",
"collection_validate",
base_dir,
node("采集目录校验", {"REQUIRE_INTRINSICS": "false"}),
xplatform_index_artifacts,
lambda artifacts: {"validCollectionCount": load_payload(first_artifact(artifacts, "dataset-validation.json")).get("validCollectionCount")},
records,
)
)
quality_artifacts = safe_run(
lambda: run_operator(
run_id,
"video_quality",
"video_quality",
base_dir,
node(
"视频质检",
{
"MIN_CLARITY": env_text("POC_ACCEPTANCE_MIN_CLARITY", "0"),
"MAX_SHAKE": env_text("POC_ACCEPTANCE_MAX_SHAKE", "1"),
"MAX_HAND_OCCLUSION": env_text("POC_ACCEPTANCE_MAX_HAND_OCCLUSION", "1"),
"CLARITY_SCALE": env_text("POC_ACCEPTANCE_CLARITY_SCALE", "35"),
},
),
xplatform_index_artifacts,
lambda artifacts: {
"videoCount": load_payload(first_artifact(artifacts, "quality-report.json")).get("videoCount"),
"passedCount": load_payload(first_artifact(artifacts, "quality-report.json")).get("passedCount"),
},
records,
)
)
quality_artifacts = quality_artifacts or []
filtered_video_artifacts = safe_run(
lambda: run_operator(
run_id,
"video_filter",
"dataset_filter",
base_dir,
node(
"数据过滤",
{
"DATE_START": env_text("POC_ACCEPTANCE_XPLATFORM_DATE_START"),
"DATE_END": env_text("POC_ACCEPTANCE_XPLATFORM_DATE_END"),
"PLACE": env_text("POC_ACCEPTANCE_XPLATFORM_PLACE"),
"TYPE": env_text("POC_ACCEPTANCE_XPLATFORM_TYPE"),
"MIN_CLARITY": env_text("POC_ACCEPTANCE_MIN_CLARITY", "0"),
"MAX_SHAKE": env_text("POC_ACCEPTANCE_MAX_SHAKE", "1"),
"MAX_HAND_OCCLUSION": env_text("POC_ACCEPTANCE_MAX_HAND_OCCLUSION", "1"),
},
),
quality_artifacts,
lambda artifacts: {"count": load_payload(first_artifact(artifacts, "filtered-videos.json")).get("count")},
records,
)
)
filtered_video_artifacts = filtered_video_artifacts or []
sliced_artifacts = safe_run(
lambda: run_operator(
run_id,
"video_slice",
"video_slicing",
base_dir,
node("视频切片", {"SEGMENT_DURATION_SECONDS": env_text("POC_ACCEPTANCE_SEGMENT_DURATION_SECONDS", "1")}),
filtered_video_artifacts,
lambda artifacts: {"clipCount": load_payload(first_artifact(artifacts, "clips-manifest.json")).get("clipCount")},
records,
)
)
sliced_artifacts = sliced_artifacts or []
target_clip_artifacts = safe_run(
lambda: run_operator(
run_id,
"video_target_clip",
"video_target_clip",
base_dir,
node(
"视频目标检测裁剪",
{
"DETECTION_MODE": env_text("POC_ACCEPTANCE_TARGET_DETECTION_MODE", "motion"),
"TARGET_LABELS": env_text("POC_ACCEPTANCE_TARGET_LABELS", "motion"),
"FRAME_INTERVAL_SECONDS": env_text("POC_ACCEPTANCE_TARGET_FRAME_INTERVAL_SECONDS", "0.25"),
"MIN_CONFIDENCE": env_text("POC_ACCEPTANCE_TARGET_MIN_CONFIDENCE", "0.05"),
"MIN_MOTION_AREA_RATIO": env_text("POC_ACCEPTANCE_TARGET_MIN_MOTION_AREA_RATIO", "0.001"),
"MERGE_GAP_SECONDS": env_text("POC_ACCEPTANCE_TARGET_MERGE_GAP_SECONDS", "0.5"),
"MIN_SEGMENT_SECONDS": env_text("POC_ACCEPTANCE_TARGET_MIN_SEGMENT_SECONDS", "0.25"),
"PRE_ROLL_SECONDS": env_text("POC_ACCEPTANCE_TARGET_PRE_ROLL_SECONDS", "0"),
"POST_ROLL_SECONDS": env_text("POC_ACCEPTANCE_TARGET_POST_ROLL_SECONDS", "0"),
"MAX_CLIPS": env_text("POC_ACCEPTANCE_TARGET_MAX_CLIPS", "5"),
},
),
sliced_artifacts,
video_target_clip_evidence,
records,
)
)
target_clip_artifacts = target_clip_artifacts or []
annotated_artifacts = safe_run(
lambda: run_operator(
run_id,
"video_annotate",
"video_annotation",
base_dir,
node("视频标注", {"FRAME_INTERVAL_SECONDS": env_text("POC_ACCEPTANCE_FRAME_INTERVAL_SECONDS", "0.5")}),
target_clip_artifacts,
lambda artifacts: {"annotationCount": load_payload(first_artifact(artifacts, "annotations-manifest.json")).get("annotationCount")},
records,
)
)
annotated_artifacts = annotated_artifacts or []
package_artifacts = safe_run(
lambda: run_operator(
run_id,
"package",
"result_package",
base_dir,
node("成品打包"),
annotated_artifacts,
lambda artifacts: {"packageCount": load_payload(first_artifact(artifacts, "packages-manifest.json")).get("packageCount")},
records,
)
)
package_artifacts = package_artifacts or []
safe_run(
lambda: run_operator(
run_id,
"audit_sample",
"audit_sampling",
base_dir,
node(
"抽检采样",
{
"RULES_JSON": json.loads(
env_text(
"POC_ACCEPTANCE_AUDIT_RULES_JSON",
'[{"id":"all-items","field":"collectionId","operator":"exists","samplePercent":100}]',
)
)
},
),
package_artifacts,
lambda artifacts: {"sampledCount": load_payload(first_artifact(artifacts, "audit-samples.json")).get("sampledCount")},
records,
)
)
safe_run(
lambda: run_operator(
run_id,
"upload",
"upload",
base_dir,
node("结果上传"),
package_artifacts,
lambda artifacts: {"copiedPaths": len(load_payload(first_artifact(artifacts, "uploaded-results.json")).get("copiedPaths") or [])},
records,
)
)
safe_run(
lambda: run_operator(
run_id,
"lerobot_summary",
"lerobot",
base_dir,
node("LeRobot 摘要", {"SOURCE_PATH": str(lerobot_root)}),
[],
lambda artifacts: {
"format": load_payload(first_artifact(artifacts, "lerobot-summary.json")).get("preview", {}).get("format"),
"videoCount": load_payload(first_artifact(artifacts, "lerobot-summary.json")).get("preview", {}).get("videoCount"),
},
records,
)
)
safe_run(
lambda: run_operator(
run_id,
"rosbag_summary",
"rosbag",
base_dir,
node("ROSBag 摘要", {"SOURCE_PATH": str(rosbag_root)}),
[],
lambda artifacts: (
lambda payload: {
"datasetType": payload.get("datasetType"),
"databaseFormat": payload.get("databasePreview", {}).get("format"),
"messageCount": payload.get("databasePreview", {}).get("messageCount"),
}
)(load_payload(first_artifact(artifacts, "rosbag-summary.json"))),
records,
)
)
safe_run(
lambda: run_operator(
run_id,
"model_inference",
"model_inference",
base_dir,
node("模型推理", {"MODEL_JSON": model}),
[json_input],
model_inference_evidence,
records,
)
)
previous_shell_policy = os.getenv("OPERATOR_ALLOW_SHELL_COMMANDS")
os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"] = "true"
shell_command = env_text("POC_ACCEPTANCE_SHELL_COMMAND")
if not shell_command:
if env_bool("POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES", False):
shell_command = "python3 -c 'import json; json.dump({\"shellAccepted\": True}, open(\"result.json\", \"w\"))'"
else:
raise AcceptanceFailure(
"Shell command acceptance requires POC_ACCEPTANCE_SHELL_COMMAND=<command>. "
"Set POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES=true only for diagnostic runs."
)
try:
safe_run(
lambda: run_operator(
run_id,
"shell_command",
"shell_command",
base_dir,
node(
"Shell 命令执行",
{"COMMAND": shell_command},
outputs=[{"id": "result", "name": "result", "type": "json"}],
),
[],
lambda artifacts: {"shellAccepted": load_payload(first_artifact(artifacts, "result.json")).get("shellAccepted")},
records,
)
)
finally:
if previous_shell_policy is None:
os.environ.pop("OPERATOR_ALLOW_SHELL_COMMANDS", None)
else:
os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"] = previous_shell_policy
mongo_uri = os.getenv("MONGO_URI", "").strip()
mongo_db = os.getenv("MONGO_DB", "").strip() or "poc"
mongo_collection = f"operator_acceptance_{run_id.replace('-', '_')}"
if mongo_uri:
mongo_artifacts = safe_run(
lambda: run_operator(
run_id,
"mongodb_write",
"mongodb_write",
base_dir,
node("MongoDB 写入", {"MONGO_URI": mongo_uri, "MONGO_DB": mongo_db, "MONGO_COLLECTION": mongo_collection}),
[json_input],
lambda artifacts: {"insertedCount": load_payload(first_artifact(artifacts, "mongodb-write-result.json")).get("insertedCount")},
records,
)
)
if mongo_artifacts:
try:
client = MODULE.build_mongo_client(mongo_uri)
try:
count = client[mongo_db][mongo_collection].count_documents({})
ensure(count >= 2, f"MongoDB verification expected at least 2 documents, got {count}")
client[mongo_db][mongo_collection].drop()
records.append(
{
"operator": "mongodb_write_external_verification",
"handler": "mongodb_write",
"status": "passed",
"evidence": {"database": mongo_db, "collection": mongo_collection, "verifiedDocuments": count, "dropped": True},
}
)
finally:
client.close()
except Exception as error:
failures.append(f"MongoDB external verification failed: {error}")
else:
record_skipped(records, "mongodb_write", "mongodb_write", "MONGO_URI is not configured in POC Job", "mongo" in required)
bos = bos_config()
if all(bos.values()):
bos_prefix = f"system/acceptance/droboticflow/{run_id}"
seed_artifact = json_artifact(
run_id,
"bos-seed",
base_dir,
"bos-seed.json",
{"runId": run_id, "kind": "bos-acceptance", "createdAt": dt.datetime.now(dt.timezone.utc).isoformat()},
)
bos_upload_artifacts = safe_run(
lambda: run_operator(
run_id,
"upload_bos_external",
"upload",
base_dir,
node(
"结果上传",
{
"DATA_SOURCE": "BOS",
"OUTPUT_PATH": f"bos://{bos['bucket']}/{bos_prefix}",
"BOS_ENDPOINT": bos["endpoint"],
"BOS_ACCESS_KEY_ID": bos["ak"],
"BOS_SECRET_ACCESS_KEY": bos["sk"],
},
),
[seed_artifact],
lambda artifacts: {
"uploadedCount": load_payload(first_artifact(artifacts, "uploaded-results.json")).get("uploadedCount"),
"destinationPath": load_payload(first_artifact(artifacts, "uploaded-results.json")).get("destinationPath"),
},
records,
)
)
if bos_upload_artifacts:
safe_run(
lambda: run_operator(
run_id,
"bos_incremental_scan",
"bos_incremental_scan",
base_dir,
node(
"BOS 增量扫描",
{
"SOURCE_PATH": f"bos://{bos['bucket']}/{bos_prefix}",
"BOS_ENDPOINT": bos["endpoint"],
"BOS_ACCESS_KEY_ID": bos["ak"],
"BOS_SECRET_ACCESS_KEY": bos["sk"],
"SCAN_MODE": "FULL",
"STORAGE_PATTERN": "*.json",
},
),
[],
lambda artifacts: (
lambda payload: {"entryCount": len(payload.get("entries") or []), "sourcePath": payload.get("sourcePath")}
)(load_payload(first_artifact(artifacts, "source-manifest.json"))),
records,
)
)
else:
missing = [key for key, value in bos.items() if not value]
record_skipped(records, "bos_incremental_scan", "bos_incremental_scan", f"missing BOS config: {', '.join(missing)}", "bos" in required)
if "bos" in required:
failures.append(f"BOS required but missing config: {', '.join(missing)}")
passed = [item for item in records if item.get("status") == "passed"]
failed = [item for item in records if item.get("status") == "failed"]
required_skips = [item for item in records if item.get("status") == "skipped-required"]
optional_skips = [item for item in records if item.get("status") == "skipped"]
status_counts: dict[str, int] = {}
for item in records:
status = str(item.get("status", "unknown"))
status_counts[status] = status_counts.get(status, 0) + 1
all_operator_names = sorted({path.parent.name for path in pathlib.Path("/app/backend/operators/builtin").glob("*/operator.json")})
if not all_operator_names:
operator_root = pathlib.Path(__file__).resolve().parents[1] / "backend" / "operators" / "builtin"
all_operator_names = sorted({path.parent.name for path in operator_root.glob("*/operator.json")})
covered_operator_names = sorted(
{
str(item.get("operator"))
for item in records
if item.get("status") == "passed" and not str(item.get("operator", "")).endswith("_external_verification")
}
)
required_slug_aliases = {"upload_bos_external": "upload"}
normalized_covered = sorted({required_slug_aliases.get(item, item) for item in covered_operator_names})
missing_coverage = sorted(set(all_operator_names) - set(normalized_covered))
allowed_missing_coverage: set[str] = set()
if "bos" not in required:
allowed_missing_coverage.add("bos_incremental_scan")
blocking_missing_coverage = sorted(set(missing_coverage) - allowed_missing_coverage)
if blocking_missing_coverage:
failures.append(f"operator coverage missing: {', '.join(blocking_missing_coverage)}")
for item in required_skips:
failures.append(f"required external skipped: {item.get('operator')} ({item.get('reason')})")
for item in failed:
failures.append(f"operator failed: {item.get('operator')} ({item.get('error')})")
scope = acceptance_scope(required)
all_operator_coverage_complete = not missing_coverage
external_summary = external_verification_summary(records, required)
repository_summary = operator_repository_summary(static_payload)
full_external_acceptance = strict_full_external_acceptance(
scope,
all_operator_coverage_complete,
failures,
external_summary,
fixture_counts,
repository_summary,
)
summary = {
"runId": run_id,
"workspace": str(base_dir),
"workspaceFallbackReason": workspace_fallback_reason,
"acceptanceScope": scope,
"fullExternalAcceptance": full_external_acceptance,
"operatorCount": len(all_operator_names),
"operatorPassedCount": len(normalized_covered),
"coveredOperatorCount": len(normalized_covered),
"allOperatorCoverageComplete": all_operator_coverage_complete,
"blockingOperatorCoverageComplete": not blocking_missing_coverage,
"missingCoverage": missing_coverage,
"blockingMissingCoverage": blocking_missing_coverage,
"requiredExternals": sorted(required),
"statusCounts": status_counts,
"passedCount": len(passed),
**external_summary,
"failedCount": len(failed),
"skippedCount": len(required_skips) + len(optional_skips),
"requiredSkippedCount": len(required_skips),
"optionalSkippedCount": len(optional_skips),
"fixtureSources": fixture_sources,
**fixture_counts,
"videoFixture": {"path": str(video_path or ""), **video_metrics},
"static": static_payload,
**repository_summary,
"records": records,
"failures": failures,
"status": "passed" if not failures else "failed",
**acceptance_provenance(),
}
write_json(base_dir / "poc-operator-full-acceptance-result.json", summary)
return summary
def main() -> int:
try:
summary = run_acceptance()
except Exception as error:
summary = {
"status": "failed",
"fatalError": f"{type(error).__name__}: {error}",
"records": [],
**acceptance_provenance(tolerate_errors=True),
}
print(RESULT_PREFIX + json.dumps(MODULE.json_compatible(summary), ensure_ascii=False), flush=True)
return 1
print(RESULT_PREFIX + json.dumps(MODULE.json_compatible(summary), ensure_ascii=False), flush=True)
return 0 if summary.get("status") == "passed" else 1
if __name__ == "__main__":
raise SystemExit(main())