droboticflow-new/scripts/wjzh_operator_regression.py
2026-06-23 06:38:41 +08:00

1870 lines
80 KiB
Python
Executable File

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import mimetypes
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
import urllib.parse
import uuid
from dataclasses import dataclass
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from lib.droboticflow_acceptance import DroboticflowClient
from lib.droboticflow_env import api_base_url_from_env
from lib.quality_thresholds import parse_positive_int, positive_float_issues, positive_int_issues, unit_interval_threshold_issues
from lib.sampling_rules import normalize_sampling_rules_json as normalize_rules_json
DEFAULT_PROJECT_PREFIX = "真实数据算子回归"
CLIENT: DroboticflowClient | None = None
class RegressionError(RuntimeError):
pass
@dataclass
class Config:
api_base_url: str
api_token: str
acceptance_email: str
acceptance_password: str
source_path: str
upload_root_prefix: str
project_prefix: str
storage_connection_id: str
storage_connection_name: str
create_connection_if_missing: bool
mongo_connection_id: str
mongo_collection_prefix: str
bos_endpoint: str
bos_bucket: str
bos_access_key_id: str
bos_secret_access_key: str
delete_all_projects: bool
delete_regression_projects: bool
preflight_only: bool
skip_runtime_acceptance: bool
max_wait_seconds: int
audit_video: str
medium_video: str
huge_video: str
huge_window_seconds: str
route_device_id: str
filter_date_start: str
filter_date_end: str
lerobot_source_path: str
rosbag_source_path: str
incremental_place: str
incremental_camera_model: str
incremental_video_subdir: str
sampling_rules_json: str
quality_min_clarity: str
quality_max_shake: str
quality_max_hand_occlusion: str
medium_window_seconds: str
medium_segment_duration_seconds: str
huge_segment_duration_seconds: str
batch_window_seconds: str
batch_quality_window_seconds: str
batch_segment_duration_seconds: str
incremental_window_seconds: str
incremental_segment_duration_seconds: str
annotation_frame_interval_seconds: str
def env_default(name: str, fallback: str = "") -> str:
return os.getenv(name, fallback).strip()
def parse_positive_int_value(name: str, value: str) -> int:
parsed, issue = parse_positive_int(name, value)
if issue:
raise RegressionError(issue)
if parsed is None:
raise RegressionError(f"{name} is required")
return parsed
def positive_int_arg(name: str):
def parse(value: str) -> int:
parsed, issue = parse_positive_int(name, value)
if issue:
raise argparse.ArgumentTypeError(issue)
if parsed is None:
raise argparse.ArgumentTypeError(f"{name} is required")
return parsed
return parse
def positive_int_env_default(name: str, fallback: int) -> int:
value = env_default(name)
if not value:
return fallback
return parse_positive_int_value(name, value)
def client_for_config(config: Config) -> DroboticflowClient:
global CLIENT
if CLIENT is None or CLIENT.api_base_url != config.api_base_url or CLIENT.api_token != config.api_token:
CLIENT = DroboticflowClient(
config.api_base_url,
api_token=config.api_token,
error_factory=RegressionError,
)
return CLIENT
def normalize_sampling_rules_json(value: str) -> str:
return normalize_rules_json(value, error_factory=RegressionError)
def quality_hyperparams(config: Config) -> dict[str, str]:
return {
"MIN_CLARITY": config.quality_min_clarity,
"MAX_SHAKE": config.quality_max_shake,
"MAX_HAND_OCCLUSION": config.quality_max_hand_occlusion,
}
def filter_hyperparams(config: Config) -> dict[str, str]:
return {
"DATE_START": config.filter_date_start,
"DATE_END": config.filter_date_end,
**quality_hyperparams(config),
}
def int_config(value: str) -> int:
return int(str(value).strip())
def segment_hyperparams(duration_seconds: str) -> dict[str, str]:
return {"SEGMENT_DURATION_SECONDS": str(duration_seconds).strip()}
def annotation_hyperparams(config: Config) -> dict[str, str]:
return {"FRAME_INTERVAL_SECONDS": config.annotation_frame_interval_seconds}
def validate_config(config: Config) -> None:
issues: list[str] = []
parsed_filter_start: time.struct_time | None = None
parsed_filter_end: time.struct_time | None = None
if bool(config.acceptance_email) != bool(config.acceptance_password):
issues.append("DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together")
for label, value in (
("SOURCE_PATH", config.source_path),
("UPLOAD_ROOT_PREFIX", config.upload_root_prefix),
):
if not value:
issues.append(f"{label} is required")
if not config.storage_connection_id and not config.storage_connection_name:
issues.append("STORAGE_CONNECTION_ID or STORAGE_CONNECTION_NAME is required")
if config.create_connection_if_missing and not config.storage_connection_name:
issues.append("STORAGE_CONNECTION_NAME is required when CREATE_CONNECTION_IF_MISSING=true")
if not config.medium_video:
issues.append("MEDIUM_VIDEO is required")
if config.max_wait_seconds < 60:
issues.append("MAX_WAIT_SECONDS must be greater than or equal to 60")
if not config.audit_video:
issues.append("AUDIT_VIDEO or MEDIUM_VIDEO is required")
if not config.huge_video:
issues.append("HUGE_VIDEO is required")
for label, value in (
("ROUTE_DEVICE_ID", config.route_device_id),
("FILTER_DATE_START", config.filter_date_start),
("FILTER_DATE_END", config.filter_date_end),
("LEROBOT_SOURCE_PATH", config.lerobot_source_path),
("ROSBAG_SOURCE_PATH", config.rosbag_source_path),
("INCREMENTAL_PLACE", config.incremental_place),
("INCREMENTAL_CAMERA_MODEL", config.incremental_camera_model),
("INCREMENTAL_VIDEO_SUBDIR", config.incremental_video_subdir),
):
if not value:
issues.append(f"{label} is required")
for label, value in (
("FILTER_DATE_START", config.filter_date_start),
("FILTER_DATE_END", config.filter_date_end),
):
if not value:
continue
try:
parsed = time.strptime(value, "%Y-%m-%d")
except ValueError:
issues.append(f"{label} must use YYYY-MM-DD format")
continue
if label == "FILTER_DATE_START":
parsed_filter_start = parsed
else:
parsed_filter_end = parsed
if parsed_filter_start is not None and parsed_filter_end is not None and parsed_filter_start > parsed_filter_end:
issues.append("FILTER_DATE_START must be earlier than or equal to FILTER_DATE_END")
for label, value in (
("LEROBOT_SOURCE_PATH", config.lerobot_source_path),
("ROSBAG_SOURCE_PATH", config.rosbag_source_path),
):
if value.startswith("bos://"):
issues.append(f"{label} must be a filesystem path visible inside operator-runner Jobs, not a bos:// path")
if config.create_connection_if_missing and not config.bos_bucket:
issues.append("BOS_BUCKET is required when CREATE_CONNECTION_IF_MISSING=true")
try:
config.sampling_rules_json = normalize_sampling_rules_json(config.sampling_rules_json)
except RegressionError as err:
issues.append(str(err))
issues.extend(
unit_interval_threshold_issues(
(
("QUALITY_MIN_CLARITY", config.quality_min_clarity),
("QUALITY_MAX_SHAKE", config.quality_max_shake),
("QUALITY_MAX_HAND_OCCLUSION", config.quality_max_hand_occlusion),
)
)
)
issues.extend(
positive_int_issues(
(
("HUGE_WINDOW_SECONDS", config.huge_window_seconds),
("MEDIUM_WINDOW_SECONDS", config.medium_window_seconds),
("MEDIUM_SEGMENT_DURATION_SECONDS", config.medium_segment_duration_seconds),
("HUGE_SEGMENT_DURATION_SECONDS", config.huge_segment_duration_seconds),
("BATCH_WINDOW_SECONDS", config.batch_window_seconds),
("BATCH_QUALITY_WINDOW_SECONDS", config.batch_quality_window_seconds),
("BATCH_SEGMENT_DURATION_SECONDS", config.batch_segment_duration_seconds),
("INCREMENTAL_WINDOW_SECONDS", config.incremental_window_seconds),
("INCREMENTAL_SEGMENT_DURATION_SECONDS", config.incremental_segment_duration_seconds),
)
)
)
issues.extend(positive_float_issues((("ANNOTATION_FRAME_INTERVAL_SECONDS", config.annotation_frame_interval_seconds),)))
if issues:
raise RegressionError(
"invalid regression configuration:\n"
+ "\n".join(f"- {item}" for item in issues)
+ "\nSet the values explicitly in the environment or CLI arguments; the script has no repository defaults for live data paths."
)
def login_if_configured(config: Config) -> None:
client_for_config(config).login_if_configured(
config.acceptance_email,
config.acceptance_password,
label="WJZH regression user",
)
def parse_args() -> Config:
parser = argparse.ArgumentParser(description="Run production-style WJZH operator regression workflows")
parser.add_argument("--api-base-url", default=api_base_url_from_env(include_base_url=False))
parser.add_argument("--api-token", default=env_default("DROBOTICFLOW_API_TOKEN"))
parser.add_argument(
"--acceptance-email",
dest="acceptance_email",
default=env_default("DROBOTICFLOW_ACCEPTANCE_EMAIL"),
)
parser.add_argument(
"--acceptance-password",
dest="acceptance_password",
default=os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""),
)
parser.add_argument("--source-path", default=env_default("SOURCE_PATH"))
parser.add_argument("--upload-root-prefix", default=env_default("UPLOAD_ROOT_PREFIX"))
parser.add_argument("--project-prefix", default=env_default("PROJECT_PREFIX", DEFAULT_PROJECT_PREFIX))
parser.add_argument("--storage-connection-id", default=env_default("STORAGE_CONNECTION_ID"))
parser.add_argument("--storage-connection-name", default=env_default("STORAGE_CONNECTION_NAME"))
parser.add_argument(
"--create-connection-if-missing",
action="store_true",
default=env_default("CREATE_CONNECTION_IF_MISSING").lower() in {"1", "true", "yes"},
)
parser.add_argument("--mongo-connection-id", default=env_default("MONGO_CONNECTION_ID"))
parser.add_argument("--mongo-collection-prefix", default=env_default("MONGO_COLLECTION_PREFIX", "wjzh_operator_regression"))
parser.add_argument("--bos-endpoint", default=env_default("BOS_ENDPOINT"))
parser.add_argument("--bos-bucket", default=env_default("BOS_BUCKET"))
parser.add_argument("--bos-access-key-id", default=env_default("BOS_ACCESS_KEY_ID"))
parser.add_argument("--bos-secret-access-key", default=env_default("BOS_SECRET_ACCESS_KEY"))
parser.add_argument("--delete-all-projects", action="store_true")
parser.add_argument("--delete-regression-projects", action="store_true")
parser.add_argument("--preflight-only", action="store_true")
parser.add_argument("--skip-runtime-acceptance", dest="skip_runtime_acceptance", action="store_true")
parser.add_argument(
"--max-wait-seconds",
type=positive_int_arg("MAX_WAIT_SECONDS"),
default=positive_int_env_default("MAX_WAIT_SECONDS", 5400),
)
parser.add_argument("--audit-video", default=env_default("AUDIT_VIDEO"))
parser.add_argument("--medium-video", default=env_default("MEDIUM_VIDEO"))
parser.add_argument("--huge-video", default=env_default("HUGE_VIDEO"))
parser.add_argument("--huge-window-seconds", default=env_default("HUGE_WINDOW_SECONDS"))
parser.add_argument("--route-device-id", default=env_default("ROUTE_DEVICE_ID"))
parser.add_argument("--filter-date-start", default=env_default("FILTER_DATE_START"))
parser.add_argument("--filter-date-end", default=env_default("FILTER_DATE_END"))
parser.add_argument("--lerobot-source-path", default=env_default("LEROBOT_SOURCE_PATH"))
parser.add_argument("--rosbag-source-path", default=env_default("ROSBAG_SOURCE_PATH"))
parser.add_argument("--incremental-place", default=env_default("INCREMENTAL_PLACE"))
parser.add_argument("--incremental-camera-model", default=env_default("INCREMENTAL_CAMERA_MODEL"))
parser.add_argument("--incremental-video-subdir", default=env_default("INCREMENTAL_VIDEO_SUBDIR"))
parser.add_argument("--sampling-rules-json", default=env_default("SAMPLING_RULES_JSON"))
parser.add_argument("--quality-min-clarity", default=env_default("QUALITY_MIN_CLARITY"))
parser.add_argument("--quality-max-shake", default=env_default("QUALITY_MAX_SHAKE"))
parser.add_argument("--quality-max-hand-occlusion", default=env_default("QUALITY_MAX_HAND_OCCLUSION"))
parser.add_argument("--medium-window-seconds", default=env_default("MEDIUM_WINDOW_SECONDS"))
parser.add_argument("--medium-segment-duration-seconds", default=env_default("MEDIUM_SEGMENT_DURATION_SECONDS"))
parser.add_argument("--huge-segment-duration-seconds", default=env_default("HUGE_SEGMENT_DURATION_SECONDS"))
parser.add_argument("--batch-window-seconds", default=env_default("BATCH_WINDOW_SECONDS"))
parser.add_argument("--batch-quality-window-seconds", default=env_default("BATCH_QUALITY_WINDOW_SECONDS"))
parser.add_argument("--batch-segment-duration-seconds", default=env_default("BATCH_SEGMENT_DURATION_SECONDS"))
parser.add_argument("--incremental-window-seconds", default=env_default("INCREMENTAL_WINDOW_SECONDS"))
parser.add_argument("--incremental-segment-duration-seconds", default=env_default("INCREMENTAL_SEGMENT_DURATION_SECONDS"))
parser.add_argument("--annotation-frame-interval-seconds", default=env_default("ANNOTATION_FRAME_INTERVAL_SECONDS"))
args = parser.parse_args()
return Config(
api_base_url=args.api_base_url.rstrip("/"),
api_token=args.api_token,
acceptance_email=args.acceptance_email,
acceptance_password=args.acceptance_password,
source_path=args.source_path,
upload_root_prefix=args.upload_root_prefix.rstrip("/"),
project_prefix=args.project_prefix,
storage_connection_id=args.storage_connection_id,
storage_connection_name=args.storage_connection_name,
create_connection_if_missing=bool(args.create_connection_if_missing),
mongo_connection_id=args.mongo_connection_id,
mongo_collection_prefix=args.mongo_collection_prefix.strip() or "wjzh_operator_regression",
bos_endpoint=args.bos_endpoint,
bos_bucket=args.bos_bucket,
bos_access_key_id=args.bos_access_key_id,
bos_secret_access_key=args.bos_secret_access_key,
delete_all_projects=bool(args.delete_all_projects),
delete_regression_projects=bool(args.delete_regression_projects),
preflight_only=bool(args.preflight_only),
skip_runtime_acceptance=bool(args.skip_runtime_acceptance),
max_wait_seconds=args.max_wait_seconds,
audit_video=args.audit_video or args.medium_video,
medium_video=args.medium_video,
huge_video=args.huge_video,
huge_window_seconds=args.huge_window_seconds,
route_device_id=args.route_device_id,
filter_date_start=args.filter_date_start,
filter_date_end=args.filter_date_end,
lerobot_source_path=args.lerobot_source_path,
rosbag_source_path=args.rosbag_source_path,
incremental_place=args.incremental_place,
incremental_camera_model=args.incremental_camera_model,
incremental_video_subdir=args.incremental_video_subdir,
sampling_rules_json=args.sampling_rules_json,
quality_min_clarity=args.quality_min_clarity,
quality_max_shake=args.quality_max_shake,
quality_max_hand_occlusion=args.quality_max_hand_occlusion,
medium_window_seconds=args.medium_window_seconds,
medium_segment_duration_seconds=args.medium_segment_duration_seconds,
huge_segment_duration_seconds=args.huge_segment_duration_seconds,
batch_window_seconds=args.batch_window_seconds,
batch_quality_window_seconds=args.batch_quality_window_seconds,
batch_segment_duration_seconds=args.batch_segment_duration_seconds,
incremental_window_seconds=args.incremental_window_seconds,
incremental_segment_duration_seconds=args.incremental_segment_duration_seconds,
annotation_frame_interval_seconds=args.annotation_frame_interval_seconds,
)
def request(config: Config, method: str, path: str, payload: Any | None = None, expected: tuple[int, ...] = (200, 201, 202, 204)) -> Any:
return client_for_config(config).request_json(method, path, payload, expected=expected)
def request_bytes(config: Config, method: str, path: str, body: bytes | None = None, headers: dict[str, str] | None = None, expected: tuple[int, ...] = (200,)) -> tuple[int, bytes, dict[str, str]]:
return client_for_config(config).request_bytes(
method,
path,
body=body,
headers=headers,
expected=expected,
)
def object_storage_list(config: Config, connection_id: str, prefix: str, *, bucket: str = "") -> list[dict[str, Any]]:
query = urllib.parse.urlencode(
{
"connectionId": connection_id,
"bucket": bucket,
"prefix": prefix,
}
)
payload = request(config, "GET", f"/api/object-storage/objects?{query}")
return payload.get("items", [])
def object_storage_read(config: Config, connection_id: str, key: str, *, bucket: str = "") -> bytes:
query = urllib.parse.urlencode(
{
"connectionId": connection_id,
"bucket": bucket,
"key": key,
}
)
_, payload, _ = request_bytes(config, "GET", f"/api/object-storage/content?{query}")
return payload
def object_storage_upload(config: Config, connection_id: str, key: str, payload: bytes, *, bucket: str = "", filename: str = "", content_type: str = "") -> Any:
boundary = f"----CodexBoundary{uuid.uuid4().hex}"
filename = filename or os.path.basename(key) or "upload.bin"
content_type = content_type or mimetypes.guess_type(filename)[0] or "application/octet-stream"
body = (
f"--{boundary}\r\n"
f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'
f"Content-Type: {content_type}\r\n\r\n"
).encode("utf-8") + payload + f"\r\n--{boundary}--\r\n".encode("utf-8")
query = urllib.parse.urlencode(
{
"connectionId": connection_id,
"bucket": bucket,
"key": key,
}
)
_, raw, _ = request_bytes(
config,
"POST",
f"/api/object-storage/objects/upload?{query}",
body=body,
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
)
return json.loads(raw.decode("utf-8"))
def object_storage_delete_prefix(config: Config, connection_id: str, key: str, *, bucket: str = "") -> None:
query = urllib.parse.urlencode(
{
"connectionId": connection_id,
"bucket": bucket,
"key": key,
"recursive": "true",
}
)
request_bytes(config, "DELETE", f"/api/object-storage/objects?{query}", expected=(200,))
def parse_bos_path(path: str) -> tuple[str, str]:
raw = str(path or "").strip()
if not raw.startswith("bos://"):
raise RegressionError(f"expected bos:// path, got: {raw}")
bucket_and_key = raw[len("bos://") :]
bucket, _, key = bucket_and_key.partition("/")
return bucket.strip(), key.strip().strip("/")
def required_operators() -> list[str]:
return [
"真实数据源",
"BOS 增量扫描",
"目录扫描",
"模型推理",
"X 平台数据集索引",
"采集目录校验",
"JSON 字段提取",
"文本模板渲染",
"JSON 过滤",
"JSON 映射重命名",
"JSON 转 CSV",
"CSV 转 JSON",
"条件分流",
"数据合并",
"视频质检",
"数据过滤",
"视频切片",
"视频标注",
"成品打包",
"抽检采样",
"人工审核",
"MongoDB 写入",
"LeRobot 数据集概览",
"ROSBag 数据概览",
"结果上传",
"Shell 命令执行",
]
def find_storage_connection(config: Config) -> dict[str, Any] | None:
items = request(config, "GET", "/api/storage/connections")["items"]
if config.storage_connection_id:
return next((item for item in items if item.get("id") == config.storage_connection_id), None)
if config.storage_connection_name:
return next((item for item in items if str(item.get("name", "")).strip() == config.storage_connection_name), None)
return None
def list_mongo_connections(config: Config) -> list[dict[str, Any]]:
return request(config, "GET", "/api/mongo/connections")["items"]
def find_mongo_connection(config: Config) -> dict[str, Any] | None:
items = list_mongo_connections(config)
if config.mongo_connection_id:
return next((item for item in items if item.get("id") == config.mongo_connection_id), None)
return items[0] if items else None
def ensure_storage_connection(config: Config) -> dict[str, Any]:
existing = find_storage_connection(config)
if existing is not None:
return existing
if not config.create_connection_if_missing:
raise RegressionError(
"missing storage connection. Set STORAGE_CONNECTION_ID / STORAGE_CONNECTION_NAME, "
"or rerun with --create-connection-if-missing plus STORAGE_CONNECTION_NAME, BOS_BUCKET, "
"BOS_ACCESS_KEY_ID, and BOS_SECRET_ACCESS_KEY."
)
missing = []
if not config.bos_access_key_id:
missing.append("BOS_ACCESS_KEY_ID")
if not config.bos_secret_access_key:
missing.append("BOS_SECRET_ACCESS_KEY")
if missing:
raise RegressionError(f"cannot create storage connection; missing env: {', '.join(missing)}")
payload = {
"name": config.storage_connection_name,
"description": "Created by scripts/wjzh_operator_regression.py",
"provider": "BOS",
"endpoint": config.bos_endpoint,
"bucket": config.bos_bucket,
"accessKeyId": config.bos_access_key_id,
"secretAccessKey": config.bos_secret_access_key,
}
return request(config, "POST", "/api/storage/connections", payload, expected=(201,))
def settings_has_workspace(settings: dict[str, Any]) -> bool:
k8s = settings.get("k8s") if isinstance(settings, dict) else None
if not isinstance(k8s, dict):
return False
value = str(k8s.get("workspacePVCName", "") or "").strip()
return value != ""
def validate_preflight(
config: Config,
connection: dict[str, Any],
operators: list[dict[str, Any]],
settings: dict[str, Any],
mongo_connection: dict[str, Any] | None,
) -> None:
source_bucket, _ = parse_bos_path(config.source_path)
upload_bucket, _ = parse_bos_path(config.upload_root_prefix)
issues: list[str] = []
if not settings_has_workspace(settings):
issues.append(
"settings.k8s.workspacePVCName is missing. Deploy the current api/worker version and configure WORKSPACE_PVC_NAME / workspacePVCName first."
)
if "enabled" in (settings.get("k8s") or {}):
issues.append(
"live /api/settings still returns k8s.enabled, which indicates the cluster is likely running an older server build than this repository."
)
if str(connection.get("provider", "")).strip().upper() != "BOS":
issues.append("storage connection provider must be BOS")
connection_bucket = str(connection.get("bucket", "")).strip()
if connection_bucket and connection_bucket != source_bucket:
issues.append(
f"storage connection bucket mismatch: connection bucket={connection_bucket}, source bucket={source_bucket}"
)
if connection_bucket and connection_bucket != upload_bucket:
issues.append(
f"storage connection bucket mismatch: connection bucket={connection_bucket}, upload bucket={upload_bucket}"
)
if mongo_connection is None:
issues.append("no Mongo connection available for MongoDB 写入 real regression")
by_name = {item.get("name"): item for item in operators}
missing_ops = [name for name in required_operators() if name not in by_name]
if missing_ops:
issues.append(f"missing operators: {', '.join(missing_ops)}")
if issues:
message = {
"summary": "preflight failed",
"issues": issues,
"requiredConfiguration": [
"deploy current api and worker images from this repository",
"configure shared workspace PVC via WORKSPACE_PVC_NAME / settings.k8s.workspacePVCName",
f"ensure a BOS storage connection exists for source bucket {source_bucket} and upload bucket {upload_bucket}",
],
}
raise RegressionError(json.dumps(message, ensure_ascii=False, indent=2))
def build_node(operator: dict[str, Any], node_id: str, x: int, y: int, overrides: dict[str, str] | None = None, *, connection_id: str = "", label: str | None = None) -> dict[str, Any]:
hyperparams = [dict(item) for item in operator.get("hyperparams", [])]
overrides = overrides or {}
seen: set[str] = set()
for item in hyperparams:
key = str(item.get("key", "")).strip()
if key in overrides:
item["value"] = overrides[key]
seen.add(key)
for key, value in overrides.items():
if key in seen:
continue
hyperparams.append({"key": key, "value": value})
return {
"id": node_id,
"type": "operator",
"data": {
"label": label or operator["name"],
"category": operator["category"],
"description": operator["description"],
"inputs": operator.get("inputs", []),
"outputs": operator.get("outputs", []),
"image": operator.get("image", ""),
"skillDefinition": operator.get("skillDefinition", ""),
"hyperparams": hyperparams,
"connectionId": connection_id,
},
"position": {"x": x, "y": y},
}
def upload_node(operator: dict[str, Any], connection_id: str, node_id: str, x: int, y: int, output_path: str) -> dict[str, Any]:
return build_node(
operator,
node_id,
x,
y,
{
"DATA_SOURCE": "BOS",
"OUTPUT_PATH": output_path,
"SOURCE_PATH": output_path,
},
connection_id=connection_id,
)
def create_project(config: Config, name: str, description: str) -> dict[str, Any]:
return request(config, "POST", "/api/projects", {"name": name, "description": description}, expected=(201,))
def create_workflow(
config: Config,
project_id: str,
name: str,
description: str,
nodes: list[dict[str, Any]],
edges: list[dict[str, Any]],
*,
workflow_type: str = "batch",
kind: str = "",
schedule: dict[str, Any] | None = None,
scan_config: dict[str, Any] | None = None,
max_concurrent_runs: int | None = None,
node_timeout_seconds: int | None = None,
run_timeout_seconds: int | None = None,
max_retry_attempts: int | None = None,
retry_backoff_seconds: int | None = None,
) -> dict[str, Any]:
payload: dict[str, Any] = {
"name": name,
"description": description,
"type": workflow_type,
"nodes": nodes,
"edges": edges,
}
if kind.strip():
payload["kind"] = kind.strip()
if schedule is not None:
payload["schedule"] = schedule
if scan_config is not None:
payload["scanConfig"] = scan_config
if max_concurrent_runs is not None:
payload["maxConcurrentRuns"] = int(max_concurrent_runs)
if node_timeout_seconds is not None:
payload["nodeTimeoutSeconds"] = int(node_timeout_seconds)
if run_timeout_seconds is not None:
payload["runTimeoutSeconds"] = int(run_timeout_seconds)
if max_retry_attempts is not None:
payload["maxRetryAttempts"] = int(max_retry_attempts)
if retry_backoff_seconds is not None:
payload["retryBackoffSeconds"] = int(retry_backoff_seconds)
return request(
config,
"POST",
f"/api/projects/{project_id}/workflows",
payload,
expected=(201,),
)
def get_workflow(config: Config, workflow_id: str) -> dict[str, Any]:
return request(config, "GET", f"/api/workflows/{workflow_id}")
def update_workflow(config: Config, workflow_id: str, payload: dict[str, Any]) -> dict[str, Any]:
return request(config, "PUT", f"/api/workflows/{workflow_id}", payload, expected=(200,))
def clone_workflow(config: Config, project_id: str, workflow_id: str, name: str) -> dict[str, Any]:
workflow = get_workflow(config, workflow_id)
return create_workflow(
config,
project_id,
name,
str(workflow.get("description", "")).strip(),
workflow.get("nodes", []),
workflow.get("edges", []),
workflow_type=str(workflow.get("type", "batch") or "batch"),
kind=str(workflow.get("kind", "") or ""),
schedule=workflow.get("schedule"),
scan_config=workflow.get("scanConfig"),
max_concurrent_runs=workflow.get("maxConcurrentRuns"),
node_timeout_seconds=workflow.get("nodeTimeoutSeconds"),
run_timeout_seconds=workflow.get("runTimeoutSeconds"),
max_retry_attempts=workflow.get("maxRetryAttempts"),
retry_backoff_seconds=workflow.get("retryBackoffSeconds"),
)
def run_workflow(config: Config, workflow_id: str) -> dict[str, Any]:
return request(config, "POST", f"/api/workflows/{workflow_id}/run", {"trigger": "manual"}, expected=(201, 202))
def list_runs(config: Config, status: str = "") -> list[dict[str, Any]]:
query = f"?status={urllib.parse.quote(status)}" if status else ""
return request(config, "GET", f"/api/runs{query}")["items"]
def get_run(config: Config, run_id: str) -> dict[str, Any]:
return request(config, "GET", f"/api/runs/{run_id}")
def get_run_logs(config: Config, run_id: str) -> list[dict[str, Any]]:
return request(config, "GET", f"/api/runs/{run_id}/logs")["items"]
def wait_for_run(config: Config, run_id: str, timeout_seconds: int) -> dict[str, Any]:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
item = get_run(config, run_id)
if item["status"] in {"completed", "failed", "cancelled", "waiting_for_audit"}:
return item
time.sleep(5)
raise RegressionError(f"timeout waiting for run {run_id}")
def wait_for_next_workflow_run(
config: Config,
workflow_id: str,
previous_run_id: str,
timeout_seconds: int,
) -> dict[str, Any]:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
workflow = get_workflow(config, workflow_id)
current_run_id = str(workflow.get("lastRunId", "")).strip()
if current_run_id and current_run_id != previous_run_id:
return wait_for_run(config, current_run_id, timeout_seconds)
time.sleep(3)
raise RegressionError(f"timeout waiting for next scheduled run of workflow {workflow_id}")
def delete_projects(config: Config) -> None:
projects = request(config, "GET", "/api/projects")["items"]
for project in projects:
name = str(project.get("name", "")).strip()
if config.delete_all_projects:
request(config, "DELETE", f"/api/projects/{project['id']}", expected=(204,))
continue
if config.delete_regression_projects and name.startswith(config.project_prefix):
request(config, "DELETE", f"/api/projects/{project['id']}", expected=(204,))
def upload_edge(source: str, target: str, edge_id: str | None = None, source_handle: str = "out-1", target_handle: str = "in-1") -> dict[str, str]:
return {
"id": edge_id or f"{source}-{target}",
"source": source,
"sourceHandle": source_handle,
"target": target,
"targetHandle": target_handle,
}
def list_audit_tasks(config: Config, status: str = "all") -> list[dict[str, Any]]:
return request(config, "GET", f"/api/audit-tasks?status={urllib.parse.quote(status)}")["items"]
def wait_for_audit_task(config: Config, run_id: str, timeout_seconds: int) -> dict[str, Any]:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
tasks = list_audit_tasks(config, "all")
for task in tasks:
if task.get("runId") == run_id and task.get("status") == "pending":
return task
time.sleep(2)
raise RegressionError(f"timeout waiting for audit task of run {run_id}")
def resolve_audit_task(config: Config, task_id: str, decision: str) -> dict[str, Any]:
decision = decision.strip().lower()
if decision not in {"approve", "reject"}:
raise RegressionError(f"unsupported audit decision: {decision}")
return request(config, "POST", f"/api/audit-tasks/{task_id}/{decision}", expected=(200,))
def query_business_collection(config: Config, connection_id: str, collection: str, limit: int = 20) -> dict[str, Any]:
query = urllib.parse.urlencode(
{
"target": "business",
"connectionId": connection_id,
"limit": str(limit),
"skip": "0",
}
)
return request(config, "GET", f"/api/database/collections/{urllib.parse.quote(collection)}/documents?{query}")
def cap_duration_command(seconds: int, *, max_items: int | None = None, max_duration: float | None = None) -> str:
return f"""
python3 - <<'PY'
import json
import os
import pathlib
input_path = pathlib.Path(os.environ['INPUT_ARTIFACTS_PATH'])
output_dir = pathlib.Path(os.environ['OUTPUT_DIR'])
artifacts = json.loads(input_path.read_text(encoding='utf-8'))
def load_payload(path_value: str):
path = pathlib.Path(str(path_value or '').strip())
if not path_value or path.suffix.lower() != '.json' or not path.exists():
return None
try:
return json.loads(path.read_text(encoding='utf-8'))
except Exception:
return None
payload = None
for artifact in artifacts:
data = artifact.get('data')
if isinstance(data, dict) and isinstance(data.get('items'), list):
payload = json.loads(json.dumps(data, ensure_ascii=False))
break
if isinstance(data, dict):
data_ref = data.get('dataRef') if isinstance(data.get('dataRef'), dict) else {{}}
payload_path = str(data_ref.get('payloadPath', '')).strip()
loaded = load_payload(payload_path)
if isinstance(loaded, dict) and isinstance(loaded.get('items'), list):
payload = loaded
break
loaded = load_payload(str(artifact.get('path', '')).strip())
if isinstance(loaded, dict) and isinstance(loaded.get('items'), list):
payload = loaded
break
if payload is None:
payload = {{}}
payload = json.loads(json.dumps(payload, ensure_ascii=False))
items = payload.get('items')
if not isinstance(items, list):
items = []
payload['items'] = items
if {max_items if max_items is not None else 'None'} is not None:
items = items[:{max_items if max_items is not None else 0}]
payload['items'] = items
if {repr(max_duration if max_duration is not None else None)} is not None:
items = [item for item in items if float(item.get('duration') or 0.0) <= float({max_duration if max_duration is not None else 0.0})]
payload['items'] = items
for item in items:
if not isinstance(item, dict):
continue
duration = float(item.get('duration') or 0.0)
item['duration'] = min(duration, {seconds}) if duration > 0 else {seconds}
payload['datasetType'] = 'x-platform-windowed'
payload['windowSeconds'] = {seconds}
payload['count'] = len(items)
payload['itemsCount'] = len(items)
payload['videoCount'] = len(items)
(output_dir / 'windowed-videos.json').write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
PY
""".strip()
def model_inference_command() -> str:
return """
python3 - <<'PY'
import json
import os
import pathlib
inputs = json.loads(pathlib.Path(os.environ["INPUT_ARTIFACTS_PATH"]).read_text(encoding="utf-8"))
summary = {
"operator": "模型推理",
"inputArtifactCount": len(inputs),
"artifactNames": [str(item.get("name", "")) for item in inputs][:20],
}
pathlib.Path(os.environ["OUTPUT_DIR"]).joinpath("model-inference-summary.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2),
encoding="utf-8",
)
PY
""".strip()
def ffprobe_video_metadata(payload: bytes, filename: str) -> dict[str, Any]:
suffix = os.path.splitext(filename)[1] or ".mp4"
try:
with tempfile.NamedTemporaryFile(suffix=suffix) as handle:
handle.write(payload)
handle.flush()
probe = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=codec_name,width,height,avg_frame_rate:format=duration,bit_rate",
"-of",
"json",
handle.name,
],
check=True,
capture_output=True,
text=True,
)
data = json.loads(probe.stdout or "{}")
except (FileNotFoundError, subprocess.CalledProcessError, json.JSONDecodeError):
return {
"filename": filename,
"codec": "",
"resolution": "",
"frameRate": 0.0,
"bitrate": 0,
"duration": 0.0,
"startTime": int(time.time()),
}
streams = data.get("streams") or []
stream = streams[0] if streams else {}
fmt = data.get("format") or {}
frame_rate = 0.0
avg_frame_rate = str(stream.get("avg_frame_rate", "0/1")).strip()
if "/" in avg_frame_rate:
numerator, denominator = avg_frame_rate.split("/", 1)
try:
denominator_value = float(denominator or 0.0)
if denominator_value:
frame_rate = float(numerator or 0.0) / denominator_value
except ValueError:
frame_rate = 0.0
return {
"filename": filename,
"codec": stream.get("codec_name") or "",
"resolution": f"{int(stream.get('width') or 0)}x{int(stream.get('height') or 0)}" if stream.get("width") and stream.get("height") else "",
"frameRate": round(frame_rate, 3) if frame_rate else 0.0,
"bitrate": int(float(fmt.get("bit_rate") or 0)),
"duration": round(float(fmt.get("duration") or 0.0), 3),
"startTime": int(time.time()),
}
def ensure_incremental_seed_directories(
config: Config,
connection_id: str,
seed_prefix: str,
source_clip_keys: list[str],
*,
bucket: str,
round_name: str,
directory_count: int,
place: str,
camera_model: str,
video_subdir: str,
) -> list[str]:
created: list[str] = []
if directory_count <= 0:
return created
normalized_seed_prefix = seed_prefix.strip().strip("/")
normalized_video_subdir = video_subdir.strip().strip("/")
safe_camera = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in camera_model.strip()).strip("_")
if not normalized_video_subdir:
raise RegressionError("INCREMENTAL_VIDEO_SUBDIR must not be empty")
if not safe_camera:
raise RegressionError("INCREMENTAL_CAMERA_MODEL must contain at least one alphanumeric, hyphen, or underscore character")
for index in range(directory_count):
clip_key = source_clip_keys[index % len(source_clip_keys)]
clip_payload = object_storage_read(config, connection_id, clip_key, bucket=bucket)
clip_name = os.path.basename(clip_key)
dir_name = f"{round_name}_{index+1:02d}_{safe_camera}_{time.strftime('%Y-%m-%d_%H-%M-%S')}"
base_key = f"{normalized_seed_prefix}/{dir_name}"
video_key = f"{base_key}/{normalized_video_subdir}/{clip_name}"
object_storage_upload(config, connection_id, video_key, clip_payload, bucket=bucket, filename=clip_name, content_type="video/mp4")
video_meta = [ffprobe_video_metadata(clip_payload, clip_name)]
meta = {
"type": "工厂",
"place": place,
"mission": f"增量回归-{round_name}",
"project_id": dir_name,
"date": time.strftime("%Y%m%d"),
"device_id": f"REG{index+1:04d}",
}
intrinsics = {
"camera_model": camera_model,
"fx": 1.0,
"fy": 1.0,
"cx": 0.5,
"cy": 0.5,
}
object_storage_upload(
config,
connection_id,
f"{base_key}/meta.json",
json.dumps(meta, ensure_ascii=False, indent=2).encode("utf-8"),
bucket=bucket,
filename="meta.json",
content_type="application/json",
)
object_storage_upload(
config,
connection_id,
f"{base_key}/video_metadata.json",
json.dumps(video_meta, ensure_ascii=False, indent=2).encode("utf-8"),
bucket=bucket,
filename="video_metadata.json",
content_type="application/json",
)
object_storage_upload(
config,
connection_id,
f"{base_key}/intrinsics.json",
json.dumps(intrinsics, ensure_ascii=False, indent=2).encode("utf-8"),
bucket=bucket,
filename="intrinsics.json",
content_type="application/json",
)
created.append(dir_name)
return created
def list_uploaded_mp4_keys(config: Config, connection_id: str, output_root: str, suffix: str) -> list[str]:
bucket, root_prefix = parse_bos_path(output_root)
prefix = f"{root_prefix.strip('/')}/{suffix.strip('/')}".strip("/")
items = object_storage_list(config, connection_id, prefix, bucket=bucket)
return [str(item.get("key", "")).strip() for item in items if str(item.get("kind", "")).strip() == "video" and str(item.get("key", "")).strip().lower().endswith(".mp4")]
def create_runtime_acceptance(config: Config, connection_id: str, operators_by_name: dict[str, dict[str, Any]]) -> tuple[str, str]:
timestamp = time.strftime("%Y%m%d%H%M%S")
project = create_project(config, f"{config.project_prefix} acceptance {timestamp}", "Temporary production-style storage connection runtime acceptance test")
nodes = [
build_node(
operators_by_name["BOS 增量扫描"],
"source",
0,
0,
{"SOURCE_PATH": config.source_path, "SCAN_MODE": "FULL"},
connection_id=connection_id,
),
build_node(operators_by_name["X 平台数据集索引"], "index", 220, 0),
build_node(operators_by_name["采集目录校验"], "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
upload_node(
operators_by_name["结果上传"],
connection_id,
"upload",
660,
0,
f"{config.upload_root_prefix}/{timestamp}/runtime-acceptance-connid",
),
]
edges = [
{"id": "source-index", "source": "source", "sourceHandle": "out-1", "target": "index", "targetHandle": "in-1"},
{"id": "index-validate", "source": "index", "sourceHandle": "out-1", "target": "validate", "targetHandle": "in-1"},
{"id": "validate-upload", "source": "validate", "sourceHandle": "out-1", "target": "upload", "targetHandle": "in-1"},
]
workflow = create_workflow(config, project["id"], "connid-runtime-acceptance", "Validate storage connection runtime path", nodes, edges)
return project["id"], workflow["id"]
def create_regression_workflows(
config: Config,
connection_id: str,
mongo_connection_id: str,
operators_by_name: dict[str, dict[str, Any]],
) -> tuple[str, list[dict[str, Any]], str, str, str]:
timestamp = time.strftime("%Y%m%d%H%M%S")
output_root = f"{config.upload_root_prefix}/{timestamp}"
mongo_collection = f"{config.mongo_collection_prefix}_{timestamp}".replace("-", "_")
source_bucket, source_prefix = parse_bos_path(config.source_path)
incremental_prefix = f"{source_prefix}/__codex_incremental_regression__/{timestamp}".strip("/")
incremental_source_path = f"bos://{source_bucket}/{incremental_prefix}"
project = create_project(config, f"{config.project_prefix} {timestamp}", "使用真实 WJZH 数据做全算子批量回归")
source_operator = operators_by_name["BOS 增量扫描"]
real_source_operator = operators_by_name["真实数据源"]
directory_scan_operator = operators_by_name["目录扫描"]
model_operator = operators_by_name["模型推理"]
index_operator = operators_by_name["X 平台数据集索引"]
validate_operator = operators_by_name["采集目录校验"]
upload_operator = operators_by_name["结果上传"]
shell_operator = operators_by_name["Shell 命令执行"]
json_filter_operator = operators_by_name["JSON 过滤"]
def scan_source(node_id: str = "source", x: int = 0, y: int = 0) -> dict[str, Any]:
return build_node(
source_operator,
node_id,
x,
y,
{"SOURCE_PATH": config.source_path, "SCAN_MODE": "FULL"},
connection_id=connection_id,
)
def scan_incremental(node_id: str = "source", x: int = 0, y: int = 0) -> dict[str, Any]:
return build_node(
source_operator,
node_id,
x,
y,
{"SOURCE_PATH": incremental_source_path, "SCAN_MODE": "INCREMENTAL"},
connection_id=connection_id,
)
def real_source(node_id: str, x: int, y: int) -> dict[str, Any]:
return build_node(
real_source_operator,
node_id,
x,
y,
{"SOURCE_PATH": config.source_path, "DATA_SOURCE": "BOS"},
connection_id=connection_id,
)
def named_upload(node_id: str, x: int, y: int, suffix: str) -> dict[str, Any]:
return upload_node(upload_operator, connection_id, node_id, x, y, f"{output_root}/{suffix}")
workflows: list[dict[str, Any]] = []
wf0_nodes = [
real_source("source", 0, 0),
build_node(directory_scan_operator, "scan_dir", 240, -90),
build_node(model_operator, "infer", 240, 90, {"COMMAND": model_inference_command()}),
named_upload("dir_upload", 500, -90, "00-source-directory/scan"),
named_upload("infer_upload", 500, 90, "00-source-directory/infer"),
]
wf0_edges = [
upload_edge("source", "scan_dir", "source-dir"),
upload_edge("source", "infer", "source-infer"),
upload_edge("scan_dir", "dir_upload", "dir-upload"),
upload_edge("infer", "infer_upload", "infer-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "00-源数据目录推理", "真实数据源、目录扫描和模型推理回归", wf0_nodes, wf0_edges),
}
)
wf1_nodes = [
scan_source(),
build_node(index_operator, "index", 220, -70),
build_node(validate_operator, "validate", 440, -70, {"REQUIRE_INTRINSICS": "false"}),
named_upload("index_upload", 680, -140, "01-index-validate/index"),
named_upload("validate_upload", 680, 0, "01-index-validate/validate"),
]
wf1_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("index", "index_upload", "index-upload"),
upload_edge("validate", "validate_upload", "validate-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "01-索引校验双上传", "扫描全量真实数据并分别上传索引与校验结果", wf1_nodes, wf1_edges),
}
)
wf2_nodes = [
scan_source(),
build_node(index_operator, "index", 220, 0),
build_node(operators_by_name["JSON 字段提取"], "extract", 460, -180, {"FIELD_PATH": "collections.0.meta.place"}),
build_node(operators_by_name["文本模板渲染"], "render", 680, -180, {"TEMPLATE_TEXT": "place={{value}}", "OUTPUT_NAME": "place.txt"}),
build_node(json_filter_operator, "duration_filter", 460, -20, {"FIELD_PATH": "duration", "OPERATOR": "lte", "EXPECTED_JSON": "700"}),
build_node(
operators_by_name["JSON 映射重命名"],
"rename",
680,
-20,
{"FIELD_MAP_JSON": "{\"collectionId\":\"collection_id\",\"filename\":\"video_name\",\"duration\":\"video_duration\"}"},
),
build_node(operators_by_name["JSON 转 CSV"], "to_csv", 900, -20),
build_node(operators_by_name["CSV 转 JSON"], "to_json", 1120, -20),
build_node(operators_by_name["条件分流"], "route", 460, 150, {"FIELD_PATH": "meta.device_id", "OPERATOR": "eq", "EXPECTED": config.route_device_id}),
build_node(operators_by_name["数据合并"], "merge", 1340, 40),
named_upload("render_upload", 1560, -180, "02-json-toolchain/render"),
named_upload("table_upload", 1560, -20, "02-json-toolchain/table"),
named_upload("matched_upload", 900, 130, "02-json-toolchain/route-matched"),
named_upload("unmatched_upload", 900, 250, "02-json-toolchain/route-unmatched"),
named_upload("merge_upload", 1560, 120, "02-json-toolchain/merged"),
]
wf2_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "extract", "index-extract"),
upload_edge("extract", "render", "extract-render"),
upload_edge("render", "render_upload", "render-upload"),
upload_edge("index", "duration_filter", "index-duration-filter"),
upload_edge("duration_filter", "rename", "filter-rename"),
upload_edge("rename", "to_csv", "rename-csv"),
upload_edge("to_csv", "to_json", "csv-json"),
upload_edge("to_json", "table_upload", "table-upload"),
upload_edge("index", "route", "index-route"),
upload_edge("route", "matched_upload", "route-matched-upload", source_handle="matched"),
upload_edge("route", "unmatched_upload", "route-unmatched-upload", source_handle="unmatched"),
upload_edge("route", "merge", "route-merge", source_handle="matched", target_handle="in-1"),
upload_edge("to_json", "merge", "json-merge", target_handle="in-2"),
upload_edge("merge", "merge_upload", "merge-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "02-JSON工具链多分支", "真实清单数据上的 JSON/CSV/分流/合并回归", wf2_nodes, wf2_edges),
}
)
def audit_workflow(name: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
nodes = [
scan_source(),
build_node(index_operator, "index", 220, 0),
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
build_node(json_filter_operator, "pick_one", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": config.audit_video}),
build_node(operators_by_name["人工审核"], "review", 880, 0),
named_upload("approved_upload", 1120, -80, f"03-audit/{name}/approved"),
named_upload("rejected_upload", 1120, 80, f"03-audit/{name}/rejected"),
]
edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("validate", "pick_one", "validate-pick"),
upload_edge("pick_one", "review", "pick-review"),
upload_edge("review", "approved_upload", "review-approved", source_handle="approved"),
upload_edge("review", "rejected_upload", "review-rejected", source_handle="rejected"),
]
return nodes, edges
audit_approve_nodes, audit_approve_edges = audit_workflow("approve")
workflows.append(
{
"workflow": create_workflow(config, project["id"], "03-人工审核通过", "真实数据人工审核 approve 分支回归", audit_approve_nodes, audit_approve_edges),
"auditDecision": "approve",
}
)
audit_reject_nodes, audit_reject_edges = audit_workflow("reject")
workflows.append(
{
"workflow": create_workflow(config, project["id"], "04-人工审核拒绝", "真实数据人工审核 reject 分支回归", audit_reject_nodes, audit_reject_edges),
"auditDecision": "reject",
}
)
wf5_nodes = [
scan_source(),
build_node(index_operator, "index", 220, 0),
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
build_node(json_filter_operator, "pick_medium", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": config.medium_video}),
build_node(
operators_by_name["MongoDB 写入"],
"mongo_write",
880,
0,
{
"MONGO_CONNECTION_ID": mongo_connection_id,
"MONGO_COLLECTION": mongo_collection,
"MONGO_MODE": "insert_many",
"DOCUMENT_PATH": "items",
},
),
named_upload("mongo_upload", 1120, 0, "05-mongo-write"),
]
wf5_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("validate", "pick_medium", "validate-pick"),
upload_edge("pick_medium", "mongo_write", "pick-mongo"),
upload_edge("mongo_write", "mongo_upload", "mongo-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "05-Mongo写入", "真实视频清单写入业务 Mongo 并上传写入结果", wf5_nodes, wf5_edges),
"mongoCollection": mongo_collection,
}
)
wf6_nodes = [
scan_source(),
build_node(index_operator, "index", 220, 0),
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
build_node(json_filter_operator, "pick_medium", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": config.medium_video}),
build_node(operators_by_name["视频质检"], "quality", 880, -60, quality_hyperparams(config)),
build_node(
shell_operator,
"window_medium",
1100,
-60,
{"COMMAND": cap_duration_command(int_config(config.medium_window_seconds), max_items=1, max_duration=int_config(config.huge_window_seconds))},
label="中视频窗口采样",
),
build_node(operators_by_name["视频切片"], "slice", 1320, -60, segment_hyperparams(config.medium_segment_duration_seconds)),
named_upload("quality_upload", 1540, -140, "06-medium-video/quality"),
named_upload("slice_upload", 1540, 20, "06-medium-video/slice"),
]
wf6_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("validate", "pick_medium", "validate-pick"),
upload_edge("pick_medium", "quality", "pick-quality"),
upload_edge("quality", "window_medium", "quality-window"),
upload_edge("window_medium", "slice", "window-slice"),
upload_edge("quality", "quality_upload", "quality-upload"),
upload_edge("slice", "slice_upload", "slice-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "06-中大型视频切片", "真实中大型视频的质检和切片回归", wf6_nodes, wf6_edges),
}
)
wf7_nodes = [
scan_source(),
build_node(index_operator, "index", 220, 0),
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
build_node(json_filter_operator, "pick_huge", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": config.huge_video}),
build_node(operators_by_name["视频质检"], "quality", 880, 0, quality_hyperparams(config)),
build_node(
operators_by_name["数据过滤"],
"filter",
1100,
0,
filter_hyperparams(config),
),
build_node(shell_operator, "window", 1320, 0, {"COMMAND": cap_duration_command(int_config(config.huge_window_seconds))}, label="大文件窗口采样"),
build_node(operators_by_name["视频切片"], "slice", 1540, 0, segment_hyperparams(config.huge_segment_duration_seconds)),
build_node(operators_by_name["视频标注"], "annotate", 1760, 0, annotation_hyperparams(config)),
build_node(operators_by_name["成品打包"], "package", 1980, 0),
build_node(
operators_by_name["抽检采样"],
"sample",
2200,
0,
{"RULES_JSON": config.sampling_rules_json},
),
named_upload("quality_upload", 2460, -240, "07-full-chain/quality"),
named_upload("filter_upload", 2460, -160, "07-full-chain/filter"),
named_upload("window_upload", 2460, -80, "07-full-chain/window"),
named_upload("slice_upload", 2460, 0, "07-full-chain/slice"),
named_upload("annotate_upload", 2460, 80, "07-full-chain/annotate"),
named_upload("package_upload", 2460, 160, "07-full-chain/package"),
named_upload("sample_upload", 2460, 240, "07-full-chain/sample"),
]
wf7_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("validate", "pick_huge", "validate-pick"),
upload_edge("pick_huge", "quality", "pick-quality"),
upload_edge("quality", "filter", "quality-filter"),
upload_edge("filter", "window", "filter-window"),
upload_edge("window", "slice", "window-slice"),
upload_edge("slice", "annotate", "slice-annotate"),
upload_edge("annotate", "package", "annotate-package"),
upload_edge("package", "sample", "package-sample"),
upload_edge("quality", "quality_upload", "quality-upload"),
upload_edge("filter", "filter_upload", "filter-upload"),
upload_edge("window", "window_upload", "window-upload"),
upload_edge("slice", "slice_upload", "slice-upload"),
upload_edge("annotate", "annotate_upload", "annotate-upload"),
upload_edge("package", "package_upload", "package-upload"),
upload_edge("sample", "sample_upload", "sample-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "07-超大视频全链路", "显式指定真实大文件的完整处理链路回归", wf7_nodes, wf7_edges),
}
)
wf8_nodes = [
scan_source(),
build_node(index_operator, "index", 220, 0),
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
build_node(json_filter_operator, "pick_batch", 660, 0, {"FIELD_PATH": "duration", "OPERATOR": "lte", "EXPECTED_JSON": "400"}),
build_node(shell_operator, "window_batch", 880, 0, {"COMMAND": cap_duration_command(int_config(config.batch_window_seconds), max_items=4)}, label="批量窗口采样"),
build_node(operators_by_name["视频质检"], "quality", 1100, 0, quality_hyperparams(config)),
build_node(operators_by_name["视频切片"], "slice", 1320, 0, segment_hyperparams(config.batch_segment_duration_seconds)),
build_node(operators_by_name["视频标注"], "annotate", 1540, 0, annotation_hyperparams(config)),
build_node(operators_by_name["成品打包"], "package", 1760, 0),
named_upload("batch_package_upload", 1980, -60, "08-batch-windowed/package"),
named_upload("batch_annotation_upload", 1980, 60, "08-batch-windowed/annotate"),
]
wf8_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("validate", "pick_batch", "validate-pick"),
upload_edge("pick_batch", "window_batch", "pick-window"),
upload_edge("window_batch", "quality", "window-quality"),
upload_edge("quality", "slice", "quality-slice"),
upload_edge("slice", "annotate", "slice-annotate"),
upload_edge("annotate", "package", "annotate-package"),
upload_edge("annotate", "batch_annotation_upload", "annotate-upload"),
upload_edge("package", "batch_package_upload", "package-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "08-多目录批量窗口化", "跨目录批量视频窗口化、切片、标注和打包回归", wf8_nodes, wf8_edges),
}
)
wf9_nodes = [
build_node(operators_by_name["LeRobot 数据集概览"], "lerobot", 0, -80, {"SOURCE_PATH": config.lerobot_source_path}),
build_node(operators_by_name["ROSBag 数据概览"], "rosbag", 0, 80, {"SOURCE_PATH": config.rosbag_source_path}),
named_upload("lerobot_upload", 240, -80, "09-robotics/lerobot"),
named_upload("rosbag_upload", 240, 80, "09-robotics/rosbag"),
]
wf9_edges = [
upload_edge("lerobot", "lerobot_upload", "lerobot-upload"),
upload_edge("rosbag", "rosbag_upload", "rosbag-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "09-Robotics真实概览", "使用显式真实 LeRobot 和 ROSBag 路径回归概览算子", wf9_nodes, wf9_edges),
}
)
wf10_nodes = [
scan_source(),
build_node(index_operator, "index", 220, 0),
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
build_node(
shell_operator,
"batch_pick",
660,
0,
{"COMMAND": cap_duration_command(int_config(config.batch_quality_window_seconds), max_items=8, max_duration=500)},
label="多目录多视频抽样",
),
build_node(operators_by_name["视频质检"], "quality", 900, 0, quality_hyperparams(config)),
named_upload("batch_pick_upload", 1140, -60, "10-batch-quality/pick"),
named_upload("quality_upload", 1140, 60, "10-batch-quality/quality"),
]
wf10_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("validate", "batch_pick", "validate-batch-pick"),
upload_edge("batch_pick", "quality", "batch-pick-quality"),
upload_edge("batch_pick", "batch_pick_upload", "batch-pick-upload"),
upload_edge("quality", "quality_upload", "quality-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "10-多目录多视频批量质检", "跨目录抽样多个真实视频,批量跑视频质检并上传结果", wf10_nodes, wf10_edges),
"rounds": 2,
}
)
wf11_nodes = [
scan_source(),
build_node(index_operator, "index", 220, 0),
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
build_node(shell_operator, "batch_pick", 660, 0, {"COMMAND": cap_duration_command(int_config(config.batch_window_seconds), max_items=6, max_duration=500)}, label="多目录全链路抽样"),
build_node(operators_by_name["视频质检"], "quality", 900, -80, quality_hyperparams(config)),
build_node(operators_by_name["数据过滤"], "filter", 1120, -80, filter_hyperparams(config)),
build_node(operators_by_name["视频切片"], "slice", 1340, -80, segment_hyperparams(config.batch_segment_duration_seconds)),
build_node(operators_by_name["视频标注"], "annotate", 1560, -80, annotation_hyperparams(config)),
build_node(operators_by_name["成品打包"], "package", 1780, -80),
build_node(
operators_by_name["抽检采样"],
"sample",
2000,
-80,
{"RULES_JSON": config.sampling_rules_json},
),
named_upload("pick_upload", 2240, -220, "11-batch-fullchain/pick"),
named_upload("quality_upload", 2240, -140, "11-batch-fullchain/quality"),
named_upload("slice_upload", 2240, -60, "11-batch-fullchain/slice"),
named_upload("annotate_upload", 2240, 20, "11-batch-fullchain/annotate"),
named_upload("package_upload", 2240, 100, "11-batch-fullchain/package"),
named_upload("sample_upload", 2240, 180, "11-batch-fullchain/sample"),
]
wf11_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("validate", "batch_pick", "validate-batch-pick"),
upload_edge("batch_pick", "quality", "batch-pick-quality"),
upload_edge("quality", "filter", "quality-filter"),
upload_edge("filter", "slice", "filter-slice"),
upload_edge("slice", "annotate", "slice-annotate"),
upload_edge("annotate", "package", "annotate-package"),
upload_edge("package", "sample", "package-sample"),
upload_edge("batch_pick", "pick_upload", "pick-upload"),
upload_edge("quality", "quality_upload", "quality-upload"),
upload_edge("slice", "slice_upload", "slice-upload"),
upload_edge("annotate", "annotate_upload", "annotate-upload"),
upload_edge("package", "package_upload", "package-upload"),
upload_edge("sample", "sample_upload", "sample-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "11-多目录多视频批量全链路", "跨目录抽样多个真实视频,批量跑过滤/切片/标注/打包/抽检", wf11_nodes, wf11_edges),
"rounds": 2,
}
)
wf12_nodes = [
scan_incremental(),
build_node(index_operator, "index", 220, 0),
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
build_node(operators_by_name["视频质检"], "quality", 660, -80, quality_hyperparams(config)),
build_node(operators_by_name["数据过滤"], "filter", 880, -80, filter_hyperparams(config)),
build_node(
shell_operator,
"window",
1100,
-80,
{"COMMAND": cap_duration_command(int_config(config.incremental_window_seconds), max_items=8, max_duration=int_config(config.huge_window_seconds))},
label="增量视频窗口抽样",
),
build_node(operators_by_name["视频切片"], "slice", 1320, -80, segment_hyperparams(config.incremental_segment_duration_seconds)),
build_node(operators_by_name["视频标注"], "annotate", 1540, -80, annotation_hyperparams(config)),
build_node(operators_by_name["成品打包"], "package", 1760, -80),
build_node(
operators_by_name["抽检采样"],
"sample",
1980,
-80,
{"RULES_JSON": config.sampling_rules_json},
),
named_upload("quality_upload", 2220, -220, "12-incremental-loop/quality"),
named_upload("filter_upload", 2220, -140, "12-incremental-loop/filter"),
named_upload("slice_upload", 2220, -60, "12-incremental-loop/slice"),
named_upload("annotate_upload", 2220, 20, "12-incremental-loop/annotate"),
named_upload("package_upload", 2220, 100, "12-incremental-loop/package"),
named_upload("sample_upload", 2220, 180, "12-incremental-loop/sample"),
]
wf12_edges = [
upload_edge("source", "index", "source-index"),
upload_edge("index", "validate", "index-validate"),
upload_edge("validate", "quality", "validate-quality"),
upload_edge("quality", "filter", "quality-filter"),
upload_edge("filter", "window", "filter-window"),
upload_edge("window", "slice", "window-slice"),
upload_edge("slice", "annotate", "slice-annotate"),
upload_edge("annotate", "package", "annotate-package"),
upload_edge("package", "sample", "package-sample"),
upload_edge("quality", "quality_upload", "quality-upload"),
upload_edge("filter", "filter_upload", "filter-upload"),
upload_edge("slice", "slice_upload", "slice-upload"),
upload_edge("annotate", "annotate_upload", "annotate-upload"),
upload_edge("package", "package_upload", "package-upload"),
upload_edge("sample", "sample_upload", "sample-upload"),
]
workflows.append(
{
"workflow": create_workflow(config, project["id"], "12-增量嗅探循环处理", "在源前缀下持续嗅探新增目录,并对新增目录执行完整处理链", wf12_nodes, wf12_edges),
"incrementalRounds": [
{"name": "round-1", "directoryCount": 2},
{"name": "round-2", "directoryCount": 2},
{"name": "round-3-no-new", "directoryCount": 0},
],
"incrementalSourcePath": incremental_source_path,
"streamingIntervalSeconds": 20,
}
)
return project["id"], workflows, mongo_collection, output_root, incremental_source_path
def summarize_run(run: dict[str, Any]) -> dict[str, Any]:
result = {
"runId": run["id"],
"status": run["status"],
"workflowId": run["workflowId"],
"workflowName": run["workflowName"],
"currentNodeId": run.get("currentNodeId"),
"durationMs": run.get("durationMs"),
}
if run["status"] != "completed":
failed = next((item for item in run.get("nodeExecutions", []) if item.get("status") == "failed"), None)
if failed:
result["failedNode"] = {
"nodeId": failed.get("nodeId"),
"nodeLabel": failed.get("nodeLabel"),
"error": failed.get("error"),
}
result["logsTail"] = get_logs_tail(failed)
return result
def get_logs_tail(node_execution: dict[str, Any]) -> list[dict[str, Any]]:
logs = node_execution.get("logs")
if not isinstance(logs, list):
return []
return logs[-20:]
def run_workflow_until_complete(
config: Config,
workflow_id: str,
timeout_seconds: int,
*,
audit_decision: str = "",
) -> dict[str, Any]:
started = run_workflow(config, workflow_id)
if not audit_decision:
return wait_for_run(config, started["id"], timeout_seconds)
deadline = time.time() + timeout_seconds
while time.time() < deadline:
current = get_run(config, started["id"])
if current["status"] in {"completed", "failed", "cancelled"}:
return current
if current["status"] == "waiting_for_audit":
task = wait_for_audit_task(config, started["id"], min(300, max(int(deadline - time.time()), 5)))
resolve_audit_task(config, task["id"], audit_decision)
time.sleep(3)
raise RegressionError(f"timeout waiting for run {started['id']}")
def main() -> int:
config = parse_args()
validate_config(config)
if config.api_token:
print("using API token auth for WJZH regression requests")
login_if_configured(config)
settings = request(config, "GET", "/api/settings")
operators = request(config, "GET", "/api/operators")["items"]
connection = ensure_storage_connection(config)
mongo_connection = find_mongo_connection(config)
validate_preflight(config, connection, operators, settings, mongo_connection)
operators_by_name = {item["name"]: item for item in operators}
if config.preflight_only:
print(
json.dumps(
{
"status": "ok",
"mode": "preflight-only",
"connectionId": connection["id"],
"mongoConnectionId": mongo_connection["id"] if mongo_connection else "",
"sourcePath": config.source_path,
"uploadRootPrefix": config.upload_root_prefix,
},
ensure_ascii=False,
indent=2,
)
)
return 0
if not config.skip_runtime_acceptance:
acceptance_project_id, acceptance_workflow_id = create_runtime_acceptance(config, connection["id"], operators_by_name)
acceptance_run = run_workflow(config, acceptance_workflow_id)
acceptance_result = wait_for_run(config, acceptance_run["id"], min(config.max_wait_seconds, 1800))
if acceptance_result["status"] != "completed":
print(
json.dumps(
{
"status": "runtime-acceptance-failed",
"projectId": acceptance_project_id,
"workflowId": acceptance_workflow_id,
"run": summarize_run(acceptance_result),
"runLogsTail": get_run_logs(config, acceptance_result["id"])[-30:],
},
ensure_ascii=False,
indent=2,
),
file=sys.stderr,
)
return 2
delete_projects(config)
project_id, workflows, mongo_collection, output_root, incremental_source_path = create_regression_workflows(
config,
connection["id"],
mongo_connection["id"] if mongo_connection else "",
operators_by_name,
)
results = []
mongo_verification: dict[str, Any] | None = None
incremental_workflow: dict[str, Any] | None = None
for item in workflows:
if item.get("incrementalRounds"):
incremental_workflow = item
continue
workflow = item["workflow"]
for round_index in range(max(int(item.get("rounds", 1) or 1), 1)):
current_workflow = workflow
if round_index > 0:
current_workflow = clone_workflow(
config,
project_id,
workflow["id"],
f"{workflow['name']} 第{round_index + 1}轮",
)
finished = run_workflow_until_complete(
config,
current_workflow["id"],
config.max_wait_seconds,
audit_decision=str(item.get("auditDecision", "")).strip(),
)
summary = summarize_run(finished)
if item.get("auditDecision"):
summary["auditDecision"] = item["auditDecision"]
if int(item.get("rounds", 1) or 1) > 1:
summary["round"] = round_index + 1
results.append(summary)
if finished["status"] != "completed":
print(
json.dumps(
{
"status": "workflow-failed",
"projectId": project_id,
"workflowId": current_workflow["id"],
"workflowName": current_workflow["name"],
"run": summary,
"runLogsTail": get_run_logs(config, finished["id"])[-30:],
},
ensure_ascii=False,
indent=2,
),
file=sys.stderr,
)
return 3
if item.get("mongoCollection"):
documents = query_business_collection(config, mongo_connection["id"], str(item["mongoCollection"]), limit=50)
mongo_verification = {
"collection": item["mongoCollection"],
"documentCount": documents.get("total", 0),
"sampleDocumentIds": [doc.get("_id") for doc in documents.get("documents", [])[:10]],
}
if int(mongo_verification["documentCount"] or 0) <= 0:
raise RegressionError(f"mongo verification failed for collection {item['mongoCollection']}")
incremental_verification: dict[str, Any] | None = None
if incremental_workflow is not None:
seed_clip_keys: list[str] = []
for suffix in ["06-medium-video/slice", "07-full-chain/slice", "11-batch-fullchain/slice"]:
seed_clip_keys.extend(list_uploaded_mp4_keys(config, connection["id"], output_root, suffix))
deduped_seed_keys = list(dict.fromkeys([key for key in seed_clip_keys if key]))
if not deduped_seed_keys:
raise RegressionError(f"failed to find seed clips under {output_root}")
incremental_bucket, incremental_prefix = parse_bos_path(incremental_source_path)
round_summaries: list[dict[str, Any]] = []
incremental_workflow_id = incremental_workflow["workflow"]["id"]
previous_run_id = ""
for round_item in incremental_workflow.get("incrementalRounds", []):
created_directories = ensure_incremental_seed_directories(
config,
connection["id"],
incremental_prefix,
deduped_seed_keys,
bucket=incremental_bucket,
round_name=str(round_item.get("name", "round")).strip() or "round",
directory_count=int(round_item.get("directoryCount", 0) or 0),
place=config.incremental_place,
camera_model=config.incremental_camera_model,
video_subdir=config.incremental_video_subdir,
)
if round_summaries:
time.sleep(2)
if not round_summaries:
scan_bucket, scan_prefix = parse_bos_path(incremental_source_path)
update_payload: dict[str, Any] = {
"kind": "scan",
"maxConcurrentRuns": 1,
"incrementalUnit": "DIRECTORY",
"directoryDepth": 1,
"scanConfig": {
"enabled": True,
"connectionId": connection["id"],
"dataSource": "BOS",
"sourcePath": incremental_source_path,
"bucket": scan_bucket,
"prefix": scan_prefix,
"scanMode": "INCREMENTAL",
},
"type": "streaming",
"schedule": {
"interval": int(incremental_workflow.get("streamingIntervalSeconds", 20) or 20),
},
}
update_workflow(
config,
incremental_workflow_id,
update_payload,
)
finished = wait_for_next_workflow_run(
config,
incremental_workflow_id,
previous_run_id,
config.max_wait_seconds,
)
summary = summarize_run(finished)
summary["incrementalRound"] = round_item.get("name")
summary["createdDirectories"] = created_directories
round_summaries.append(summary)
results.append(summary)
previous_run_id = str(finished.get("id", "")).strip()
if finished["status"] != "completed":
print(
json.dumps(
{
"status": "incremental-workflow-failed",
"projectId": project_id,
"workflowId": incremental_workflow_id,
"workflowName": incremental_workflow["workflow"]["name"],
"run": summary,
"runLogsTail": get_run_logs(config, finished["id"])[-30:],
},
ensure_ascii=False,
indent=2,
),
file=sys.stderr,
)
return 4
incremental_verification = {
"sourcePath": incremental_source_path,
"seedClipKeys": deduped_seed_keys,
"rounds": round_summaries,
}
print(
json.dumps(
{
"status": "completed",
"projectId": project_id,
"connectionId": connection["id"],
"mongoConnectionId": mongo_connection["id"] if mongo_connection else "",
"mongoCollection": mongo_collection,
"mongoVerification": mongo_verification,
"incrementalVerification": incremental_verification,
"sourcePath": config.source_path,
"uploadRootPrefix": config.upload_root_prefix,
"workflows": results,
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except RegressionError as err:
print(str(err), file=sys.stderr)
raise SystemExit(1)