forked from longtao.wu/droboticflow
1862 lines
80 KiB
Python
Executable File
1862 lines
80 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import urllib.parse
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
if str(SCRIPT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
|
|
from lib.droboticflow_acceptance import DroboticflowClient
|
|
from lib.droboticflow_env import api_base_url_from_env
|
|
from lib.quality_thresholds import parse_positive_int, positive_float_issues, positive_int_issues, unit_interval_threshold_issues
|
|
from lib.sampling_rules import normalize_sampling_rules_json as normalize_rules_json
|
|
|
|
|
|
DEFAULT_PROJECT_PREFIX = "真实数据算子回归"
|
|
CLIENT: DroboticflowClient | None = None
|
|
|
|
|
|
class RegressionError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class Config:
|
|
api_base_url: str
|
|
api_token: str
|
|
smoke_email: str
|
|
smoke_password: str
|
|
source_path: str
|
|
upload_root_prefix: str
|
|
project_prefix: str
|
|
storage_connection_id: str
|
|
storage_connection_name: str
|
|
create_connection_if_missing: bool
|
|
mongo_connection_id: str
|
|
mongo_collection_prefix: str
|
|
bos_endpoint: str
|
|
bos_bucket: str
|
|
bos_access_key_id: str
|
|
bos_secret_access_key: str
|
|
delete_all_projects: bool
|
|
delete_regression_projects: bool
|
|
preflight_only: bool
|
|
skip_runtime_smoke: bool
|
|
max_wait_seconds: int
|
|
audit_video: str
|
|
medium_video: str
|
|
huge_video: str
|
|
huge_window_seconds: str
|
|
route_device_id: str
|
|
filter_date_start: str
|
|
filter_date_end: str
|
|
lerobot_source_path: str
|
|
rosbag_source_path: str
|
|
incremental_place: str
|
|
incremental_camera_model: str
|
|
incremental_video_subdir: str
|
|
sampling_rules_json: str
|
|
quality_min_clarity: str
|
|
quality_max_shake: str
|
|
quality_max_hand_occlusion: str
|
|
medium_window_seconds: str
|
|
medium_segment_duration_seconds: str
|
|
huge_segment_duration_seconds: str
|
|
batch_window_seconds: str
|
|
batch_quality_window_seconds: str
|
|
batch_segment_duration_seconds: str
|
|
incremental_window_seconds: str
|
|
incremental_segment_duration_seconds: str
|
|
annotation_frame_interval_seconds: str
|
|
|
|
|
|
def env_default(name: str, fallback: str = "") -> str:
|
|
return os.getenv(name, fallback).strip()
|
|
|
|
|
|
def parse_positive_int_value(name: str, value: str) -> int:
|
|
parsed, issue = parse_positive_int(name, value)
|
|
if issue:
|
|
raise RegressionError(issue)
|
|
if parsed is None:
|
|
raise RegressionError(f"{name} is required")
|
|
return parsed
|
|
|
|
|
|
def positive_int_arg(name: str):
|
|
def parse(value: str) -> int:
|
|
parsed, issue = parse_positive_int(name, value)
|
|
if issue:
|
|
raise argparse.ArgumentTypeError(issue)
|
|
if parsed is None:
|
|
raise argparse.ArgumentTypeError(f"{name} is required")
|
|
return parsed
|
|
|
|
return parse
|
|
|
|
|
|
def positive_int_env_default(name: str, fallback: int) -> int:
|
|
value = env_default(name)
|
|
if not value:
|
|
return fallback
|
|
return parse_positive_int_value(name, value)
|
|
|
|
|
|
def client_for_config(config: Config) -> DroboticflowClient:
|
|
global CLIENT
|
|
if CLIENT is None or CLIENT.api_base_url != config.api_base_url or CLIENT.api_token != config.api_token:
|
|
CLIENT = DroboticflowClient(
|
|
config.api_base_url,
|
|
api_token=config.api_token,
|
|
error_factory=RegressionError,
|
|
)
|
|
return CLIENT
|
|
|
|
|
|
def normalize_sampling_rules_json(value: str) -> str:
|
|
return normalize_rules_json(value, error_factory=RegressionError)
|
|
|
|
|
|
def quality_hyperparams(config: Config) -> dict[str, str]:
|
|
return {
|
|
"MIN_CLARITY": config.quality_min_clarity,
|
|
"MAX_SHAKE": config.quality_max_shake,
|
|
"MAX_HAND_OCCLUSION": config.quality_max_hand_occlusion,
|
|
}
|
|
|
|
|
|
def filter_hyperparams(config: Config) -> dict[str, str]:
|
|
return {
|
|
"DATE_START": config.filter_date_start,
|
|
"DATE_END": config.filter_date_end,
|
|
**quality_hyperparams(config),
|
|
}
|
|
|
|
|
|
def int_config(value: str) -> int:
|
|
return int(str(value).strip())
|
|
|
|
|
|
def segment_hyperparams(duration_seconds: str) -> dict[str, str]:
|
|
return {"SEGMENT_DURATION_SECONDS": str(duration_seconds).strip()}
|
|
|
|
|
|
def annotation_hyperparams(config: Config) -> dict[str, str]:
|
|
return {"FRAME_INTERVAL_SECONDS": config.annotation_frame_interval_seconds}
|
|
|
|
|
|
def validate_config(config: Config) -> None:
|
|
issues: list[str] = []
|
|
parsed_filter_start: time.struct_time | None = None
|
|
parsed_filter_end: time.struct_time | None = None
|
|
if bool(config.smoke_email) != bool(config.smoke_password):
|
|
issues.append("DROBOTICFLOW_SMOKE_EMAIL and DROBOTICFLOW_SMOKE_PASSWORD must be set together")
|
|
for label, value in (
|
|
("SOURCE_PATH", config.source_path),
|
|
("UPLOAD_ROOT_PREFIX", config.upload_root_prefix),
|
|
):
|
|
if not value:
|
|
issues.append(f"{label} is required")
|
|
if not config.storage_connection_id and not config.storage_connection_name:
|
|
issues.append("STORAGE_CONNECTION_ID or STORAGE_CONNECTION_NAME is required")
|
|
if config.create_connection_if_missing and not config.storage_connection_name:
|
|
issues.append("STORAGE_CONNECTION_NAME is required when CREATE_CONNECTION_IF_MISSING=true")
|
|
if not config.medium_video:
|
|
issues.append("MEDIUM_VIDEO is required")
|
|
if config.max_wait_seconds < 60:
|
|
issues.append("MAX_WAIT_SECONDS must be greater than or equal to 60")
|
|
if not config.audit_video:
|
|
issues.append("AUDIT_VIDEO or MEDIUM_VIDEO is required")
|
|
if not config.huge_video:
|
|
issues.append("HUGE_VIDEO is required")
|
|
for label, value in (
|
|
("ROUTE_DEVICE_ID", config.route_device_id),
|
|
("FILTER_DATE_START", config.filter_date_start),
|
|
("FILTER_DATE_END", config.filter_date_end),
|
|
("LEROBOT_SOURCE_PATH", config.lerobot_source_path),
|
|
("ROSBAG_SOURCE_PATH", config.rosbag_source_path),
|
|
("INCREMENTAL_PLACE", config.incremental_place),
|
|
("INCREMENTAL_CAMERA_MODEL", config.incremental_camera_model),
|
|
("INCREMENTAL_VIDEO_SUBDIR", config.incremental_video_subdir),
|
|
):
|
|
if not value:
|
|
issues.append(f"{label} is required")
|
|
for label, value in (
|
|
("FILTER_DATE_START", config.filter_date_start),
|
|
("FILTER_DATE_END", config.filter_date_end),
|
|
):
|
|
if not value:
|
|
continue
|
|
try:
|
|
parsed = time.strptime(value, "%Y-%m-%d")
|
|
except ValueError:
|
|
issues.append(f"{label} must use YYYY-MM-DD format")
|
|
continue
|
|
if label == "FILTER_DATE_START":
|
|
parsed_filter_start = parsed
|
|
else:
|
|
parsed_filter_end = parsed
|
|
if parsed_filter_start is not None and parsed_filter_end is not None and parsed_filter_start > parsed_filter_end:
|
|
issues.append("FILTER_DATE_START must be earlier than or equal to FILTER_DATE_END")
|
|
for label, value in (
|
|
("LEROBOT_SOURCE_PATH", config.lerobot_source_path),
|
|
("ROSBAG_SOURCE_PATH", config.rosbag_source_path),
|
|
):
|
|
if value.startswith("bos://"):
|
|
issues.append(f"{label} must be a filesystem path visible inside operator-runner Jobs, not a bos:// path")
|
|
if config.create_connection_if_missing and not config.bos_bucket:
|
|
issues.append("BOS_BUCKET is required when CREATE_CONNECTION_IF_MISSING=true")
|
|
try:
|
|
config.sampling_rules_json = normalize_sampling_rules_json(config.sampling_rules_json)
|
|
except RegressionError as err:
|
|
issues.append(str(err))
|
|
issues.extend(
|
|
unit_interval_threshold_issues(
|
|
(
|
|
("QUALITY_MIN_CLARITY", config.quality_min_clarity),
|
|
("QUALITY_MAX_SHAKE", config.quality_max_shake),
|
|
("QUALITY_MAX_HAND_OCCLUSION", config.quality_max_hand_occlusion),
|
|
)
|
|
)
|
|
)
|
|
issues.extend(
|
|
positive_int_issues(
|
|
(
|
|
("HUGE_WINDOW_SECONDS", config.huge_window_seconds),
|
|
("MEDIUM_WINDOW_SECONDS", config.medium_window_seconds),
|
|
("MEDIUM_SEGMENT_DURATION_SECONDS", config.medium_segment_duration_seconds),
|
|
("HUGE_SEGMENT_DURATION_SECONDS", config.huge_segment_duration_seconds),
|
|
("BATCH_WINDOW_SECONDS", config.batch_window_seconds),
|
|
("BATCH_QUALITY_WINDOW_SECONDS", config.batch_quality_window_seconds),
|
|
("BATCH_SEGMENT_DURATION_SECONDS", config.batch_segment_duration_seconds),
|
|
("INCREMENTAL_WINDOW_SECONDS", config.incremental_window_seconds),
|
|
("INCREMENTAL_SEGMENT_DURATION_SECONDS", config.incremental_segment_duration_seconds),
|
|
)
|
|
)
|
|
)
|
|
issues.extend(positive_float_issues((("ANNOTATION_FRAME_INTERVAL_SECONDS", config.annotation_frame_interval_seconds),)))
|
|
if issues:
|
|
raise RegressionError(
|
|
"invalid regression configuration:\n"
|
|
+ "\n".join(f"- {item}" for item in issues)
|
|
+ "\nSet the values explicitly in the environment or CLI arguments; the script has no repository defaults for live data paths."
|
|
)
|
|
|
|
|
|
def login_if_configured(config: Config) -> None:
|
|
client_for_config(config).login_if_configured(
|
|
config.smoke_email,
|
|
config.smoke_password,
|
|
label="WJZH regression user",
|
|
)
|
|
|
|
|
|
def parse_args() -> Config:
|
|
parser = argparse.ArgumentParser(description="Run production-style WJZH operator regression workflows")
|
|
parser.add_argument("--api-base-url", default=api_base_url_from_env(include_base_url=False))
|
|
parser.add_argument("--api-token", default=env_default("DROBOTICFLOW_API_TOKEN"))
|
|
parser.add_argument("--smoke-email", default=env_default("DROBOTICFLOW_SMOKE_EMAIL"))
|
|
parser.add_argument("--smoke-password", default=os.getenv("DROBOTICFLOW_SMOKE_PASSWORD", ""))
|
|
parser.add_argument("--source-path", default=env_default("SOURCE_PATH"))
|
|
parser.add_argument("--upload-root-prefix", default=env_default("UPLOAD_ROOT_PREFIX"))
|
|
parser.add_argument("--project-prefix", default=env_default("PROJECT_PREFIX", DEFAULT_PROJECT_PREFIX))
|
|
parser.add_argument("--storage-connection-id", default=env_default("STORAGE_CONNECTION_ID"))
|
|
parser.add_argument("--storage-connection-name", default=env_default("STORAGE_CONNECTION_NAME"))
|
|
parser.add_argument(
|
|
"--create-connection-if-missing",
|
|
action="store_true",
|
|
default=env_default("CREATE_CONNECTION_IF_MISSING").lower() in {"1", "true", "yes"},
|
|
)
|
|
parser.add_argument("--mongo-connection-id", default=env_default("MONGO_CONNECTION_ID"))
|
|
parser.add_argument("--mongo-collection-prefix", default=env_default("MONGO_COLLECTION_PREFIX", "wjzh_operator_regression"))
|
|
parser.add_argument("--bos-endpoint", default=env_default("BOS_ENDPOINT"))
|
|
parser.add_argument("--bos-bucket", default=env_default("BOS_BUCKET"))
|
|
parser.add_argument("--bos-access-key-id", default=env_default("BOS_ACCESS_KEY_ID"))
|
|
parser.add_argument("--bos-secret-access-key", default=env_default("BOS_SECRET_ACCESS_KEY"))
|
|
parser.add_argument("--delete-all-projects", action="store_true")
|
|
parser.add_argument("--delete-regression-projects", action="store_true")
|
|
parser.add_argument("--preflight-only", action="store_true")
|
|
parser.add_argument("--skip-runtime-smoke", action="store_true")
|
|
parser.add_argument(
|
|
"--max-wait-seconds",
|
|
type=positive_int_arg("MAX_WAIT_SECONDS"),
|
|
default=positive_int_env_default("MAX_WAIT_SECONDS", 5400),
|
|
)
|
|
parser.add_argument("--audit-video", default=env_default("AUDIT_VIDEO"))
|
|
parser.add_argument("--medium-video", default=env_default("MEDIUM_VIDEO"))
|
|
parser.add_argument("--huge-video", default=env_default("HUGE_VIDEO"))
|
|
parser.add_argument("--huge-window-seconds", default=env_default("HUGE_WINDOW_SECONDS"))
|
|
parser.add_argument("--route-device-id", default=env_default("ROUTE_DEVICE_ID"))
|
|
parser.add_argument("--filter-date-start", default=env_default("FILTER_DATE_START"))
|
|
parser.add_argument("--filter-date-end", default=env_default("FILTER_DATE_END"))
|
|
parser.add_argument("--lerobot-source-path", default=env_default("LEROBOT_SOURCE_PATH"))
|
|
parser.add_argument("--rosbag-source-path", default=env_default("ROSBAG_SOURCE_PATH"))
|
|
parser.add_argument("--incremental-place", default=env_default("INCREMENTAL_PLACE"))
|
|
parser.add_argument("--incremental-camera-model", default=env_default("INCREMENTAL_CAMERA_MODEL"))
|
|
parser.add_argument("--incremental-video-subdir", default=env_default("INCREMENTAL_VIDEO_SUBDIR"))
|
|
parser.add_argument("--sampling-rules-json", default=env_default("SAMPLING_RULES_JSON"))
|
|
parser.add_argument("--quality-min-clarity", default=env_default("QUALITY_MIN_CLARITY"))
|
|
parser.add_argument("--quality-max-shake", default=env_default("QUALITY_MAX_SHAKE"))
|
|
parser.add_argument("--quality-max-hand-occlusion", default=env_default("QUALITY_MAX_HAND_OCCLUSION"))
|
|
parser.add_argument("--medium-window-seconds", default=env_default("MEDIUM_WINDOW_SECONDS"))
|
|
parser.add_argument("--medium-segment-duration-seconds", default=env_default("MEDIUM_SEGMENT_DURATION_SECONDS"))
|
|
parser.add_argument("--huge-segment-duration-seconds", default=env_default("HUGE_SEGMENT_DURATION_SECONDS"))
|
|
parser.add_argument("--batch-window-seconds", default=env_default("BATCH_WINDOW_SECONDS"))
|
|
parser.add_argument("--batch-quality-window-seconds", default=env_default("BATCH_QUALITY_WINDOW_SECONDS"))
|
|
parser.add_argument("--batch-segment-duration-seconds", default=env_default("BATCH_SEGMENT_DURATION_SECONDS"))
|
|
parser.add_argument("--incremental-window-seconds", default=env_default("INCREMENTAL_WINDOW_SECONDS"))
|
|
parser.add_argument("--incremental-segment-duration-seconds", default=env_default("INCREMENTAL_SEGMENT_DURATION_SECONDS"))
|
|
parser.add_argument("--annotation-frame-interval-seconds", default=env_default("ANNOTATION_FRAME_INTERVAL_SECONDS"))
|
|
args = parser.parse_args()
|
|
return Config(
|
|
api_base_url=args.api_base_url.rstrip("/"),
|
|
api_token=args.api_token,
|
|
smoke_email=args.smoke_email,
|
|
smoke_password=args.smoke_password,
|
|
source_path=args.source_path,
|
|
upload_root_prefix=args.upload_root_prefix.rstrip("/"),
|
|
project_prefix=args.project_prefix,
|
|
storage_connection_id=args.storage_connection_id,
|
|
storage_connection_name=args.storage_connection_name,
|
|
create_connection_if_missing=bool(args.create_connection_if_missing),
|
|
mongo_connection_id=args.mongo_connection_id,
|
|
mongo_collection_prefix=args.mongo_collection_prefix.strip() or "wjzh_operator_regression",
|
|
bos_endpoint=args.bos_endpoint,
|
|
bos_bucket=args.bos_bucket,
|
|
bos_access_key_id=args.bos_access_key_id,
|
|
bos_secret_access_key=args.bos_secret_access_key,
|
|
delete_all_projects=bool(args.delete_all_projects),
|
|
delete_regression_projects=bool(args.delete_regression_projects),
|
|
preflight_only=bool(args.preflight_only),
|
|
skip_runtime_smoke=bool(args.skip_runtime_smoke),
|
|
max_wait_seconds=args.max_wait_seconds,
|
|
audit_video=args.audit_video or args.medium_video,
|
|
medium_video=args.medium_video,
|
|
huge_video=args.huge_video,
|
|
huge_window_seconds=args.huge_window_seconds,
|
|
route_device_id=args.route_device_id,
|
|
filter_date_start=args.filter_date_start,
|
|
filter_date_end=args.filter_date_end,
|
|
lerobot_source_path=args.lerobot_source_path,
|
|
rosbag_source_path=args.rosbag_source_path,
|
|
incremental_place=args.incremental_place,
|
|
incremental_camera_model=args.incremental_camera_model,
|
|
incremental_video_subdir=args.incremental_video_subdir,
|
|
sampling_rules_json=args.sampling_rules_json,
|
|
quality_min_clarity=args.quality_min_clarity,
|
|
quality_max_shake=args.quality_max_shake,
|
|
quality_max_hand_occlusion=args.quality_max_hand_occlusion,
|
|
medium_window_seconds=args.medium_window_seconds,
|
|
medium_segment_duration_seconds=args.medium_segment_duration_seconds,
|
|
huge_segment_duration_seconds=args.huge_segment_duration_seconds,
|
|
batch_window_seconds=args.batch_window_seconds,
|
|
batch_quality_window_seconds=args.batch_quality_window_seconds,
|
|
batch_segment_duration_seconds=args.batch_segment_duration_seconds,
|
|
incremental_window_seconds=args.incremental_window_seconds,
|
|
incremental_segment_duration_seconds=args.incremental_segment_duration_seconds,
|
|
annotation_frame_interval_seconds=args.annotation_frame_interval_seconds,
|
|
)
|
|
|
|
|
|
def request(config: Config, method: str, path: str, payload: Any | None = None, expected: tuple[int, ...] = (200, 201, 202, 204)) -> Any:
|
|
return client_for_config(config).request_json(method, path, payload, expected=expected)
|
|
|
|
|
|
def request_bytes(config: Config, method: str, path: str, body: bytes | None = None, headers: dict[str, str] | None = None, expected: tuple[int, ...] = (200,)) -> tuple[int, bytes, dict[str, str]]:
|
|
return client_for_config(config).request_bytes(
|
|
method,
|
|
path,
|
|
body=body,
|
|
headers=headers,
|
|
expected=expected,
|
|
)
|
|
|
|
|
|
def object_storage_list(config: Config, connection_id: str, prefix: str, *, bucket: str = "") -> list[dict[str, Any]]:
|
|
query = urllib.parse.urlencode(
|
|
{
|
|
"connectionId": connection_id,
|
|
"bucket": bucket,
|
|
"prefix": prefix,
|
|
}
|
|
)
|
|
payload = request(config, "GET", f"/api/object-storage/objects?{query}")
|
|
return payload.get("items", [])
|
|
|
|
|
|
def object_storage_read(config: Config, connection_id: str, key: str, *, bucket: str = "") -> bytes:
|
|
query = urllib.parse.urlencode(
|
|
{
|
|
"connectionId": connection_id,
|
|
"bucket": bucket,
|
|
"key": key,
|
|
}
|
|
)
|
|
_, payload, _ = request_bytes(config, "GET", f"/api/object-storage/content?{query}")
|
|
return payload
|
|
|
|
|
|
def object_storage_upload(config: Config, connection_id: str, key: str, payload: bytes, *, bucket: str = "", filename: str = "", content_type: str = "") -> Any:
|
|
boundary = f"----CodexBoundary{uuid.uuid4().hex}"
|
|
filename = filename or os.path.basename(key) or "upload.bin"
|
|
content_type = content_type or mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
|
body = (
|
|
f"--{boundary}\r\n"
|
|
f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'
|
|
f"Content-Type: {content_type}\r\n\r\n"
|
|
).encode("utf-8") + payload + f"\r\n--{boundary}--\r\n".encode("utf-8")
|
|
query = urllib.parse.urlencode(
|
|
{
|
|
"connectionId": connection_id,
|
|
"bucket": bucket,
|
|
"key": key,
|
|
}
|
|
)
|
|
_, raw, _ = request_bytes(
|
|
config,
|
|
"POST",
|
|
f"/api/object-storage/objects/upload?{query}",
|
|
body=body,
|
|
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
|
|
)
|
|
return json.loads(raw.decode("utf-8"))
|
|
|
|
|
|
def object_storage_delete_prefix(config: Config, connection_id: str, key: str, *, bucket: str = "") -> None:
|
|
query = urllib.parse.urlencode(
|
|
{
|
|
"connectionId": connection_id,
|
|
"bucket": bucket,
|
|
"key": key,
|
|
"recursive": "true",
|
|
}
|
|
)
|
|
request_bytes(config, "DELETE", f"/api/object-storage/objects?{query}", expected=(200,))
|
|
|
|
|
|
def parse_bos_path(path: str) -> tuple[str, str]:
|
|
raw = str(path or "").strip()
|
|
if not raw.startswith("bos://"):
|
|
raise RegressionError(f"expected bos:// path, got: {raw}")
|
|
bucket_and_key = raw[len("bos://") :]
|
|
bucket, _, key = bucket_and_key.partition("/")
|
|
return bucket.strip(), key.strip().strip("/")
|
|
|
|
|
|
def required_operators() -> list[str]:
|
|
return [
|
|
"真实数据源",
|
|
"BOS 增量扫描",
|
|
"目录扫描",
|
|
"模型推理",
|
|
"X 平台数据集索引",
|
|
"采集目录校验",
|
|
"JSON 字段提取",
|
|
"文本模板渲染",
|
|
"JSON 过滤",
|
|
"JSON 映射重命名",
|
|
"JSON 转 CSV",
|
|
"CSV 转 JSON",
|
|
"条件分流",
|
|
"数据合并",
|
|
"视频质检",
|
|
"数据过滤",
|
|
"视频切片",
|
|
"视频标注",
|
|
"成品打包",
|
|
"抽检采样",
|
|
"人工审核",
|
|
"MongoDB 写入",
|
|
"LeRobot 数据集概览",
|
|
"ROSBag 数据概览",
|
|
"结果上传",
|
|
"Shell 命令执行",
|
|
]
|
|
|
|
|
|
def find_storage_connection(config: Config) -> dict[str, Any] | None:
|
|
items = request(config, "GET", "/api/storage/connections")["items"]
|
|
if config.storage_connection_id:
|
|
return next((item for item in items if item.get("id") == config.storage_connection_id), None)
|
|
if config.storage_connection_name:
|
|
return next((item for item in items if str(item.get("name", "")).strip() == config.storage_connection_name), None)
|
|
return None
|
|
|
|
|
|
def list_mongo_connections(config: Config) -> list[dict[str, Any]]:
|
|
return request(config, "GET", "/api/mongo/connections")["items"]
|
|
|
|
|
|
def find_mongo_connection(config: Config) -> dict[str, Any] | None:
|
|
items = list_mongo_connections(config)
|
|
if config.mongo_connection_id:
|
|
return next((item for item in items if item.get("id") == config.mongo_connection_id), None)
|
|
return items[0] if items else None
|
|
|
|
|
|
def ensure_storage_connection(config: Config) -> dict[str, Any]:
|
|
existing = find_storage_connection(config)
|
|
if existing is not None:
|
|
return existing
|
|
if not config.create_connection_if_missing:
|
|
raise RegressionError(
|
|
"missing storage connection. Set STORAGE_CONNECTION_ID / STORAGE_CONNECTION_NAME, "
|
|
"or rerun with --create-connection-if-missing plus STORAGE_CONNECTION_NAME, BOS_BUCKET, "
|
|
"BOS_ACCESS_KEY_ID, and BOS_SECRET_ACCESS_KEY."
|
|
)
|
|
missing = []
|
|
if not config.bos_access_key_id:
|
|
missing.append("BOS_ACCESS_KEY_ID")
|
|
if not config.bos_secret_access_key:
|
|
missing.append("BOS_SECRET_ACCESS_KEY")
|
|
if missing:
|
|
raise RegressionError(f"cannot create storage connection; missing env: {', '.join(missing)}")
|
|
payload = {
|
|
"name": config.storage_connection_name,
|
|
"description": "Created by scripts/wjzh_operator_regression.py",
|
|
"provider": "BOS",
|
|
"endpoint": config.bos_endpoint,
|
|
"bucket": config.bos_bucket,
|
|
"accessKeyId": config.bos_access_key_id,
|
|
"secretAccessKey": config.bos_secret_access_key,
|
|
}
|
|
return request(config, "POST", "/api/storage/connections", payload, expected=(201,))
|
|
|
|
|
|
def settings_has_workspace(settings: dict[str, Any]) -> bool:
|
|
k8s = settings.get("k8s") if isinstance(settings, dict) else None
|
|
if not isinstance(k8s, dict):
|
|
return False
|
|
value = str(k8s.get("workspacePVCName", "") or "").strip()
|
|
return value != ""
|
|
|
|
|
|
def validate_preflight(
|
|
config: Config,
|
|
connection: dict[str, Any],
|
|
operators: list[dict[str, Any]],
|
|
settings: dict[str, Any],
|
|
mongo_connection: dict[str, Any] | None,
|
|
) -> None:
|
|
source_bucket, _ = parse_bos_path(config.source_path)
|
|
upload_bucket, _ = parse_bos_path(config.upload_root_prefix)
|
|
issues: list[str] = []
|
|
|
|
if not settings_has_workspace(settings):
|
|
issues.append(
|
|
"settings.k8s.workspacePVCName is missing. Deploy the current api/worker version and configure WORKSPACE_PVC_NAME / workspacePVCName first."
|
|
)
|
|
if "enabled" in (settings.get("k8s") or {}):
|
|
issues.append(
|
|
"live /api/settings still returns k8s.enabled, which indicates the cluster is likely running an older server build than this repository."
|
|
)
|
|
if str(connection.get("provider", "")).strip().upper() != "BOS":
|
|
issues.append("storage connection provider must be BOS")
|
|
connection_bucket = str(connection.get("bucket", "")).strip()
|
|
if connection_bucket and connection_bucket != source_bucket:
|
|
issues.append(
|
|
f"storage connection bucket mismatch: connection bucket={connection_bucket}, source bucket={source_bucket}"
|
|
)
|
|
if connection_bucket and connection_bucket != upload_bucket:
|
|
issues.append(
|
|
f"storage connection bucket mismatch: connection bucket={connection_bucket}, upload bucket={upload_bucket}"
|
|
)
|
|
if mongo_connection is None:
|
|
issues.append("no Mongo connection available for MongoDB 写入 real regression")
|
|
|
|
by_name = {item.get("name"): item for item in operators}
|
|
missing_ops = [name for name in required_operators() if name not in by_name]
|
|
if missing_ops:
|
|
issues.append(f"missing operators: {', '.join(missing_ops)}")
|
|
|
|
if issues:
|
|
message = {
|
|
"summary": "preflight failed",
|
|
"issues": issues,
|
|
"requiredConfiguration": [
|
|
"deploy current api and worker images from this repository",
|
|
"configure shared workspace PVC via WORKSPACE_PVC_NAME / settings.k8s.workspacePVCName",
|
|
f"ensure a BOS storage connection exists for source bucket {source_bucket} and upload bucket {upload_bucket}",
|
|
],
|
|
}
|
|
raise RegressionError(json.dumps(message, ensure_ascii=False, indent=2))
|
|
|
|
|
|
def build_node(operator: dict[str, Any], node_id: str, x: int, y: int, overrides: dict[str, str] | None = None, *, connection_id: str = "", label: str | None = None) -> dict[str, Any]:
|
|
hyperparams = [dict(item) for item in operator.get("hyperparams", [])]
|
|
overrides = overrides or {}
|
|
seen: set[str] = set()
|
|
for item in hyperparams:
|
|
key = str(item.get("key", "")).strip()
|
|
if key in overrides:
|
|
item["value"] = overrides[key]
|
|
seen.add(key)
|
|
for key, value in overrides.items():
|
|
if key in seen:
|
|
continue
|
|
hyperparams.append({"key": key, "value": value})
|
|
return {
|
|
"id": node_id,
|
|
"type": "operator",
|
|
"data": {
|
|
"label": label or operator["name"],
|
|
"category": operator["category"],
|
|
"description": operator["description"],
|
|
"inputs": operator.get("inputs", []),
|
|
"outputs": operator.get("outputs", []),
|
|
"image": operator.get("image", ""),
|
|
"skillDefinition": operator.get("skillDefinition", ""),
|
|
"hyperparams": hyperparams,
|
|
"connectionId": connection_id,
|
|
},
|
|
"position": {"x": x, "y": y},
|
|
}
|
|
|
|
|
|
def upload_node(operator: dict[str, Any], connection_id: str, node_id: str, x: int, y: int, output_path: str) -> dict[str, Any]:
|
|
return build_node(
|
|
operator,
|
|
node_id,
|
|
x,
|
|
y,
|
|
{
|
|
"DATA_SOURCE": "BOS",
|
|
"OUTPUT_PATH": output_path,
|
|
"SOURCE_PATH": output_path,
|
|
},
|
|
connection_id=connection_id,
|
|
)
|
|
|
|
|
|
def create_project(config: Config, name: str, description: str) -> dict[str, Any]:
|
|
return request(config, "POST", "/api/projects", {"name": name, "description": description}, expected=(201,))
|
|
|
|
|
|
def create_workflow(
|
|
config: Config,
|
|
project_id: str,
|
|
name: str,
|
|
description: str,
|
|
nodes: list[dict[str, Any]],
|
|
edges: list[dict[str, Any]],
|
|
*,
|
|
workflow_type: str = "batch",
|
|
kind: str = "",
|
|
schedule: dict[str, Any] | None = None,
|
|
scan_config: dict[str, Any] | None = None,
|
|
max_concurrent_runs: int | None = None,
|
|
node_timeout_seconds: int | None = None,
|
|
run_timeout_seconds: int | None = None,
|
|
max_retry_attempts: int | None = None,
|
|
retry_backoff_seconds: int | None = None,
|
|
) -> dict[str, Any]:
|
|
payload: dict[str, Any] = {
|
|
"name": name,
|
|
"description": description,
|
|
"type": workflow_type,
|
|
"nodes": nodes,
|
|
"edges": edges,
|
|
}
|
|
if kind.strip():
|
|
payload["kind"] = kind.strip()
|
|
if schedule is not None:
|
|
payload["schedule"] = schedule
|
|
if scan_config is not None:
|
|
payload["scanConfig"] = scan_config
|
|
if max_concurrent_runs is not None:
|
|
payload["maxConcurrentRuns"] = int(max_concurrent_runs)
|
|
if node_timeout_seconds is not None:
|
|
payload["nodeTimeoutSeconds"] = int(node_timeout_seconds)
|
|
if run_timeout_seconds is not None:
|
|
payload["runTimeoutSeconds"] = int(run_timeout_seconds)
|
|
if max_retry_attempts is not None:
|
|
payload["maxRetryAttempts"] = int(max_retry_attempts)
|
|
if retry_backoff_seconds is not None:
|
|
payload["retryBackoffSeconds"] = int(retry_backoff_seconds)
|
|
return request(
|
|
config,
|
|
"POST",
|
|
f"/api/projects/{project_id}/workflows",
|
|
payload,
|
|
expected=(201,),
|
|
)
|
|
|
|
|
|
def get_workflow(config: Config, workflow_id: str) -> dict[str, Any]:
|
|
return request(config, "GET", f"/api/workflows/{workflow_id}")
|
|
|
|
|
|
def update_workflow(config: Config, workflow_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
return request(config, "PUT", f"/api/workflows/{workflow_id}", payload, expected=(200,))
|
|
|
|
|
|
def clone_workflow(config: Config, project_id: str, workflow_id: str, name: str) -> dict[str, Any]:
|
|
workflow = get_workflow(config, workflow_id)
|
|
return create_workflow(
|
|
config,
|
|
project_id,
|
|
name,
|
|
str(workflow.get("description", "")).strip(),
|
|
workflow.get("nodes", []),
|
|
workflow.get("edges", []),
|
|
workflow_type=str(workflow.get("type", "batch") or "batch"),
|
|
kind=str(workflow.get("kind", "") or ""),
|
|
schedule=workflow.get("schedule"),
|
|
scan_config=workflow.get("scanConfig"),
|
|
max_concurrent_runs=workflow.get("maxConcurrentRuns"),
|
|
node_timeout_seconds=workflow.get("nodeTimeoutSeconds"),
|
|
run_timeout_seconds=workflow.get("runTimeoutSeconds"),
|
|
max_retry_attempts=workflow.get("maxRetryAttempts"),
|
|
retry_backoff_seconds=workflow.get("retryBackoffSeconds"),
|
|
)
|
|
|
|
|
|
def run_workflow(config: Config, workflow_id: str) -> dict[str, Any]:
|
|
return request(config, "POST", f"/api/workflows/{workflow_id}/run", {"trigger": "manual"}, expected=(201, 202))
|
|
|
|
|
|
def list_runs(config: Config, status: str = "") -> list[dict[str, Any]]:
|
|
query = f"?status={urllib.parse.quote(status)}" if status else ""
|
|
return request(config, "GET", f"/api/runs{query}")["items"]
|
|
|
|
|
|
def get_run(config: Config, run_id: str) -> dict[str, Any]:
|
|
return request(config, "GET", f"/api/runs/{run_id}")
|
|
|
|
|
|
def get_run_logs(config: Config, run_id: str) -> list[dict[str, Any]]:
|
|
return request(config, "GET", f"/api/runs/{run_id}/logs")["items"]
|
|
|
|
|
|
def wait_for_run(config: Config, run_id: str, timeout_seconds: int) -> dict[str, Any]:
|
|
deadline = time.time() + timeout_seconds
|
|
while time.time() < deadline:
|
|
item = get_run(config, run_id)
|
|
if item["status"] in {"completed", "failed", "cancelled", "waiting_for_audit"}:
|
|
return item
|
|
time.sleep(5)
|
|
raise RegressionError(f"timeout waiting for run {run_id}")
|
|
|
|
|
|
def wait_for_next_workflow_run(
|
|
config: Config,
|
|
workflow_id: str,
|
|
previous_run_id: str,
|
|
timeout_seconds: int,
|
|
) -> dict[str, Any]:
|
|
deadline = time.time() + timeout_seconds
|
|
while time.time() < deadline:
|
|
workflow = get_workflow(config, workflow_id)
|
|
current_run_id = str(workflow.get("lastRunId", "")).strip()
|
|
if current_run_id and current_run_id != previous_run_id:
|
|
return wait_for_run(config, current_run_id, timeout_seconds)
|
|
time.sleep(3)
|
|
raise RegressionError(f"timeout waiting for next scheduled run of workflow {workflow_id}")
|
|
|
|
|
|
def delete_projects(config: Config) -> None:
|
|
projects = request(config, "GET", "/api/projects")["items"]
|
|
for project in projects:
|
|
name = str(project.get("name", "")).strip()
|
|
if config.delete_all_projects:
|
|
request(config, "DELETE", f"/api/projects/{project['id']}", expected=(204,))
|
|
continue
|
|
if config.delete_regression_projects and name.startswith(config.project_prefix):
|
|
request(config, "DELETE", f"/api/projects/{project['id']}", expected=(204,))
|
|
|
|
|
|
def upload_edge(source: str, target: str, edge_id: str | None = None, source_handle: str = "out-1", target_handle: str = "in-1") -> dict[str, str]:
|
|
return {
|
|
"id": edge_id or f"{source}-{target}",
|
|
"source": source,
|
|
"sourceHandle": source_handle,
|
|
"target": target,
|
|
"targetHandle": target_handle,
|
|
}
|
|
|
|
|
|
def list_audit_tasks(config: Config, status: str = "all") -> list[dict[str, Any]]:
|
|
return request(config, "GET", f"/api/audit-tasks?status={urllib.parse.quote(status)}")["items"]
|
|
|
|
|
|
def wait_for_audit_task(config: Config, run_id: str, timeout_seconds: int) -> dict[str, Any]:
|
|
deadline = time.time() + timeout_seconds
|
|
while time.time() < deadline:
|
|
tasks = list_audit_tasks(config, "all")
|
|
for task in tasks:
|
|
if task.get("runId") == run_id and task.get("status") == "pending":
|
|
return task
|
|
time.sleep(2)
|
|
raise RegressionError(f"timeout waiting for audit task of run {run_id}")
|
|
|
|
|
|
def resolve_audit_task(config: Config, task_id: str, decision: str) -> dict[str, Any]:
|
|
decision = decision.strip().lower()
|
|
if decision not in {"approve", "reject"}:
|
|
raise RegressionError(f"unsupported audit decision: {decision}")
|
|
return request(config, "POST", f"/api/audit-tasks/{task_id}/{decision}", expected=(200,))
|
|
|
|
|
|
def query_business_collection(config: Config, connection_id: str, collection: str, limit: int = 20) -> dict[str, Any]:
|
|
query = urllib.parse.urlencode(
|
|
{
|
|
"target": "business",
|
|
"connectionId": connection_id,
|
|
"limit": str(limit),
|
|
"skip": "0",
|
|
}
|
|
)
|
|
return request(config, "GET", f"/api/database/collections/{urllib.parse.quote(collection)}/documents?{query}")
|
|
|
|
|
|
def cap_duration_command(seconds: int, *, max_items: int | None = None, max_duration: float | None = None) -> str:
|
|
return f"""
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import pathlib
|
|
|
|
input_path = pathlib.Path(os.environ['INPUT_ARTIFACTS_PATH'])
|
|
output_dir = pathlib.Path(os.environ['OUTPUT_DIR'])
|
|
artifacts = json.loads(input_path.read_text(encoding='utf-8'))
|
|
|
|
|
|
def load_payload(path_value: str):
|
|
path = pathlib.Path(str(path_value or '').strip())
|
|
if not path_value or path.suffix.lower() != '.json' or not path.exists():
|
|
return None
|
|
try:
|
|
return json.loads(path.read_text(encoding='utf-8'))
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
payload = None
|
|
for artifact in artifacts:
|
|
data = artifact.get('data')
|
|
if isinstance(data, dict) and isinstance(data.get('items'), list):
|
|
payload = json.loads(json.dumps(data, ensure_ascii=False))
|
|
break
|
|
if isinstance(data, dict):
|
|
data_ref = data.get('dataRef') if isinstance(data.get('dataRef'), dict) else {{}}
|
|
payload_path = str(data_ref.get('payloadPath', '')).strip()
|
|
loaded = load_payload(payload_path)
|
|
if isinstance(loaded, dict) and isinstance(loaded.get('items'), list):
|
|
payload = loaded
|
|
break
|
|
loaded = load_payload(str(artifact.get('path', '')).strip())
|
|
if isinstance(loaded, dict) and isinstance(loaded.get('items'), list):
|
|
payload = loaded
|
|
break
|
|
if payload is None:
|
|
payload = {{}}
|
|
payload = json.loads(json.dumps(payload, ensure_ascii=False))
|
|
items = payload.get('items')
|
|
if not isinstance(items, list):
|
|
items = []
|
|
payload['items'] = items
|
|
if {max_items if max_items is not None else 'None'} is not None:
|
|
items = items[:{max_items if max_items is not None else 0}]
|
|
payload['items'] = items
|
|
if {repr(max_duration if max_duration is not None else None)} is not None:
|
|
items = [item for item in items if float(item.get('duration') or 0.0) <= float({max_duration if max_duration is not None else 0.0})]
|
|
payload['items'] = items
|
|
for item in items:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
duration = float(item.get('duration') or 0.0)
|
|
item['duration'] = min(duration, {seconds}) if duration > 0 else {seconds}
|
|
payload['datasetType'] = 'x-platform-windowed'
|
|
payload['windowSeconds'] = {seconds}
|
|
payload['count'] = len(items)
|
|
payload['itemsCount'] = len(items)
|
|
payload['videoCount'] = len(items)
|
|
(output_dir / 'windowed-videos.json').write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
PY
|
|
""".strip()
|
|
|
|
|
|
def model_inference_command() -> str:
|
|
return """
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import pathlib
|
|
|
|
inputs = json.loads(pathlib.Path(os.environ["INPUT_ARTIFACTS_PATH"]).read_text(encoding="utf-8"))
|
|
summary = {
|
|
"operator": "模型推理",
|
|
"inputArtifactCount": len(inputs),
|
|
"artifactNames": [str(item.get("name", "")) for item in inputs][:20],
|
|
}
|
|
pathlib.Path(os.environ["OUTPUT_DIR"]).joinpath("model-inference-summary.json").write_text(
|
|
json.dumps(summary, ensure_ascii=False, indent=2),
|
|
encoding="utf-8",
|
|
)
|
|
PY
|
|
""".strip()
|
|
|
|
|
|
def ffprobe_video_metadata(payload: bytes, filename: str) -> dict[str, Any]:
|
|
suffix = os.path.splitext(filename)[1] or ".mp4"
|
|
try:
|
|
with tempfile.NamedTemporaryFile(suffix=suffix) as handle:
|
|
handle.write(payload)
|
|
handle.flush()
|
|
probe = subprocess.run(
|
|
[
|
|
"ffprobe",
|
|
"-v",
|
|
"error",
|
|
"-select_streams",
|
|
"v:0",
|
|
"-show_entries",
|
|
"stream=codec_name,width,height,avg_frame_rate:format=duration,bit_rate",
|
|
"-of",
|
|
"json",
|
|
handle.name,
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
data = json.loads(probe.stdout or "{}")
|
|
except (FileNotFoundError, subprocess.CalledProcessError, json.JSONDecodeError):
|
|
return {
|
|
"filename": filename,
|
|
"codec": "",
|
|
"resolution": "",
|
|
"frameRate": 0.0,
|
|
"bitrate": 0,
|
|
"duration": 0.0,
|
|
"startTime": int(time.time()),
|
|
}
|
|
streams = data.get("streams") or []
|
|
stream = streams[0] if streams else {}
|
|
fmt = data.get("format") or {}
|
|
frame_rate = 0.0
|
|
avg_frame_rate = str(stream.get("avg_frame_rate", "0/1")).strip()
|
|
if "/" in avg_frame_rate:
|
|
numerator, denominator = avg_frame_rate.split("/", 1)
|
|
try:
|
|
denominator_value = float(denominator or 0.0)
|
|
if denominator_value:
|
|
frame_rate = float(numerator or 0.0) / denominator_value
|
|
except ValueError:
|
|
frame_rate = 0.0
|
|
return {
|
|
"filename": filename,
|
|
"codec": stream.get("codec_name") or "",
|
|
"resolution": f"{int(stream.get('width') or 0)}x{int(stream.get('height') or 0)}" if stream.get("width") and stream.get("height") else "",
|
|
"frameRate": round(frame_rate, 3) if frame_rate else 0.0,
|
|
"bitrate": int(float(fmt.get("bit_rate") or 0)),
|
|
"duration": round(float(fmt.get("duration") or 0.0), 3),
|
|
"startTime": int(time.time()),
|
|
}
|
|
|
|
|
|
def ensure_incremental_seed_directories(
|
|
config: Config,
|
|
connection_id: str,
|
|
seed_prefix: str,
|
|
source_clip_keys: list[str],
|
|
*,
|
|
bucket: str,
|
|
round_name: str,
|
|
directory_count: int,
|
|
place: str,
|
|
camera_model: str,
|
|
video_subdir: str,
|
|
) -> list[str]:
|
|
created: list[str] = []
|
|
if directory_count <= 0:
|
|
return created
|
|
normalized_seed_prefix = seed_prefix.strip().strip("/")
|
|
normalized_video_subdir = video_subdir.strip().strip("/")
|
|
safe_camera = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in camera_model.strip()).strip("_")
|
|
if not normalized_video_subdir:
|
|
raise RegressionError("INCREMENTAL_VIDEO_SUBDIR must not be empty")
|
|
if not safe_camera:
|
|
raise RegressionError("INCREMENTAL_CAMERA_MODEL must contain at least one alphanumeric, hyphen, or underscore character")
|
|
for index in range(directory_count):
|
|
clip_key = source_clip_keys[index % len(source_clip_keys)]
|
|
clip_payload = object_storage_read(config, connection_id, clip_key, bucket=bucket)
|
|
clip_name = os.path.basename(clip_key)
|
|
dir_name = f"{round_name}_{index+1:02d}_{safe_camera}_{time.strftime('%Y-%m-%d_%H-%M-%S')}"
|
|
base_key = f"{normalized_seed_prefix}/{dir_name}"
|
|
video_key = f"{base_key}/{normalized_video_subdir}/{clip_name}"
|
|
object_storage_upload(config, connection_id, video_key, clip_payload, bucket=bucket, filename=clip_name, content_type="video/mp4")
|
|
video_meta = [ffprobe_video_metadata(clip_payload, clip_name)]
|
|
meta = {
|
|
"type": "工厂",
|
|
"place": place,
|
|
"mission": f"增量回归-{round_name}",
|
|
"project_id": dir_name,
|
|
"date": time.strftime("%Y%m%d"),
|
|
"device_id": f"REG{index+1:04d}",
|
|
}
|
|
intrinsics = {
|
|
"camera_model": camera_model,
|
|
"fx": 1.0,
|
|
"fy": 1.0,
|
|
"cx": 0.5,
|
|
"cy": 0.5,
|
|
}
|
|
object_storage_upload(
|
|
config,
|
|
connection_id,
|
|
f"{base_key}/meta.json",
|
|
json.dumps(meta, ensure_ascii=False, indent=2).encode("utf-8"),
|
|
bucket=bucket,
|
|
filename="meta.json",
|
|
content_type="application/json",
|
|
)
|
|
object_storage_upload(
|
|
config,
|
|
connection_id,
|
|
f"{base_key}/video_metadata.json",
|
|
json.dumps(video_meta, ensure_ascii=False, indent=2).encode("utf-8"),
|
|
bucket=bucket,
|
|
filename="video_metadata.json",
|
|
content_type="application/json",
|
|
)
|
|
object_storage_upload(
|
|
config,
|
|
connection_id,
|
|
f"{base_key}/intrinsics.json",
|
|
json.dumps(intrinsics, ensure_ascii=False, indent=2).encode("utf-8"),
|
|
bucket=bucket,
|
|
filename="intrinsics.json",
|
|
content_type="application/json",
|
|
)
|
|
created.append(dir_name)
|
|
return created
|
|
|
|
|
|
def list_uploaded_mp4_keys(config: Config, connection_id: str, output_root: str, suffix: str) -> list[str]:
|
|
bucket, root_prefix = parse_bos_path(output_root)
|
|
prefix = f"{root_prefix.strip('/')}/{suffix.strip('/')}".strip("/")
|
|
items = object_storage_list(config, connection_id, prefix, bucket=bucket)
|
|
return [str(item.get("key", "")).strip() for item in items if str(item.get("kind", "")).strip() == "video" and str(item.get("key", "")).strip().lower().endswith(".mp4")]
|
|
|
|
|
|
def create_runtime_smoke(config: Config, connection_id: str, operators_by_name: dict[str, dict[str, Any]]) -> tuple[str, str]:
|
|
timestamp = time.strftime("%Y%m%d%H%M%S")
|
|
project = create_project(config, f"{config.project_prefix} smoke {timestamp}", "Temporary production-style storage connection smoke test")
|
|
nodes = [
|
|
build_node(
|
|
operators_by_name["BOS 增量扫描"],
|
|
"source",
|
|
0,
|
|
0,
|
|
{"SOURCE_PATH": config.source_path, "SCAN_MODE": "FULL"},
|
|
connection_id=connection_id,
|
|
),
|
|
build_node(operators_by_name["X 平台数据集索引"], "index", 220, 0),
|
|
build_node(operators_by_name["采集目录校验"], "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
upload_node(
|
|
operators_by_name["结果上传"],
|
|
connection_id,
|
|
"upload",
|
|
660,
|
|
0,
|
|
f"{config.upload_root_prefix}/{timestamp}/smoke-connid",
|
|
),
|
|
]
|
|
edges = [
|
|
{"id": "source-index", "source": "source", "sourceHandle": "out-1", "target": "index", "targetHandle": "in-1"},
|
|
{"id": "index-validate", "source": "index", "sourceHandle": "out-1", "target": "validate", "targetHandle": "in-1"},
|
|
{"id": "validate-upload", "source": "validate", "sourceHandle": "out-1", "target": "upload", "targetHandle": "in-1"},
|
|
]
|
|
workflow = create_workflow(config, project["id"], "connid-smoke", "Validate storage connection runtime path", nodes, edges)
|
|
return project["id"], workflow["id"]
|
|
|
|
|
|
def create_regression_workflows(
|
|
config: Config,
|
|
connection_id: str,
|
|
mongo_connection_id: str,
|
|
operators_by_name: dict[str, dict[str, Any]],
|
|
) -> tuple[str, list[dict[str, Any]], str, str, str]:
|
|
timestamp = time.strftime("%Y%m%d%H%M%S")
|
|
output_root = f"{config.upload_root_prefix}/{timestamp}"
|
|
mongo_collection = f"{config.mongo_collection_prefix}_{timestamp}".replace("-", "_")
|
|
source_bucket, source_prefix = parse_bos_path(config.source_path)
|
|
incremental_prefix = f"{source_prefix}/__codex_incremental_regression__/{timestamp}".strip("/")
|
|
incremental_source_path = f"bos://{source_bucket}/{incremental_prefix}"
|
|
project = create_project(config, f"{config.project_prefix} {timestamp}", "使用真实 WJZH 数据做全算子批量回归")
|
|
|
|
source_operator = operators_by_name["BOS 增量扫描"]
|
|
real_source_operator = operators_by_name["真实数据源"]
|
|
directory_scan_operator = operators_by_name["目录扫描"]
|
|
model_operator = operators_by_name["模型推理"]
|
|
index_operator = operators_by_name["X 平台数据集索引"]
|
|
validate_operator = operators_by_name["采集目录校验"]
|
|
upload_operator = operators_by_name["结果上传"]
|
|
shell_operator = operators_by_name["Shell 命令执行"]
|
|
json_filter_operator = operators_by_name["JSON 过滤"]
|
|
|
|
def scan_source(node_id: str = "source", x: int = 0, y: int = 0) -> dict[str, Any]:
|
|
return build_node(
|
|
source_operator,
|
|
node_id,
|
|
x,
|
|
y,
|
|
{"SOURCE_PATH": config.source_path, "SCAN_MODE": "FULL"},
|
|
connection_id=connection_id,
|
|
)
|
|
|
|
def scan_incremental(node_id: str = "source", x: int = 0, y: int = 0) -> dict[str, Any]:
|
|
return build_node(
|
|
source_operator,
|
|
node_id,
|
|
x,
|
|
y,
|
|
{"SOURCE_PATH": incremental_source_path, "SCAN_MODE": "INCREMENTAL"},
|
|
connection_id=connection_id,
|
|
)
|
|
|
|
def real_source(node_id: str, x: int, y: int) -> dict[str, Any]:
|
|
return build_node(
|
|
real_source_operator,
|
|
node_id,
|
|
x,
|
|
y,
|
|
{"SOURCE_PATH": config.source_path, "DATA_SOURCE": "BOS"},
|
|
connection_id=connection_id,
|
|
)
|
|
|
|
def named_upload(node_id: str, x: int, y: int, suffix: str) -> dict[str, Any]:
|
|
return upload_node(upload_operator, connection_id, node_id, x, y, f"{output_root}/{suffix}")
|
|
|
|
workflows: list[dict[str, Any]] = []
|
|
|
|
wf0_nodes = [
|
|
real_source("source", 0, 0),
|
|
build_node(directory_scan_operator, "scan_dir", 240, -90),
|
|
build_node(model_operator, "infer", 240, 90, {"COMMAND": model_inference_command()}),
|
|
named_upload("dir_upload", 500, -90, "00-source-directory/scan"),
|
|
named_upload("infer_upload", 500, 90, "00-source-directory/infer"),
|
|
]
|
|
wf0_edges = [
|
|
upload_edge("source", "scan_dir", "source-dir"),
|
|
upload_edge("source", "infer", "source-infer"),
|
|
upload_edge("scan_dir", "dir_upload", "dir-upload"),
|
|
upload_edge("infer", "infer_upload", "infer-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "00-源数据目录推理", "真实数据源、目录扫描和模型推理回归", wf0_nodes, wf0_edges),
|
|
}
|
|
)
|
|
|
|
wf1_nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, -70),
|
|
build_node(validate_operator, "validate", 440, -70, {"REQUIRE_INTRINSICS": "false"}),
|
|
named_upload("index_upload", 680, -140, "01-index-validate/index"),
|
|
named_upload("validate_upload", 680, 0, "01-index-validate/validate"),
|
|
]
|
|
wf1_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("index", "index_upload", "index-upload"),
|
|
upload_edge("validate", "validate_upload", "validate-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "01-索引校验双上传", "扫描全量真实数据并分别上传索引与校验结果", wf1_nodes, wf1_edges),
|
|
}
|
|
)
|
|
|
|
wf2_nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(operators_by_name["JSON 字段提取"], "extract", 460, -180, {"FIELD_PATH": "collections.0.meta.place"}),
|
|
build_node(operators_by_name["文本模板渲染"], "render", 680, -180, {"TEMPLATE_TEXT": "place={{value}}", "OUTPUT_NAME": "place.txt"}),
|
|
build_node(json_filter_operator, "duration_filter", 460, -20, {"FIELD_PATH": "duration", "OPERATOR": "lte", "EXPECTED_JSON": "700"}),
|
|
build_node(
|
|
operators_by_name["JSON 映射重命名"],
|
|
"rename",
|
|
680,
|
|
-20,
|
|
{"FIELD_MAP_JSON": "{\"collectionId\":\"collection_id\",\"filename\":\"video_name\",\"duration\":\"video_duration\"}"},
|
|
),
|
|
build_node(operators_by_name["JSON 转 CSV"], "to_csv", 900, -20),
|
|
build_node(operators_by_name["CSV 转 JSON"], "to_json", 1120, -20),
|
|
build_node(operators_by_name["条件分流"], "route", 460, 150, {"FIELD_PATH": "meta.device_id", "OPERATOR": "eq", "EXPECTED": config.route_device_id}),
|
|
build_node(operators_by_name["数据合并"], "merge", 1340, 40),
|
|
named_upload("render_upload", 1560, -180, "02-json-toolchain/render"),
|
|
named_upload("table_upload", 1560, -20, "02-json-toolchain/table"),
|
|
named_upload("matched_upload", 900, 130, "02-json-toolchain/route-matched"),
|
|
named_upload("unmatched_upload", 900, 250, "02-json-toolchain/route-unmatched"),
|
|
named_upload("merge_upload", 1560, 120, "02-json-toolchain/merged"),
|
|
]
|
|
wf2_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "extract", "index-extract"),
|
|
upload_edge("extract", "render", "extract-render"),
|
|
upload_edge("render", "render_upload", "render-upload"),
|
|
upload_edge("index", "duration_filter", "index-duration-filter"),
|
|
upload_edge("duration_filter", "rename", "filter-rename"),
|
|
upload_edge("rename", "to_csv", "rename-csv"),
|
|
upload_edge("to_csv", "to_json", "csv-json"),
|
|
upload_edge("to_json", "table_upload", "table-upload"),
|
|
upload_edge("index", "route", "index-route"),
|
|
upload_edge("route", "matched_upload", "route-matched-upload", source_handle="matched"),
|
|
upload_edge("route", "unmatched_upload", "route-unmatched-upload", source_handle="unmatched"),
|
|
upload_edge("route", "merge", "route-merge", source_handle="matched", target_handle="in-1"),
|
|
upload_edge("to_json", "merge", "json-merge", target_handle="in-2"),
|
|
upload_edge("merge", "merge_upload", "merge-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "02-JSON工具链多分支", "真实清单数据上的 JSON/CSV/分流/合并回归", wf2_nodes, wf2_edges),
|
|
}
|
|
)
|
|
|
|
def audit_workflow(name: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
build_node(json_filter_operator, "pick_one", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": config.audit_video}),
|
|
build_node(operators_by_name["人工审核"], "review", 880, 0),
|
|
named_upload("approved_upload", 1120, -80, f"03-audit/{name}/approved"),
|
|
named_upload("rejected_upload", 1120, 80, f"03-audit/{name}/rejected"),
|
|
]
|
|
edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("validate", "pick_one", "validate-pick"),
|
|
upload_edge("pick_one", "review", "pick-review"),
|
|
upload_edge("review", "approved_upload", "review-approved", source_handle="approved"),
|
|
upload_edge("review", "rejected_upload", "review-rejected", source_handle="rejected"),
|
|
]
|
|
return nodes, edges
|
|
|
|
audit_approve_nodes, audit_approve_edges = audit_workflow("approve")
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "03-人工审核通过", "真实数据人工审核 approve 分支回归", audit_approve_nodes, audit_approve_edges),
|
|
"auditDecision": "approve",
|
|
}
|
|
)
|
|
audit_reject_nodes, audit_reject_edges = audit_workflow("reject")
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "04-人工审核拒绝", "真实数据人工审核 reject 分支回归", audit_reject_nodes, audit_reject_edges),
|
|
"auditDecision": "reject",
|
|
}
|
|
)
|
|
|
|
wf5_nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
build_node(json_filter_operator, "pick_medium", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": config.medium_video}),
|
|
build_node(
|
|
operators_by_name["MongoDB 写入"],
|
|
"mongo_write",
|
|
880,
|
|
0,
|
|
{
|
|
"MONGO_CONNECTION_ID": mongo_connection_id,
|
|
"MONGO_COLLECTION": mongo_collection,
|
|
"MONGO_MODE": "insert_many",
|
|
"DOCUMENT_PATH": "items",
|
|
},
|
|
),
|
|
named_upload("mongo_upload", 1120, 0, "05-mongo-write"),
|
|
]
|
|
wf5_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("validate", "pick_medium", "validate-pick"),
|
|
upload_edge("pick_medium", "mongo_write", "pick-mongo"),
|
|
upload_edge("mongo_write", "mongo_upload", "mongo-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "05-Mongo写入", "真实视频清单写入业务 Mongo 并上传写入结果", wf5_nodes, wf5_edges),
|
|
"mongoCollection": mongo_collection,
|
|
}
|
|
)
|
|
|
|
wf6_nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
build_node(json_filter_operator, "pick_medium", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": config.medium_video}),
|
|
build_node(operators_by_name["视频质检"], "quality", 880, -60, quality_hyperparams(config)),
|
|
build_node(
|
|
shell_operator,
|
|
"window_medium",
|
|
1100,
|
|
-60,
|
|
{"COMMAND": cap_duration_command(int_config(config.medium_window_seconds), max_items=1, max_duration=int_config(config.huge_window_seconds))},
|
|
label="中视频窗口采样",
|
|
),
|
|
build_node(operators_by_name["视频切片"], "slice", 1320, -60, segment_hyperparams(config.medium_segment_duration_seconds)),
|
|
named_upload("quality_upload", 1540, -140, "06-medium-video/quality"),
|
|
named_upload("slice_upload", 1540, 20, "06-medium-video/slice"),
|
|
]
|
|
wf6_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("validate", "pick_medium", "validate-pick"),
|
|
upload_edge("pick_medium", "quality", "pick-quality"),
|
|
upload_edge("quality", "window_medium", "quality-window"),
|
|
upload_edge("window_medium", "slice", "window-slice"),
|
|
upload_edge("quality", "quality_upload", "quality-upload"),
|
|
upload_edge("slice", "slice_upload", "slice-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "06-中大型视频切片", "真实中大型视频的质检和切片回归", wf6_nodes, wf6_edges),
|
|
}
|
|
)
|
|
|
|
wf7_nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
build_node(json_filter_operator, "pick_huge", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": config.huge_video}),
|
|
build_node(operators_by_name["视频质检"], "quality", 880, 0, quality_hyperparams(config)),
|
|
build_node(
|
|
operators_by_name["数据过滤"],
|
|
"filter",
|
|
1100,
|
|
0,
|
|
filter_hyperparams(config),
|
|
),
|
|
build_node(shell_operator, "window", 1320, 0, {"COMMAND": cap_duration_command(int_config(config.huge_window_seconds))}, label="大文件窗口采样"),
|
|
build_node(operators_by_name["视频切片"], "slice", 1540, 0, segment_hyperparams(config.huge_segment_duration_seconds)),
|
|
build_node(operators_by_name["视频标注"], "annotate", 1760, 0, annotation_hyperparams(config)),
|
|
build_node(operators_by_name["成品打包"], "package", 1980, 0),
|
|
build_node(
|
|
operators_by_name["抽检采样"],
|
|
"sample",
|
|
2200,
|
|
0,
|
|
{"RULES_JSON": config.sampling_rules_json},
|
|
),
|
|
named_upload("quality_upload", 2460, -240, "07-full-chain/quality"),
|
|
named_upload("filter_upload", 2460, -160, "07-full-chain/filter"),
|
|
named_upload("window_upload", 2460, -80, "07-full-chain/window"),
|
|
named_upload("slice_upload", 2460, 0, "07-full-chain/slice"),
|
|
named_upload("annotate_upload", 2460, 80, "07-full-chain/annotate"),
|
|
named_upload("package_upload", 2460, 160, "07-full-chain/package"),
|
|
named_upload("sample_upload", 2460, 240, "07-full-chain/sample"),
|
|
]
|
|
wf7_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("validate", "pick_huge", "validate-pick"),
|
|
upload_edge("pick_huge", "quality", "pick-quality"),
|
|
upload_edge("quality", "filter", "quality-filter"),
|
|
upload_edge("filter", "window", "filter-window"),
|
|
upload_edge("window", "slice", "window-slice"),
|
|
upload_edge("slice", "annotate", "slice-annotate"),
|
|
upload_edge("annotate", "package", "annotate-package"),
|
|
upload_edge("package", "sample", "package-sample"),
|
|
upload_edge("quality", "quality_upload", "quality-upload"),
|
|
upload_edge("filter", "filter_upload", "filter-upload"),
|
|
upload_edge("window", "window_upload", "window-upload"),
|
|
upload_edge("slice", "slice_upload", "slice-upload"),
|
|
upload_edge("annotate", "annotate_upload", "annotate-upload"),
|
|
upload_edge("package", "package_upload", "package-upload"),
|
|
upload_edge("sample", "sample_upload", "sample-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "07-超大视频全链路", "显式指定真实大文件的完整处理链路回归", wf7_nodes, wf7_edges),
|
|
}
|
|
)
|
|
|
|
wf8_nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
build_node(json_filter_operator, "pick_batch", 660, 0, {"FIELD_PATH": "duration", "OPERATOR": "lte", "EXPECTED_JSON": "400"}),
|
|
build_node(shell_operator, "window_batch", 880, 0, {"COMMAND": cap_duration_command(int_config(config.batch_window_seconds), max_items=4)}, label="批量窗口采样"),
|
|
build_node(operators_by_name["视频质检"], "quality", 1100, 0, quality_hyperparams(config)),
|
|
build_node(operators_by_name["视频切片"], "slice", 1320, 0, segment_hyperparams(config.batch_segment_duration_seconds)),
|
|
build_node(operators_by_name["视频标注"], "annotate", 1540, 0, annotation_hyperparams(config)),
|
|
build_node(operators_by_name["成品打包"], "package", 1760, 0),
|
|
named_upload("batch_package_upload", 1980, -60, "08-batch-windowed/package"),
|
|
named_upload("batch_annotation_upload", 1980, 60, "08-batch-windowed/annotate"),
|
|
]
|
|
wf8_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("validate", "pick_batch", "validate-pick"),
|
|
upload_edge("pick_batch", "window_batch", "pick-window"),
|
|
upload_edge("window_batch", "quality", "window-quality"),
|
|
upload_edge("quality", "slice", "quality-slice"),
|
|
upload_edge("slice", "annotate", "slice-annotate"),
|
|
upload_edge("annotate", "package", "annotate-package"),
|
|
upload_edge("annotate", "batch_annotation_upload", "annotate-upload"),
|
|
upload_edge("package", "batch_package_upload", "package-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "08-多目录批量窗口化", "跨目录批量视频窗口化、切片、标注和打包回归", wf8_nodes, wf8_edges),
|
|
}
|
|
)
|
|
|
|
wf9_nodes = [
|
|
build_node(operators_by_name["LeRobot 数据集概览"], "lerobot", 0, -80, {"SOURCE_PATH": config.lerobot_source_path}),
|
|
build_node(operators_by_name["ROSBag 数据概览"], "rosbag", 0, 80, {"SOURCE_PATH": config.rosbag_source_path}),
|
|
named_upload("lerobot_upload", 240, -80, "09-robotics/lerobot"),
|
|
named_upload("rosbag_upload", 240, 80, "09-robotics/rosbag"),
|
|
]
|
|
wf9_edges = [
|
|
upload_edge("lerobot", "lerobot_upload", "lerobot-upload"),
|
|
upload_edge("rosbag", "rosbag_upload", "rosbag-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "09-Robotics真实概览", "使用显式真实 LeRobot 和 ROSBag 路径回归概览算子", wf9_nodes, wf9_edges),
|
|
}
|
|
)
|
|
|
|
wf10_nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
build_node(
|
|
shell_operator,
|
|
"batch_pick",
|
|
660,
|
|
0,
|
|
{"COMMAND": cap_duration_command(int_config(config.batch_quality_window_seconds), max_items=8, max_duration=500)},
|
|
label="多目录多视频抽样",
|
|
),
|
|
build_node(operators_by_name["视频质检"], "quality", 900, 0, quality_hyperparams(config)),
|
|
named_upload("batch_pick_upload", 1140, -60, "10-batch-quality/pick"),
|
|
named_upload("quality_upload", 1140, 60, "10-batch-quality/quality"),
|
|
]
|
|
wf10_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("validate", "batch_pick", "validate-batch-pick"),
|
|
upload_edge("batch_pick", "quality", "batch-pick-quality"),
|
|
upload_edge("batch_pick", "batch_pick_upload", "batch-pick-upload"),
|
|
upload_edge("quality", "quality_upload", "quality-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "10-多目录多视频批量质检", "跨目录抽样多个真实视频,批量跑视频质检并上传结果", wf10_nodes, wf10_edges),
|
|
"rounds": 2,
|
|
}
|
|
)
|
|
|
|
wf11_nodes = [
|
|
scan_source(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
build_node(shell_operator, "batch_pick", 660, 0, {"COMMAND": cap_duration_command(int_config(config.batch_window_seconds), max_items=6, max_duration=500)}, label="多目录全链路抽样"),
|
|
build_node(operators_by_name["视频质检"], "quality", 900, -80, quality_hyperparams(config)),
|
|
build_node(operators_by_name["数据过滤"], "filter", 1120, -80, filter_hyperparams(config)),
|
|
build_node(operators_by_name["视频切片"], "slice", 1340, -80, segment_hyperparams(config.batch_segment_duration_seconds)),
|
|
build_node(operators_by_name["视频标注"], "annotate", 1560, -80, annotation_hyperparams(config)),
|
|
build_node(operators_by_name["成品打包"], "package", 1780, -80),
|
|
build_node(
|
|
operators_by_name["抽检采样"],
|
|
"sample",
|
|
2000,
|
|
-80,
|
|
{"RULES_JSON": config.sampling_rules_json},
|
|
),
|
|
named_upload("pick_upload", 2240, -220, "11-batch-fullchain/pick"),
|
|
named_upload("quality_upload", 2240, -140, "11-batch-fullchain/quality"),
|
|
named_upload("slice_upload", 2240, -60, "11-batch-fullchain/slice"),
|
|
named_upload("annotate_upload", 2240, 20, "11-batch-fullchain/annotate"),
|
|
named_upload("package_upload", 2240, 100, "11-batch-fullchain/package"),
|
|
named_upload("sample_upload", 2240, 180, "11-batch-fullchain/sample"),
|
|
]
|
|
wf11_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("validate", "batch_pick", "validate-batch-pick"),
|
|
upload_edge("batch_pick", "quality", "batch-pick-quality"),
|
|
upload_edge("quality", "filter", "quality-filter"),
|
|
upload_edge("filter", "slice", "filter-slice"),
|
|
upload_edge("slice", "annotate", "slice-annotate"),
|
|
upload_edge("annotate", "package", "annotate-package"),
|
|
upload_edge("package", "sample", "package-sample"),
|
|
upload_edge("batch_pick", "pick_upload", "pick-upload"),
|
|
upload_edge("quality", "quality_upload", "quality-upload"),
|
|
upload_edge("slice", "slice_upload", "slice-upload"),
|
|
upload_edge("annotate", "annotate_upload", "annotate-upload"),
|
|
upload_edge("package", "package_upload", "package-upload"),
|
|
upload_edge("sample", "sample_upload", "sample-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "11-多目录多视频批量全链路", "跨目录抽样多个真实视频,批量跑过滤/切片/标注/打包/抽检", wf11_nodes, wf11_edges),
|
|
"rounds": 2,
|
|
}
|
|
)
|
|
|
|
wf12_nodes = [
|
|
scan_incremental(),
|
|
build_node(index_operator, "index", 220, 0),
|
|
build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}),
|
|
build_node(operators_by_name["视频质检"], "quality", 660, -80, quality_hyperparams(config)),
|
|
build_node(operators_by_name["数据过滤"], "filter", 880, -80, filter_hyperparams(config)),
|
|
build_node(
|
|
shell_operator,
|
|
"window",
|
|
1100,
|
|
-80,
|
|
{"COMMAND": cap_duration_command(int_config(config.incremental_window_seconds), max_items=8, max_duration=int_config(config.huge_window_seconds))},
|
|
label="增量视频窗口抽样",
|
|
),
|
|
build_node(operators_by_name["视频切片"], "slice", 1320, -80, segment_hyperparams(config.incremental_segment_duration_seconds)),
|
|
build_node(operators_by_name["视频标注"], "annotate", 1540, -80, annotation_hyperparams(config)),
|
|
build_node(operators_by_name["成品打包"], "package", 1760, -80),
|
|
build_node(
|
|
operators_by_name["抽检采样"],
|
|
"sample",
|
|
1980,
|
|
-80,
|
|
{"RULES_JSON": config.sampling_rules_json},
|
|
),
|
|
named_upload("quality_upload", 2220, -220, "12-incremental-loop/quality"),
|
|
named_upload("filter_upload", 2220, -140, "12-incremental-loop/filter"),
|
|
named_upload("slice_upload", 2220, -60, "12-incremental-loop/slice"),
|
|
named_upload("annotate_upload", 2220, 20, "12-incremental-loop/annotate"),
|
|
named_upload("package_upload", 2220, 100, "12-incremental-loop/package"),
|
|
named_upload("sample_upload", 2220, 180, "12-incremental-loop/sample"),
|
|
]
|
|
wf12_edges = [
|
|
upload_edge("source", "index", "source-index"),
|
|
upload_edge("index", "validate", "index-validate"),
|
|
upload_edge("validate", "quality", "validate-quality"),
|
|
upload_edge("quality", "filter", "quality-filter"),
|
|
upload_edge("filter", "window", "filter-window"),
|
|
upload_edge("window", "slice", "window-slice"),
|
|
upload_edge("slice", "annotate", "slice-annotate"),
|
|
upload_edge("annotate", "package", "annotate-package"),
|
|
upload_edge("package", "sample", "package-sample"),
|
|
upload_edge("quality", "quality_upload", "quality-upload"),
|
|
upload_edge("filter", "filter_upload", "filter-upload"),
|
|
upload_edge("slice", "slice_upload", "slice-upload"),
|
|
upload_edge("annotate", "annotate_upload", "annotate-upload"),
|
|
upload_edge("package", "package_upload", "package-upload"),
|
|
upload_edge("sample", "sample_upload", "sample-upload"),
|
|
]
|
|
workflows.append(
|
|
{
|
|
"workflow": create_workflow(config, project["id"], "12-增量嗅探循环处理", "在源前缀下持续嗅探新增目录,并对新增目录执行完整处理链", wf12_nodes, wf12_edges),
|
|
"incrementalRounds": [
|
|
{"name": "round-1", "directoryCount": 2},
|
|
{"name": "round-2", "directoryCount": 2},
|
|
{"name": "round-3-no-new", "directoryCount": 0},
|
|
],
|
|
"incrementalSourcePath": incremental_source_path,
|
|
"streamingIntervalSeconds": 20,
|
|
}
|
|
)
|
|
|
|
return project["id"], workflows, mongo_collection, output_root, incremental_source_path
|
|
|
|
|
|
def summarize_run(run: dict[str, Any]) -> dict[str, Any]:
|
|
result = {
|
|
"runId": run["id"],
|
|
"status": run["status"],
|
|
"workflowId": run["workflowId"],
|
|
"workflowName": run["workflowName"],
|
|
"currentNodeId": run.get("currentNodeId"),
|
|
"durationMs": run.get("durationMs"),
|
|
}
|
|
if run["status"] != "completed":
|
|
failed = next((item for item in run.get("nodeExecutions", []) if item.get("status") == "failed"), None)
|
|
if failed:
|
|
result["failedNode"] = {
|
|
"nodeId": failed.get("nodeId"),
|
|
"nodeLabel": failed.get("nodeLabel"),
|
|
"error": failed.get("error"),
|
|
}
|
|
result["logsTail"] = get_logs_tail(failed)
|
|
return result
|
|
|
|
|
|
def get_logs_tail(node_execution: dict[str, Any]) -> list[dict[str, Any]]:
|
|
logs = node_execution.get("logs")
|
|
if not isinstance(logs, list):
|
|
return []
|
|
return logs[-20:]
|
|
|
|
|
|
def run_workflow_until_complete(
|
|
config: Config,
|
|
workflow_id: str,
|
|
timeout_seconds: int,
|
|
*,
|
|
audit_decision: str = "",
|
|
) -> dict[str, Any]:
|
|
started = run_workflow(config, workflow_id)
|
|
if not audit_decision:
|
|
return wait_for_run(config, started["id"], timeout_seconds)
|
|
|
|
deadline = time.time() + timeout_seconds
|
|
while time.time() < deadline:
|
|
current = get_run(config, started["id"])
|
|
if current["status"] in {"completed", "failed", "cancelled"}:
|
|
return current
|
|
if current["status"] == "waiting_for_audit":
|
|
task = wait_for_audit_task(config, started["id"], min(300, max(int(deadline - time.time()), 5)))
|
|
resolve_audit_task(config, task["id"], audit_decision)
|
|
time.sleep(3)
|
|
raise RegressionError(f"timeout waiting for run {started['id']}")
|
|
|
|
|
|
def main() -> int:
|
|
config = parse_args()
|
|
validate_config(config)
|
|
if config.api_token:
|
|
print("using API token auth for WJZH regression requests")
|
|
login_if_configured(config)
|
|
settings = request(config, "GET", "/api/settings")
|
|
operators = request(config, "GET", "/api/operators")["items"]
|
|
connection = ensure_storage_connection(config)
|
|
mongo_connection = find_mongo_connection(config)
|
|
validate_preflight(config, connection, operators, settings, mongo_connection)
|
|
operators_by_name = {item["name"]: item for item in operators}
|
|
|
|
if config.preflight_only:
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "ok",
|
|
"mode": "preflight-only",
|
|
"connectionId": connection["id"],
|
|
"mongoConnectionId": mongo_connection["id"] if mongo_connection else "",
|
|
"sourcePath": config.source_path,
|
|
"uploadRootPrefix": config.upload_root_prefix,
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
if not config.skip_runtime_smoke:
|
|
smoke_project_id, smoke_workflow_id = create_runtime_smoke(config, connection["id"], operators_by_name)
|
|
smoke_run = run_workflow(config, smoke_workflow_id)
|
|
smoke_result = wait_for_run(config, smoke_run["id"], min(config.max_wait_seconds, 1800))
|
|
if smoke_result["status"] != "completed":
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "runtime-smoke-failed",
|
|
"projectId": smoke_project_id,
|
|
"workflowId": smoke_workflow_id,
|
|
"run": summarize_run(smoke_result),
|
|
"runLogsTail": get_run_logs(config, smoke_result["id"])[-30:],
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
),
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
delete_projects(config)
|
|
project_id, workflows, mongo_collection, output_root, incremental_source_path = create_regression_workflows(
|
|
config,
|
|
connection["id"],
|
|
mongo_connection["id"] if mongo_connection else "",
|
|
operators_by_name,
|
|
)
|
|
|
|
results = []
|
|
mongo_verification: dict[str, Any] | None = None
|
|
incremental_workflow: dict[str, Any] | None = None
|
|
for item in workflows:
|
|
if item.get("incrementalRounds"):
|
|
incremental_workflow = item
|
|
continue
|
|
workflow = item["workflow"]
|
|
for round_index in range(max(int(item.get("rounds", 1) or 1), 1)):
|
|
current_workflow = workflow
|
|
if round_index > 0:
|
|
current_workflow = clone_workflow(
|
|
config,
|
|
project_id,
|
|
workflow["id"],
|
|
f"{workflow['name']} 第{round_index + 1}轮",
|
|
)
|
|
finished = run_workflow_until_complete(
|
|
config,
|
|
current_workflow["id"],
|
|
config.max_wait_seconds,
|
|
audit_decision=str(item.get("auditDecision", "")).strip(),
|
|
)
|
|
summary = summarize_run(finished)
|
|
if item.get("auditDecision"):
|
|
summary["auditDecision"] = item["auditDecision"]
|
|
if int(item.get("rounds", 1) or 1) > 1:
|
|
summary["round"] = round_index + 1
|
|
results.append(summary)
|
|
if finished["status"] != "completed":
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "workflow-failed",
|
|
"projectId": project_id,
|
|
"workflowId": current_workflow["id"],
|
|
"workflowName": current_workflow["name"],
|
|
"run": summary,
|
|
"runLogsTail": get_run_logs(config, finished["id"])[-30:],
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
),
|
|
file=sys.stderr,
|
|
)
|
|
return 3
|
|
if item.get("mongoCollection"):
|
|
documents = query_business_collection(config, mongo_connection["id"], str(item["mongoCollection"]), limit=50)
|
|
mongo_verification = {
|
|
"collection": item["mongoCollection"],
|
|
"documentCount": documents.get("total", 0),
|
|
"sampleDocumentIds": [doc.get("_id") for doc in documents.get("documents", [])[:10]],
|
|
}
|
|
if int(mongo_verification["documentCount"] or 0) <= 0:
|
|
raise RegressionError(f"mongo verification failed for collection {item['mongoCollection']}")
|
|
|
|
incremental_verification: dict[str, Any] | None = None
|
|
if incremental_workflow is not None:
|
|
seed_clip_keys: list[str] = []
|
|
for suffix in ["06-medium-video/slice", "07-full-chain/slice", "11-batch-fullchain/slice"]:
|
|
seed_clip_keys.extend(list_uploaded_mp4_keys(config, connection["id"], output_root, suffix))
|
|
deduped_seed_keys = list(dict.fromkeys([key for key in seed_clip_keys if key]))
|
|
if not deduped_seed_keys:
|
|
raise RegressionError(f"failed to find seed clips under {output_root}")
|
|
|
|
incremental_bucket, incremental_prefix = parse_bos_path(incremental_source_path)
|
|
round_summaries: list[dict[str, Any]] = []
|
|
incremental_workflow_id = incremental_workflow["workflow"]["id"]
|
|
previous_run_id = ""
|
|
for round_item in incremental_workflow.get("incrementalRounds", []):
|
|
created_directories = ensure_incremental_seed_directories(
|
|
config,
|
|
connection["id"],
|
|
incremental_prefix,
|
|
deduped_seed_keys,
|
|
bucket=incremental_bucket,
|
|
round_name=str(round_item.get("name", "round")).strip() or "round",
|
|
directory_count=int(round_item.get("directoryCount", 0) or 0),
|
|
place=config.incremental_place,
|
|
camera_model=config.incremental_camera_model,
|
|
video_subdir=config.incremental_video_subdir,
|
|
)
|
|
if round_summaries:
|
|
time.sleep(2)
|
|
if not round_summaries:
|
|
scan_bucket, scan_prefix = parse_bos_path(incremental_source_path)
|
|
update_payload: dict[str, Any] = {
|
|
"kind": "scan",
|
|
"maxConcurrentRuns": 1,
|
|
"incrementalUnit": "DIRECTORY",
|
|
"directoryDepth": 1,
|
|
"scanConfig": {
|
|
"enabled": True,
|
|
"connectionId": connection["id"],
|
|
"dataSource": "BOS",
|
|
"sourcePath": incremental_source_path,
|
|
"bucket": scan_bucket,
|
|
"prefix": scan_prefix,
|
|
"scanMode": "INCREMENTAL",
|
|
},
|
|
"type": "streaming",
|
|
"schedule": {
|
|
"interval": int(incremental_workflow.get("streamingIntervalSeconds", 20) or 20),
|
|
},
|
|
}
|
|
update_workflow(
|
|
config,
|
|
incremental_workflow_id,
|
|
update_payload,
|
|
)
|
|
finished = wait_for_next_workflow_run(
|
|
config,
|
|
incremental_workflow_id,
|
|
previous_run_id,
|
|
config.max_wait_seconds,
|
|
)
|
|
summary = summarize_run(finished)
|
|
summary["incrementalRound"] = round_item.get("name")
|
|
summary["createdDirectories"] = created_directories
|
|
round_summaries.append(summary)
|
|
results.append(summary)
|
|
previous_run_id = str(finished.get("id", "")).strip()
|
|
if finished["status"] != "completed":
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "incremental-workflow-failed",
|
|
"projectId": project_id,
|
|
"workflowId": incremental_workflow_id,
|
|
"workflowName": incremental_workflow["workflow"]["name"],
|
|
"run": summary,
|
|
"runLogsTail": get_run_logs(config, finished["id"])[-30:],
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
),
|
|
file=sys.stderr,
|
|
)
|
|
return 4
|
|
incremental_verification = {
|
|
"sourcePath": incremental_source_path,
|
|
"seedClipKeys": deduped_seed_keys,
|
|
"rounds": round_summaries,
|
|
}
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "completed",
|
|
"projectId": project_id,
|
|
"connectionId": connection["id"],
|
|
"mongoConnectionId": mongo_connection["id"] if mongo_connection else "",
|
|
"mongoCollection": mongo_collection,
|
|
"mongoVerification": mongo_verification,
|
|
"incrementalVerification": incremental_verification,
|
|
"sourcePath": config.source_path,
|
|
"uploadRootPrefix": config.upload_root_prefix,
|
|
"workflows": results,
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
raise SystemExit(main())
|
|
except RegressionError as err:
|
|
print(str(err), file=sys.stderr)
|
|
raise SystemExit(1)
|