686 lines
27 KiB
Python
686 lines
27 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
if str(SCRIPT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
|
|
from lib.droboticflow_acceptance import DroboticflowClient
|
|
from lib.droboticflow_env import api_base_url_from_env
|
|
from lib.quality_thresholds import parse_positive_int, positive_float_issues, positive_int_issues, unit_interval_threshold_issues
|
|
from lib.sampling_rules import normalize_sampling_rules_json as normalize_rules_json
|
|
|
|
|
|
DEFAULT_PROJECT_PREFIX = "XPlatform Real Acceptance"
|
|
CLIENT: DroboticflowClient | None = None
|
|
|
|
|
|
class AcceptanceError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class Config:
|
|
api_base_url: str
|
|
api_token: str
|
|
acceptance_email: str
|
|
acceptance_password: str
|
|
source_path: str
|
|
storage_connection_id: str
|
|
storage_connection_name: str
|
|
storage_connection_bucket: str
|
|
upload_output_path: str
|
|
project_prefix: str
|
|
min_clarity: str
|
|
max_shake: str
|
|
max_hand_occlusion: str
|
|
clarity_scale: str
|
|
date_start: str
|
|
date_end: str
|
|
place: str
|
|
dataset_type: str
|
|
segment_duration_seconds: str
|
|
frame_interval_seconds: str
|
|
rules_json: str
|
|
min_collection_count: int | None
|
|
min_valid_collection_count: int | None
|
|
min_video_count: int | None
|
|
min_filtered_count: int | None
|
|
min_clip_count: int | None
|
|
min_annotation_count: int | None
|
|
min_package_count: int | None
|
|
min_sampled_count: int | None
|
|
preflight_only: bool
|
|
max_wait_seconds: int
|
|
|
|
|
|
def env_default(name: str, fallback: str = "") -> str:
|
|
return os.getenv(name, fallback).strip()
|
|
|
|
|
|
def parse_positive_int_value(name: str, value: str) -> int:
|
|
parsed, issue = parse_positive_int(name, value)
|
|
if issue:
|
|
raise AcceptanceError(issue)
|
|
if parsed is None:
|
|
raise AcceptanceError(f"{name} is required")
|
|
return parsed
|
|
|
|
|
|
def positive_int_arg(name: str):
|
|
def parse(value: str) -> int:
|
|
parsed, issue = parse_positive_int(name, value)
|
|
if issue:
|
|
raise argparse.ArgumentTypeError(issue)
|
|
if parsed is None:
|
|
raise argparse.ArgumentTypeError(f"{name} is required")
|
|
return parsed
|
|
|
|
return parse
|
|
|
|
|
|
def positive_int_env_default(name: str, fallback: int) -> int:
|
|
value = env_default(name)
|
|
if not value:
|
|
return fallback
|
|
return parse_positive_int_value(name, value)
|
|
|
|
|
|
def optional_positive_int_env_default(name: str) -> int | None:
|
|
value = env_default(name)
|
|
if not value:
|
|
return None
|
|
return parse_positive_int_value(name, value)
|
|
|
|
|
|
def parse_args() -> Config:
|
|
parser = argparse.ArgumentParser(description="Run X-platform operator acceptance against an explicit real dataset")
|
|
parser.add_argument("--api-base-url", default=api_base_url_from_env())
|
|
parser.add_argument("--api-token", default=env_default("DROBOTICFLOW_API_TOKEN"))
|
|
parser.add_argument(
|
|
"--acceptance-email",
|
|
dest="acceptance_email",
|
|
default=env_default("DROBOTICFLOW_ACCEPTANCE_EMAIL"),
|
|
)
|
|
parser.add_argument(
|
|
"--acceptance-password",
|
|
dest="acceptance_password",
|
|
default=os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""),
|
|
)
|
|
parser.add_argument("--source-path", default=env_default("XPLATFORM_SOURCE_PATH", env_default("SOURCE_PATH")))
|
|
parser.add_argument("--storage-connection-id", default=env_default("XPLATFORM_STORAGE_CONNECTION_ID", env_default("STORAGE_CONNECTION_ID")))
|
|
parser.add_argument("--storage-connection-name", default=env_default("XPLATFORM_STORAGE_CONNECTION_NAME", env_default("STORAGE_CONNECTION_NAME")))
|
|
parser.add_argument("--storage-connection-bucket", default=env_default("XPLATFORM_STORAGE_CONNECTION_BUCKET"))
|
|
parser.add_argument("--upload-output-path", default=env_default("XPLATFORM_UPLOAD_OUTPUT_PATH", env_default("UPLOAD_OUTPUT_PATH")))
|
|
parser.add_argument("--project-prefix", default=env_default("XPLATFORM_PROJECT_PREFIX", DEFAULT_PROJECT_PREFIX))
|
|
parser.add_argument("--min-clarity", default=env_default("XPLATFORM_MIN_CLARITY"))
|
|
parser.add_argument("--max-shake", default=env_default("XPLATFORM_MAX_SHAKE"))
|
|
parser.add_argument("--max-hand-occlusion", default=env_default("XPLATFORM_MAX_HAND_OCCLUSION"))
|
|
parser.add_argument("--clarity-scale", default=env_default("XPLATFORM_CLARITY_SCALE"))
|
|
parser.add_argument("--date-start", default=env_default("XPLATFORM_DATE_START"))
|
|
parser.add_argument("--date-end", default=env_default("XPLATFORM_DATE_END"))
|
|
parser.add_argument("--place", default=env_default("XPLATFORM_PLACE"))
|
|
parser.add_argument("--dataset-type", default=env_default("XPLATFORM_TYPE"))
|
|
parser.add_argument("--segment-duration-seconds", default=env_default("XPLATFORM_SEGMENT_DURATION_SECONDS"))
|
|
parser.add_argument("--frame-interval-seconds", default=env_default("XPLATFORM_FRAME_INTERVAL_SECONDS"))
|
|
parser.add_argument("--rules-json", default=env_default("XPLATFORM_RULES_JSON"))
|
|
parser.add_argument(
|
|
"--min-collection-count",
|
|
type=positive_int_arg("XPLATFORM_MIN_COLLECTION_COUNT"),
|
|
default=optional_positive_int_env_default("XPLATFORM_MIN_COLLECTION_COUNT"),
|
|
)
|
|
parser.add_argument(
|
|
"--min-valid-collection-count",
|
|
type=positive_int_arg("XPLATFORM_MIN_VALID_COLLECTION_COUNT"),
|
|
default=optional_positive_int_env_default("XPLATFORM_MIN_VALID_COLLECTION_COUNT"),
|
|
)
|
|
parser.add_argument(
|
|
"--min-video-count",
|
|
type=positive_int_arg("XPLATFORM_MIN_VIDEO_COUNT"),
|
|
default=optional_positive_int_env_default("XPLATFORM_MIN_VIDEO_COUNT"),
|
|
)
|
|
parser.add_argument(
|
|
"--min-filtered-count",
|
|
type=positive_int_arg("XPLATFORM_MIN_FILTERED_COUNT"),
|
|
default=optional_positive_int_env_default("XPLATFORM_MIN_FILTERED_COUNT"),
|
|
)
|
|
parser.add_argument(
|
|
"--min-clip-count",
|
|
type=positive_int_arg("XPLATFORM_MIN_CLIP_COUNT"),
|
|
default=optional_positive_int_env_default("XPLATFORM_MIN_CLIP_COUNT"),
|
|
)
|
|
parser.add_argument(
|
|
"--min-annotation-count",
|
|
type=positive_int_arg("XPLATFORM_MIN_ANNOTATION_COUNT"),
|
|
default=optional_positive_int_env_default("XPLATFORM_MIN_ANNOTATION_COUNT"),
|
|
)
|
|
parser.add_argument(
|
|
"--min-package-count",
|
|
type=positive_int_arg("XPLATFORM_MIN_PACKAGE_COUNT"),
|
|
default=optional_positive_int_env_default("XPLATFORM_MIN_PACKAGE_COUNT"),
|
|
)
|
|
parser.add_argument(
|
|
"--min-sampled-count",
|
|
type=positive_int_arg("XPLATFORM_MIN_SAMPLED_COUNT"),
|
|
default=optional_positive_int_env_default("XPLATFORM_MIN_SAMPLED_COUNT"),
|
|
)
|
|
parser.add_argument("--preflight-only", action="store_true")
|
|
parser.add_argument(
|
|
"--max-wait-seconds",
|
|
type=positive_int_arg("XPLATFORM_MAX_WAIT_SECONDS"),
|
|
default=positive_int_env_default("XPLATFORM_MAX_WAIT_SECONDS", 7200),
|
|
)
|
|
args = parser.parse_args()
|
|
return Config(
|
|
api_base_url=args.api_base_url.rstrip("/"),
|
|
api_token=args.api_token,
|
|
acceptance_email=args.acceptance_email,
|
|
acceptance_password=args.acceptance_password,
|
|
source_path=args.source_path,
|
|
storage_connection_id=args.storage_connection_id,
|
|
storage_connection_name=args.storage_connection_name,
|
|
storage_connection_bucket=args.storage_connection_bucket,
|
|
upload_output_path=args.upload_output_path.rstrip("/"),
|
|
project_prefix=args.project_prefix,
|
|
min_clarity=args.min_clarity,
|
|
max_shake=args.max_shake,
|
|
max_hand_occlusion=args.max_hand_occlusion,
|
|
clarity_scale=args.clarity_scale,
|
|
date_start=args.date_start,
|
|
date_end=args.date_end,
|
|
place=args.place,
|
|
dataset_type=args.dataset_type,
|
|
segment_duration_seconds=args.segment_duration_seconds,
|
|
frame_interval_seconds=args.frame_interval_seconds,
|
|
rules_json=args.rules_json,
|
|
min_collection_count=args.min_collection_count,
|
|
min_valid_collection_count=args.min_valid_collection_count,
|
|
min_video_count=args.min_video_count,
|
|
min_filtered_count=args.min_filtered_count,
|
|
min_clip_count=args.min_clip_count,
|
|
min_annotation_count=args.min_annotation_count,
|
|
min_package_count=args.min_package_count,
|
|
min_sampled_count=args.min_sampled_count,
|
|
preflight_only=bool(args.preflight_only),
|
|
max_wait_seconds=args.max_wait_seconds,
|
|
)
|
|
|
|
|
|
def parse_bos_path(value: str) -> tuple[str, str]:
|
|
parsed = urllib.parse.urlparse(value)
|
|
if parsed.scheme.lower() != "bos":
|
|
return "", ""
|
|
return parsed.netloc.strip(), parsed.path.lstrip("/")
|
|
|
|
|
|
def source_is_bos(value: str) -> bool:
|
|
bucket, _ = parse_bos_path(value)
|
|
return bool(bucket)
|
|
|
|
|
|
def normalize_sampling_rules_json(value: str) -> str:
|
|
return normalize_rules_json(value, error_factory=AcceptanceError, label="XPLATFORM_RULES_JSON")
|
|
|
|
|
|
def client_for_config(config: Config) -> DroboticflowClient:
|
|
global CLIENT
|
|
if CLIENT is None or CLIENT.api_base_url != config.api_base_url or CLIENT.api_token != config.api_token:
|
|
CLIENT = DroboticflowClient(
|
|
config.api_base_url,
|
|
api_token=config.api_token,
|
|
error_factory=AcceptanceError,
|
|
)
|
|
return CLIENT
|
|
|
|
|
|
def request(
|
|
config: Config,
|
|
method: str,
|
|
path: str,
|
|
payload: Any | None = None,
|
|
expected: tuple[int, ...] = (200, 201, 202, 204),
|
|
) -> Any:
|
|
return client_for_config(config).request_json(method, path, payload, expected=expected)
|
|
|
|
|
|
def validate_config(config: Config) -> None:
|
|
issues: list[str] = []
|
|
if bool(config.acceptance_email) != bool(config.acceptance_password):
|
|
issues.append("DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together")
|
|
if not config.source_path:
|
|
issues.append("XPLATFORM_SOURCE_PATH or --source-path is required")
|
|
if (source_is_bos(config.source_path) or config.upload_output_path.startswith("bos://")) and not (
|
|
config.storage_connection_id or config.storage_connection_name or config.storage_connection_bucket
|
|
):
|
|
issues.append(
|
|
"XPLATFORM_STORAGE_CONNECTION_ID, XPLATFORM_STORAGE_CONNECTION_NAME, or XPLATFORM_STORAGE_CONNECTION_BUCKET is required for bos:// source or upload paths"
|
|
)
|
|
|
|
issues.extend(
|
|
unit_interval_threshold_issues(
|
|
(
|
|
("XPLATFORM_MIN_CLARITY", config.min_clarity),
|
|
("XPLATFORM_MAX_SHAKE", config.max_shake),
|
|
("XPLATFORM_MAX_HAND_OCCLUSION", config.max_hand_occlusion),
|
|
)
|
|
)
|
|
)
|
|
issues.extend(
|
|
positive_float_issues(
|
|
(
|
|
("XPLATFORM_CLARITY_SCALE", config.clarity_scale),
|
|
("XPLATFORM_FRAME_INTERVAL_SECONDS", config.frame_interval_seconds),
|
|
)
|
|
)
|
|
)
|
|
issues.extend(positive_int_issues((("XPLATFORM_SEGMENT_DURATION_SECONDS", config.segment_duration_seconds),)))
|
|
|
|
for label, value in (
|
|
("XPLATFORM_MIN_COLLECTION_COUNT", config.min_collection_count),
|
|
("XPLATFORM_MIN_VALID_COLLECTION_COUNT", config.min_valid_collection_count),
|
|
("XPLATFORM_MIN_VIDEO_COUNT", config.min_video_count),
|
|
("XPLATFORM_MIN_FILTERED_COUNT", config.min_filtered_count),
|
|
("XPLATFORM_MIN_CLIP_COUNT", config.min_clip_count),
|
|
("XPLATFORM_MIN_ANNOTATION_COUNT", config.min_annotation_count),
|
|
("XPLATFORM_MIN_PACKAGE_COUNT", config.min_package_count),
|
|
("XPLATFORM_MIN_SAMPLED_COUNT", config.min_sampled_count),
|
|
):
|
|
if value is None:
|
|
issues.append(f"{label} or the matching CLI argument is required")
|
|
elif value < 1:
|
|
issues.append(f"{label} must be greater than or equal to 1")
|
|
if config.max_wait_seconds < 60:
|
|
issues.append("XPLATFORM_MAX_WAIT_SECONDS must be greater than or equal to 60")
|
|
|
|
try:
|
|
config.rules_json = normalize_sampling_rules_json(config.rules_json)
|
|
except AcceptanceError as exc:
|
|
issues.append(str(exc))
|
|
if issues:
|
|
raise AcceptanceError(
|
|
"invalid X-platform acceptance configuration:\n"
|
|
+ "\n".join(f"- {item}" for item in issues)
|
|
+ "\nThe script has no synthetic dataset fallback and no built-in acceptance thresholds; provide explicit real input data and criteria."
|
|
)
|
|
|
|
|
|
def login_if_configured(config: Config) -> None:
|
|
client_for_config(config).login_if_configured(
|
|
config.acceptance_email,
|
|
config.acceptance_password,
|
|
label="X-platform acceptance user",
|
|
)
|
|
|
|
|
|
def resolve_storage_connection(config: Config) -> dict[str, Any] | None:
|
|
if not source_is_bos(config.source_path) and not config.upload_output_path.startswith("bos://"):
|
|
return None
|
|
payload = request(config, "GET", "/api/storage/connections")
|
|
items = payload.get("items", []) if isinstance(payload, dict) else []
|
|
matched = None
|
|
bucket_selector = config.storage_connection_bucket
|
|
if not bucket_selector:
|
|
bucket_selector, _ = parse_bos_path(config.source_path)
|
|
if not bucket_selector:
|
|
bucket_selector, _ = parse_bos_path(config.upload_output_path)
|
|
|
|
if config.storage_connection_id:
|
|
matched = next((item for item in items if item.get("id") == config.storage_connection_id), None)
|
|
elif config.storage_connection_name:
|
|
matched = next((item for item in items if item.get("name") == config.storage_connection_name), None)
|
|
elif bucket_selector:
|
|
matched = next((item for item in items if item.get("bucket") == bucket_selector), None)
|
|
|
|
if not matched or not matched.get("id"):
|
|
selectors = [
|
|
config.storage_connection_id and f"id={config.storage_connection_id}",
|
|
config.storage_connection_name and f"name={config.storage_connection_name}",
|
|
bucket_selector and f"bucket={bucket_selector}",
|
|
]
|
|
raise AcceptanceError(f"no storage connection matched {', '.join(item for item in selectors if item)}")
|
|
|
|
source_bucket, _ = parse_bos_path(config.source_path)
|
|
upload_bucket, _ = parse_bos_path(config.upload_output_path)
|
|
connection_bucket = str(matched.get("bucket", "")).strip()
|
|
if source_bucket and connection_bucket and connection_bucket != source_bucket:
|
|
raise AcceptanceError(f"storage connection bucket mismatch: connection={connection_bucket}, source={source_bucket}")
|
|
if upload_bucket and connection_bucket and connection_bucket != upload_bucket:
|
|
raise AcceptanceError(f"storage connection bucket mismatch: connection={connection_bucket}, upload={upload_bucket}")
|
|
return matched
|
|
|
|
|
|
def build_node(
|
|
operator: dict[str, Any],
|
|
node_id: str,
|
|
x: int,
|
|
y: int,
|
|
hyper_overrides: dict[str, str] | None = None,
|
|
*,
|
|
connection_id: str = "",
|
|
bucket: str = "",
|
|
prefix: str = "",
|
|
) -> dict[str, Any]:
|
|
hyper = [dict(item) for item in operator.get("hyperparams", [])]
|
|
overrides = hyper_overrides or {}
|
|
seen = set()
|
|
for item in hyper:
|
|
key = item.get("key")
|
|
if key in overrides:
|
|
item["value"] = overrides[key]
|
|
seen.add(key)
|
|
for key, value in overrides.items():
|
|
if key in seen:
|
|
continue
|
|
hyper.append({"key": key, "value": value})
|
|
data = {
|
|
"label": operator["name"],
|
|
"category": operator["category"],
|
|
"description": operator["description"],
|
|
"inputs": operator.get("inputs", []),
|
|
"outputs": operator.get("outputs", []),
|
|
"image": operator.get("image", ""),
|
|
"skillDefinition": operator.get("skillDefinition", ""),
|
|
"hyperparams": hyper,
|
|
}
|
|
if connection_id:
|
|
data["connectionId"] = connection_id
|
|
if bucket:
|
|
data["bucket"] = bucket
|
|
if prefix:
|
|
data["prefix"] = prefix
|
|
return {
|
|
"id": node_id,
|
|
"type": "operator",
|
|
"data": data,
|
|
"position": {"x": x, "y": y},
|
|
}
|
|
|
|
|
|
def required_operator_names(config: Config) -> list[str]:
|
|
names = [
|
|
"X 平台数据集索引",
|
|
"采集目录校验",
|
|
"视频质检",
|
|
"数据过滤",
|
|
"视频切片",
|
|
"视频标注",
|
|
"成品打包",
|
|
"抽检采样",
|
|
"结果上传",
|
|
]
|
|
if source_is_bos(config.source_path):
|
|
names.insert(0, "BOS 增量扫描")
|
|
return names
|
|
|
|
|
|
def wait_for_run(config: Config, run_id: str) -> dict[str, Any]:
|
|
deadline = time.time() + config.max_wait_seconds
|
|
while time.time() < deadline:
|
|
current = request(config, "GET", f"/api/runs/{run_id}")
|
|
if current["status"] in {"completed", "failed", "cancelled", "waiting_for_audit"}:
|
|
return current
|
|
time.sleep(2)
|
|
raise AcceptanceError(f"timeout waiting for run {run_id}")
|
|
|
|
|
|
def artifact_summaries(artifacts: list[dict[str, Any]]) -> dict[str, Any]:
|
|
interesting_names = {
|
|
"source-manifest.json",
|
|
"dataset-index.json",
|
|
"dataset-validation.json",
|
|
"quality-report.json",
|
|
"filtered-videos.json",
|
|
"clips-manifest.json",
|
|
"annotations-manifest.json",
|
|
"packages-manifest.json",
|
|
"audit-samples.json",
|
|
"uploaded-results.json",
|
|
}
|
|
interesting: dict[str, Any] = {}
|
|
for item in artifacts:
|
|
if item["name"] in interesting_names:
|
|
interesting[item["name"]] = item.get("data")
|
|
|
|
summary_artifacts: dict[str, Any] = {}
|
|
for name, data in interesting.items():
|
|
if not isinstance(data, dict):
|
|
summary_artifacts[name] = data
|
|
continue
|
|
summary_artifacts[name] = {
|
|
key: data.get(key)
|
|
for key in (
|
|
"listedCount",
|
|
"newCount",
|
|
"collectionCount",
|
|
"validCollectionCount",
|
|
"videoCount",
|
|
"passedCount",
|
|
"count",
|
|
"clipCount",
|
|
"annotationCount",
|
|
"packageCount",
|
|
"sampledCount",
|
|
"uploadedCount",
|
|
)
|
|
if key in data
|
|
}
|
|
return summary_artifacts
|
|
|
|
|
|
def required_minimum(value: int | None, label: str) -> int:
|
|
if value is None:
|
|
raise AcceptanceError(f"{label} is required before running X-platform acceptance")
|
|
return value
|
|
|
|
|
|
def acceptance_minimums(config: Config) -> dict[str, int]:
|
|
return {
|
|
"collectionCount": required_minimum(config.min_collection_count, "XPLATFORM_MIN_COLLECTION_COUNT"),
|
|
"validCollectionCount": required_minimum(config.min_valid_collection_count, "XPLATFORM_MIN_VALID_COLLECTION_COUNT"),
|
|
"videoCount": required_minimum(config.min_video_count, "XPLATFORM_MIN_VIDEO_COUNT"),
|
|
"filteredCount": required_minimum(config.min_filtered_count, "XPLATFORM_MIN_FILTERED_COUNT"),
|
|
"clipCount": required_minimum(config.min_clip_count, "XPLATFORM_MIN_CLIP_COUNT"),
|
|
"annotationCount": required_minimum(config.min_annotation_count, "XPLATFORM_MIN_ANNOTATION_COUNT"),
|
|
"packageCount": required_minimum(config.min_package_count, "XPLATFORM_MIN_PACKAGE_COUNT"),
|
|
"sampledCount": required_minimum(config.min_sampled_count, "XPLATFORM_MIN_SAMPLED_COUNT"),
|
|
}
|
|
|
|
|
|
def assert_minimums(summary_artifacts: dict[str, Any], config: Config) -> None:
|
|
minimums = acceptance_minimums(config)
|
|
checks = {
|
|
"dataset-index.json": ("collectionCount", minimums["collectionCount"]),
|
|
"dataset-validation.json": ("validCollectionCount", minimums["validCollectionCount"]),
|
|
"quality-report.json": ("videoCount", minimums["videoCount"]),
|
|
"filtered-videos.json": ("count", minimums["filteredCount"]),
|
|
"clips-manifest.json": ("clipCount", minimums["clipCount"]),
|
|
"annotations-manifest.json": ("annotationCount", minimums["annotationCount"]),
|
|
"packages-manifest.json": ("packageCount", minimums["packageCount"]),
|
|
"audit-samples.json": ("sampledCount", minimums["sampledCount"]),
|
|
}
|
|
for artifact_name, (field, minimum) in checks.items():
|
|
data = summary_artifacts.get(artifact_name)
|
|
actual = data.get(field) if isinstance(data, dict) else None
|
|
if not isinstance(actual, int) or actual < minimum:
|
|
raise AcceptanceError(f"{artifact_name} expected {field}>={minimum}, got {data}")
|
|
|
|
|
|
def create_workflow(
|
|
config: Config,
|
|
operators_by_name: dict[str, dict[str, Any]],
|
|
connection: dict[str, Any] | None,
|
|
) -> tuple[str, str]:
|
|
timestamp = time.strftime("%Y%m%d%H%M%S")
|
|
project_name = f"{config.project_prefix} {timestamp}"
|
|
workflow_name = f"XPlatform Real Pipeline {timestamp}"
|
|
source_bucket, source_prefix = parse_bos_path(config.source_path)
|
|
connection_id = str(connection.get("id", "")) if connection else ""
|
|
|
|
project = request(config, "POST", "/api/projects", {"name": project_name, "description": "Created by real X-platform acceptance"})
|
|
project_id = project["id"]
|
|
|
|
nodes: list[dict[str, Any]] = []
|
|
edges: list[dict[str, Any]] = []
|
|
start_x = 0
|
|
if source_is_bos(config.source_path):
|
|
nodes.append(
|
|
build_node(
|
|
operators_by_name["BOS 增量扫描"],
|
|
"source",
|
|
0,
|
|
0,
|
|
{"DATA_SOURCE": "BOS", "SOURCE_PATH": config.source_path, "SCAN_MODE": "FULL"},
|
|
connection_id=connection_id,
|
|
bucket=source_bucket,
|
|
prefix=source_prefix,
|
|
)
|
|
)
|
|
edges.append({"id": "source-index", "source": "source", "sourceHandle": "out-1", "target": "index", "targetHandle": "in-1"})
|
|
start_x = 220
|
|
|
|
index_overrides = {} if source_is_bos(config.source_path) else {"SOURCE_PATH": config.source_path}
|
|
nodes.extend(
|
|
[
|
|
build_node(operators_by_name["X 平台数据集索引"], "index", start_x, 0, index_overrides),
|
|
build_node(operators_by_name["采集目录校验"], "validate", start_x + 220, 0),
|
|
build_node(
|
|
operators_by_name["视频质检"],
|
|
"quality",
|
|
start_x + 440,
|
|
0,
|
|
{
|
|
"MIN_CLARITY": config.min_clarity,
|
|
"MAX_SHAKE": config.max_shake,
|
|
"MAX_HAND_OCCLUSION": config.max_hand_occlusion,
|
|
"CLARITY_SCALE": config.clarity_scale,
|
|
},
|
|
),
|
|
build_node(
|
|
operators_by_name["数据过滤"],
|
|
"filter",
|
|
start_x + 660,
|
|
0,
|
|
{
|
|
"DATE_START": config.date_start,
|
|
"DATE_END": config.date_end,
|
|
"PLACE": config.place,
|
|
"TYPE": config.dataset_type,
|
|
"MIN_CLARITY": config.min_clarity,
|
|
"MAX_SHAKE": config.max_shake,
|
|
"MAX_HAND_OCCLUSION": config.max_hand_occlusion,
|
|
},
|
|
),
|
|
build_node(operators_by_name["视频切片"], "slice", start_x + 880, 0, {"SEGMENT_DURATION_SECONDS": config.segment_duration_seconds}),
|
|
build_node(operators_by_name["视频标注"], "annotate", start_x + 1100, 0, {"FRAME_INTERVAL_SECONDS": config.frame_interval_seconds}),
|
|
build_node(operators_by_name["成品打包"], "package", start_x + 1320, 0),
|
|
build_node(operators_by_name["抽检采样"], "sample", start_x + 1540, -80, {"RULES_JSON": config.rules_json}),
|
|
build_node(
|
|
operators_by_name["结果上传"],
|
|
"upload",
|
|
start_x + 1540,
|
|
100,
|
|
{"OUTPUT_PATH": config.upload_output_path} if config.upload_output_path else {},
|
|
connection_id=connection_id if config.upload_output_path.startswith("bos://") else "",
|
|
),
|
|
]
|
|
)
|
|
edges.extend(
|
|
[
|
|
{"id": "index-validate", "source": "index", "sourceHandle": "out-1", "target": "validate", "targetHandle": "in-1"},
|
|
{"id": "validate-quality", "source": "validate", "sourceHandle": "out-1", "target": "quality", "targetHandle": "in-1"},
|
|
{"id": "quality-filter", "source": "quality", "sourceHandle": "out-1", "target": "filter", "targetHandle": "in-1"},
|
|
{"id": "filter-slice", "source": "filter", "sourceHandle": "out-1", "target": "slice", "targetHandle": "in-1"},
|
|
{"id": "slice-annotate", "source": "slice", "sourceHandle": "out-1", "target": "annotate", "targetHandle": "in-1"},
|
|
{"id": "annotate-package", "source": "annotate", "sourceHandle": "out-1", "target": "package", "targetHandle": "in-1"},
|
|
{"id": "package-sample", "source": "package", "sourceHandle": "out-1", "target": "sample", "targetHandle": "in-1"},
|
|
{"id": "package-upload", "source": "package", "sourceHandle": "out-1", "target": "upload", "targetHandle": "in-1"},
|
|
]
|
|
)
|
|
|
|
workflow = request(
|
|
config,
|
|
"POST",
|
|
f"/api/projects/{project_id}/workflows",
|
|
{
|
|
"name": workflow_name,
|
|
"description": "Live acceptance for the X-platform operator chain using an explicit real dataset",
|
|
"type": "batch",
|
|
"nodes": nodes,
|
|
"edges": edges,
|
|
},
|
|
)
|
|
return project_id, workflow["id"]
|
|
|
|
|
|
def main() -> int:
|
|
config = parse_args()
|
|
validate_config(config)
|
|
if config.api_token:
|
|
print("using API token auth for X-platform acceptance requests")
|
|
login_if_configured(config)
|
|
|
|
operators = request(config, "GET", "/api/operators")["items"]
|
|
operators_by_name = {item["name"]: item for item in operators}
|
|
missing = [name for name in required_operator_names(config) if name not in operators_by_name]
|
|
if missing:
|
|
raise AcceptanceError(f"missing operators: {missing}")
|
|
connection = resolve_storage_connection(config)
|
|
|
|
if config.preflight_only:
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "preflight-ok",
|
|
"sourcePath": config.source_path,
|
|
"sourceIsBos": source_is_bos(config.source_path),
|
|
"storageConnectionId": connection.get("id") if connection else None,
|
|
"requiredOperators": required_operator_names(config),
|
|
"minimums": acceptance_minimums(config),
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
project_id, workflow_id = create_workflow(config, operators_by_name, connection)
|
|
run = request(config, "POST", f"/api/workflows/{workflow_id}/run", {"trigger": "manual"})
|
|
run_id = run["id"]
|
|
terminal = wait_for_run(config, run_id)
|
|
artifacts = request(config, "GET", f"/api/runs/{run_id}/artifacts")["items"]
|
|
logs = request(config, "GET", f"/api/runs/{run_id}/logs")["items"]
|
|
summary_artifacts = artifact_summaries(artifacts)
|
|
summary = {
|
|
"projectId": project_id,
|
|
"workflowId": workflow_id,
|
|
"runId": run_id,
|
|
"status": terminal["status"],
|
|
"artifactCount": len(artifacts),
|
|
"nodeStatuses": {item["nodeId"]: item["status"] for item in terminal.get("nodeExecutions", [])},
|
|
"interestingArtifacts": summary_artifacts,
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
if terminal["status"] != "completed":
|
|
print(json.dumps({"logs": logs[-20:]}, ensure_ascii=False, indent=2), file=sys.stderr)
|
|
return 1
|
|
assert_minimums(summary_artifacts, config)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
raise SystemExit(main())
|
|
except AcceptanceError as exc:
|
|
print(str(exc), file=sys.stderr)
|
|
raise SystemExit(1) from exc
|