droboticflow/scripts/wjzh_operator_multi_project_regression.py
2026-06-16 21:29:08 +08:00

617 lines
23 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import importlib.util
import json
import os
import sys
import time
from collections import OrderedDict
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parent
BASE_PATH = ROOT / "wjzh_operator_regression.py"
spec = importlib.util.spec_from_file_location("wjzh_regression_base", BASE_PATH)
if spec is None or spec.loader is None:
raise SystemExit(f"failed to load regression base from {BASE_PATH}")
base = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = base
spec.loader.exec_module(base)
RUNNER_IMAGE_OVERRIDE = os.getenv("OPERATOR_RUNNER_IMAGE", "").strip()
SOURCE_TEXT_ENV = "WJZH_REGRESSION_SOURCE_TEXT"
SOURCE_TEXT_FILE_ENV = "WJZH_REGRESSION_SOURCE_TEXT_FILE"
SOURCE_JSON_ENV = "WJZH_REGRESSION_SOURCE_JSON"
SOURCE_JSON_FILE_ENV = "WJZH_REGRESSION_SOURCE_JSON_FILE"
def read_required_env_or_file(value_env: str, file_env: str) -> str:
value = os.getenv(value_env)
file_path = os.getenv(file_env, "").strip()
if value and file_path:
raise base.RegressionError(f"set only one of {value_env} or {file_env}")
if file_path:
path = Path(file_path).expanduser()
if not path.is_file():
raise base.RegressionError(f"{file_env} does not point to a readable file: {file_path}")
value = path.read_text(encoding="utf-8")
if value is None or not value.strip():
raise base.RegressionError(f"{value_env} or {file_env} is required for the multi-project SOURCE_TEXT/SOURCE_JSON regression")
return value
def load_source_variant_payloads() -> tuple[str, str]:
source_text = read_required_env_or_file(SOURCE_TEXT_ENV, SOURCE_TEXT_FILE_ENV)
source_json = read_required_env_or_file(SOURCE_JSON_ENV, SOURCE_JSON_FILE_ENV)
try:
parsed = json.loads(source_json)
except json.JSONDecodeError as err:
raise base.RegressionError(f"{SOURCE_JSON_ENV} / {SOURCE_JSON_FILE_ENV} must contain valid JSON: {err}") from err
return source_text, json.dumps(parsed, ensure_ascii=False)
def refresh_workflow_runner_image(config: Any, workflow_id: str, runner_image: str) -> dict[str, Any]:
if not runner_image:
return base.get_workflow(config, workflow_id)
workflow = base.get_workflow(config, workflow_id)
nodes = workflow.get("nodes", [])
changed = False
for node in nodes:
data = node.get("data") or {}
image = str(data.get("image", "")).strip()
if "droboticflow-operator-runner:" not in image:
continue
if image == runner_image:
continue
data["image"] = runner_image
changed = True
if changed:
workflow = base.update_workflow(config, workflow_id, {"nodes": nodes})
return workflow
def build_group_projects(config: Any, timestamp: str) -> OrderedDict[str, dict[str, Any]]:
groups: OrderedDict[str, dict[str, Any]] = OrderedDict(
[
(
"基础数据与索引",
{
"description": "真实数据源、目录扫描、模型推理、BOS 扫描、索引、校验与上传回归。",
},
),
(
"JSON与逻辑",
{
"description": "JSON/CSV/分流/合并等逻辑链路回归。",
},
),
(
"审核与业务写入",
{
"description": "人工审核 approve/reject 分支及 Mongo 写入回归。",
},
),
(
"视频处理与打包",
{
"description": "视频质检、过滤、切片、标注、打包、抽检回归。",
},
),
(
"Robotics与增量",
{
"description": "LeRobot / ROSBag 概览和目录级增量循环处理回归。",
},
),
]
)
for group_name, payload in groups.items():
payload["project"] = base.create_project(
config,
f"{config.project_prefix} {timestamp} {group_name}",
payload["description"],
)
return groups
def group_name_for_workflow(workflow_name: str) -> str:
if workflow_name.startswith(("00-", "01-")):
return "基础数据与索引"
if workflow_name.startswith("02-"):
return "JSON与逻辑"
if workflow_name.startswith(("03-", "04-", "05-")):
return "审核与业务写入"
if workflow_name.startswith(("06-", "07-", "08-", "10-", "11-")):
return "视频处理与打包"
if workflow_name.startswith(("09-", "12-")):
return "Robotics与增量"
raise base.RegressionError(f"unmapped workflow group for {workflow_name}")
def clone_temp_workflows(
config: Any,
temp_project_id: str,
temp_workflows: list[dict[str, Any]],
grouped_projects: OrderedDict[str, dict[str, Any]],
) -> list[dict[str, Any]]:
cloned: list[dict[str, Any]] = []
for item in temp_workflows:
original = item["workflow"]
target_group = group_name_for_workflow(str(original.get("name", "")))
target_project_id = grouped_projects[target_group]["project"]["id"]
copy = base.clone_workflow(config, target_project_id, original["id"], original["name"])
copy = refresh_workflow_runner_image(config, copy["id"], RUNNER_IMAGE_OVERRIDE)
cloned_item = dict(item)
cloned_item["workflow"] = copy
cloned_item["targetGroup"] = target_group
cloned.append(cloned_item)
base.request(config, "DELETE", f"/api/projects/{temp_project_id}", expected=(204,))
return cloned
def create_source_variants_workflow(
config: Any,
project_id: str,
operators_by_name: dict[str, dict[str, Any]],
connection_id: str,
output_root: str,
source_text: str,
source_json: str,
) -> dict[str, Any]:
nodes = [
base.build_node(
operators_by_name["真实数据源"],
"source_text",
0,
-90,
{"SOURCE_TEXT": source_text},
connection_id=connection_id,
label="真实数据源-文本",
),
base.build_node(
operators_by_name["模型推理"],
"infer_text",
240,
-90,
{"COMMAND": base.model_inference_command()},
),
base.upload_node(
operators_by_name["结果上传"],
connection_id,
"upload_text",
480,
-90,
f"{output_root}/00-source-variants/text",
),
base.build_node(
operators_by_name["真实数据源"],
"source_json",
0,
80,
{"SOURCE_JSON": source_json},
connection_id=connection_id,
label="真实数据源-JSON",
),
base.upload_node(
operators_by_name["结果上传"],
connection_id,
"upload_json",
240,
80,
f"{output_root}/00-source-variants/json",
),
]
edges = [
base.upload_edge("source_text", "infer_text", "source-text-infer"),
base.upload_edge("infer_text", "upload_text", "infer-text-upload"),
base.upload_edge("source_json", "upload_json", "source-json-upload"),
]
workflow = base.create_workflow(
config,
project_id,
"00-真实数据源多入口",
"使用调用方显式提供的真实文本和 JSON payload 覆盖 SOURCE_TEXT / SOURCE_JSON 子功能。",
nodes,
edges,
)
return refresh_workflow_runner_image(config, workflow["id"], RUNNER_IMAGE_OVERRIDE)
def create_manual_sampling_workflow(
config: Any,
project_id: str,
operators_by_name: dict[str, dict[str, Any]],
connection_id: str,
output_root: str,
source_path: str,
) -> dict[str, Any]:
nodes = [
base.build_node(
operators_by_name["BOS 增量扫描"],
"source",
0,
0,
{"SOURCE_PATH": source_path, "SCAN_MODE": "FULL"},
connection_id=connection_id,
),
base.build_node(operators_by_name["X 平台数据集索引"], "index", 220, 0),
base.build_node(
operators_by_name["采集目录校验"],
"validate",
440,
0,
{"REQUIRE_INTRINSICS": "false"},
),
base.build_node(
operators_by_name["Shell 命令执行"],
"batch_pick",
660,
0,
{"COMMAND": base.cap_duration_command(base.int_config(config.batch_window_seconds), max_items=4, max_duration=500)},
label="人工抽检前抽样",
),
base.build_node(
operators_by_name["视频质检"],
"quality",
880,
-60,
base.quality_hyperparams(config),
),
base.build_node(
operators_by_name["视频切片"],
"slice",
1100,
-60,
base.segment_hyperparams(config.batch_segment_duration_seconds),
),
base.build_node(
operators_by_name["视频标注"],
"annotate",
1320,
-60,
base.annotation_hyperparams(config),
),
base.build_node(operators_by_name["成品打包"], "package", 1540, -60),
base.build_node(
operators_by_name["抽检采样"],
"manual_sample",
1760,
-60,
{"RULES_JSON": config.sampling_rules_json},
label="人工抽检采样",
),
base.upload_node(
operators_by_name["结果上传"],
connection_id,
"manual_sample_upload",
1980,
-60,
f"{output_root}/11b-manual-sampling/sample",
),
]
edges = [
base.upload_edge("source", "index", "source-index"),
base.upload_edge("index", "validate", "index-validate"),
base.upload_edge("validate", "batch_pick", "validate-batch-pick"),
base.upload_edge("batch_pick", "quality", "batch-pick-quality"),
base.upload_edge("quality", "slice", "quality-slice"),
base.upload_edge("slice", "annotate", "slice-annotate"),
base.upload_edge("annotate", "package", "annotate-package"),
base.upload_edge("package", "manual_sample", "package-manual-sample"),
base.upload_edge("manual_sample", "manual_sample_upload", "manual-sample-upload"),
]
workflow = base.create_workflow(
config,
project_id,
"11B-人工抽检采样",
"使用人工抽检采样算子对成品包执行规则采样回归。",
nodes,
edges,
)
return refresh_workflow_runner_image(config, workflow["id"], RUNNER_IMAGE_OVERRIDE)
def create_multi_project_regression(
config: Any,
source_text: str,
source_json: str,
) -> tuple[OrderedDict[str, dict[str, Any]], list[dict[str, Any]], str, str, str]:
settings = base.request(config, "GET", "/api/settings")
operators = base.request(config, "GET", "/api/operators")["items"]
connection = base.ensure_storage_connection(config)
mongo_connection = base.find_mongo_connection(config)
base.validate_preflight(config, connection, operators, settings, mongo_connection)
operators_by_name = {item["name"]: item for item in operators}
base.delete_projects(config)
timestamp = time.strftime("%Y%m%d%H%M%S")
temp_project_id, temp_workflows, mongo_collection, output_root, incremental_source_path = base.create_regression_workflows(
config,
connection["id"],
mongo_connection["id"] if mongo_connection else "",
operators_by_name,
)
grouped_projects = build_group_projects(config, timestamp)
cloned = clone_temp_workflows(config, temp_project_id, temp_workflows, grouped_projects)
extras: list[dict[str, Any]] = []
extras.append(
{
"workflow": create_source_variants_workflow(
config,
grouped_projects["基础数据与索引"]["project"]["id"],
operators_by_name,
connection["id"],
output_root,
source_text,
source_json,
),
"targetGroup": "基础数据与索引",
}
)
extras.append(
{
"workflow": create_manual_sampling_workflow(
config,
grouped_projects["视频处理与打包"]["project"]["id"],
operators_by_name,
connection["id"],
output_root,
config.source_path,
),
"targetGroup": "视频处理与打包",
}
)
return grouped_projects, cloned + extras, mongo_collection, output_root, incremental_source_path
def configure_directory_incremental_workflow(
config: Any,
workflow_id: str,
incremental_source_path: str,
connection_id: str,
interval_seconds: int,
) -> None:
scan_bucket, scan_prefix = base.parse_bos_path(incremental_source_path)
payload: dict[str, Any] = {
"kind": "scan",
"maxConcurrentRuns": 1,
"incrementalUnit": "DIRECTORY",
"directoryDepth": 1,
"scanConfig": {
"enabled": True,
"connectionId": connection_id,
"dataSource": "BOS",
"sourcePath": incremental_source_path,
"bucket": scan_bucket,
"prefix": scan_prefix,
"scanMode": "INCREMENTAL",
},
"type": "streaming",
"schedule": {
"interval": interval_seconds,
},
}
base.update_workflow(config, workflow_id, payload)
def find_seed_clip_keys(config: Any, connection_id: str, output_root: str) -> list[str]:
seed_clip_keys: list[str] = []
for suffix in ["06-medium-video/slice", "07-full-chain/slice", "11-batch-fullchain/slice"]:
seed_clip_keys.extend(base.list_uploaded_mp4_keys(config, connection_id, output_root, suffix))
return list(dict.fromkeys([key for key in seed_clip_keys if key]))
def preflight_multi_project_regression(config: Any, source_text: str, source_json: str) -> dict[str, Any]:
settings = base.request(config, "GET", "/api/settings")
operators = base.request(config, "GET", "/api/operators")["items"]
connection = base.ensure_storage_connection(config)
mongo_connection = base.find_mongo_connection(config)
base.validate_preflight(config, connection, operators, settings, mongo_connection)
return {
"status": "ok",
"mode": "preflight-only",
"connectionId": connection["id"],
"mongoConnectionId": mongo_connection["id"] if mongo_connection else "",
"sourcePath": config.source_path,
"uploadRootPrefix": config.upload_root_prefix,
"sourceTextBytes": len(source_text.encode("utf-8")),
"sourceJsonBytes": len(source_json.encode("utf-8")),
"roboticsSourcePaths": {
"lerobot": config.lerobot_source_path,
"rosbag": config.rosbag_source_path,
},
}
def run_grouped_regression(config: Any, source_text: str, source_json: str) -> dict[str, Any]:
grouped_projects, workflows, mongo_collection, output_root, incremental_source_path = create_multi_project_regression(
config,
source_text,
source_json,
)
connection = base.ensure_storage_connection(config)
mongo_connection = base.find_mongo_connection(config)
workflow_items_by_name = {item["workflow"]["name"]: item for item in workflows}
execution_order = [
"00-源数据目录推理",
"00-真实数据源多入口",
"01-索引校验双上传",
"02-JSON工具链多分支",
"03-人工审核通过",
"04-人工审核拒绝",
"05-Mongo写入",
"06-中大型视频切片",
"07-超大视频全链路",
"08-多目录批量窗口化",
"09-Robotics真实概览",
"10-多目录多视频批量质检",
"11-多目录多视频批量全链路",
"11B-人工抽检采样",
]
results: list[dict[str, Any]] = []
mongo_verification: dict[str, Any] | None = None
for name in execution_order:
item = workflow_items_by_name[name]
workflow = item["workflow"]
rounds = max(int(item.get("rounds", 1) or 1), 1)
for round_index in range(rounds):
current_workflow = workflow
if round_index > 0:
group_name = item["targetGroup"]
current_workflow = base.clone_workflow(
config,
grouped_projects[group_name]["project"]["id"],
workflow["id"],
f"{workflow['name']} 第{round_index + 1}轮",
)
current_workflow = refresh_workflow_runner_image(config, current_workflow["id"], RUNNER_IMAGE_OVERRIDE)
finished = base.run_workflow_until_complete(
config,
current_workflow["id"],
config.max_wait_seconds,
audit_decision=str(item.get("auditDecision", "")).strip(),
)
summary = base.summarize_run(finished)
summary["projectId"] = grouped_projects[item["targetGroup"]]["project"]["id"]
summary["projectName"] = grouped_projects[item["targetGroup"]]["project"]["name"]
if rounds > 1:
summary["round"] = round_index + 1
if item.get("auditDecision"):
summary["auditDecision"] = item["auditDecision"]
results.append(summary)
if finished["status"] != "completed":
raise base.RegressionError(
json.dumps(
{
"status": "workflow-failed",
"workflowName": current_workflow["name"],
"run": summary,
"runLogsTail": base.get_run_logs(config, finished["id"])[-30:],
},
ensure_ascii=False,
indent=2,
)
)
if item.get("mongoCollection"):
documents = base.query_business_collection(config, mongo_connection["id"], str(item["mongoCollection"]), limit=50)
mongo_verification = {
"collection": item["mongoCollection"],
"documentCount": documents.get("total", 0),
"sampleDocumentIds": [doc.get("_id") for doc in documents.get("documents", [])[:10]],
}
if int(mongo_verification["documentCount"] or 0) <= 0:
raise base.RegressionError(f"mongo verification failed for collection {item['mongoCollection']}")
incremental_item = workflow_items_by_name["12-增量嗅探循环处理"]
incremental_workflow = incremental_item["workflow"]
seed_clip_keys = find_seed_clip_keys(config, connection["id"], output_root)
if not seed_clip_keys:
raise base.RegressionError(f"failed to find seed clips under {output_root}")
incremental_bucket, incremental_prefix = base.parse_bos_path(incremental_source_path)
configure_directory_incremental_workflow(
config,
incremental_workflow["id"],
incremental_source_path,
connection["id"],
int(incremental_item.get("streamingIntervalSeconds", 20) or 20),
)
round_summaries: list[dict[str, Any]] = []
previous_run_id = ""
for round_item in incremental_item.get("incrementalRounds", []):
created_directories = base.ensure_incremental_seed_directories(
config,
connection["id"],
incremental_prefix,
seed_clip_keys,
bucket=incremental_bucket,
round_name=str(round_item.get("name", "round")).strip() or "round",
directory_count=int(round_item.get("directoryCount", 0) or 0),
place=config.incremental_place,
camera_model=config.incremental_camera_model,
video_subdir=config.incremental_video_subdir,
)
if round_summaries:
time.sleep(2)
finished = base.wait_for_next_workflow_run(
config,
incremental_workflow["id"],
previous_run_id,
config.max_wait_seconds,
)
summary = base.summarize_run(finished)
summary["incrementalRound"] = round_item.get("name")
summary["createdDirectories"] = created_directories
summary["projectId"] = grouped_projects[incremental_item["targetGroup"]]["project"]["id"]
summary["projectName"] = grouped_projects[incremental_item["targetGroup"]]["project"]["name"]
results.append(summary)
round_summaries.append(summary)
previous_run_id = str(finished.get("id", "")).strip()
if finished["status"] != "completed":
raise base.RegressionError(
json.dumps(
{
"status": "incremental-workflow-failed",
"workflowName": incremental_workflow["name"],
"run": summary,
"runLogsTail": base.get_run_logs(config, finished["id"])[-30:],
},
ensure_ascii=False,
indent=2,
)
)
return {
"status": "completed",
"projects": [
{
"name": payload["project"]["name"],
"id": payload["project"]["id"],
"description": payload["description"],
}
for payload in grouped_projects.values()
],
"mongoCollection": mongo_collection,
"mongoVerification": mongo_verification,
"incrementalSourcePath": incremental_source_path,
"incrementalVerification": {
"seedClipKeys": seed_clip_keys,
"rounds": round_summaries,
},
"sourcePath": config.source_path,
"uploadRootPrefix": output_root,
"workflows": results,
}
def main() -> int:
config = base.parse_args()
base.validate_config(config)
source_text, source_json = load_source_variant_payloads()
if config.api_token:
print("using API token auth for WJZH multi-project regression requests")
base.login_if_configured(config)
if config.preflight_only:
payload = preflight_multi_project_regression(config, source_text, source_json)
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 0
payload = run_grouped_regression(config, source_text, source_json)
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except base.RegressionError as err:
print(str(err), file=sys.stderr)
raise SystemExit(1)