#!/usr/bin/env python3 from __future__ import annotations import importlib.util import json import os import sys import time from collections import OrderedDict from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parent BASE_PATH = ROOT / "wjzh_operator_regression.py" spec = importlib.util.spec_from_file_location("wjzh_regression_base", BASE_PATH) if spec is None or spec.loader is None: raise SystemExit(f"failed to load regression base from {BASE_PATH}") base = importlib.util.module_from_spec(spec) sys.modules[spec.name] = base spec.loader.exec_module(base) RUNNER_IMAGE_OVERRIDE = os.getenv("OPERATOR_RUNNER_IMAGE", "").strip() SOURCE_TEXT_ENV = "WJZH_REGRESSION_SOURCE_TEXT" SOURCE_TEXT_FILE_ENV = "WJZH_REGRESSION_SOURCE_TEXT_FILE" SOURCE_JSON_ENV = "WJZH_REGRESSION_SOURCE_JSON" SOURCE_JSON_FILE_ENV = "WJZH_REGRESSION_SOURCE_JSON_FILE" def read_required_env_or_file(value_env: str, file_env: str) -> str: value = os.getenv(value_env) file_path = os.getenv(file_env, "").strip() if value and file_path: raise base.RegressionError(f"set only one of {value_env} or {file_env}") if file_path: path = Path(file_path).expanduser() if not path.is_file(): raise base.RegressionError(f"{file_env} does not point to a readable file: {file_path}") value = path.read_text(encoding="utf-8") if value is None or not value.strip(): raise base.RegressionError(f"{value_env} or {file_env} is required for the multi-project SOURCE_TEXT/SOURCE_JSON regression") return value def load_source_variant_payloads() -> tuple[str, str]: source_text = read_required_env_or_file(SOURCE_TEXT_ENV, SOURCE_TEXT_FILE_ENV) source_json = read_required_env_or_file(SOURCE_JSON_ENV, SOURCE_JSON_FILE_ENV) try: parsed = json.loads(source_json) except json.JSONDecodeError as err: raise base.RegressionError(f"{SOURCE_JSON_ENV} / {SOURCE_JSON_FILE_ENV} must contain valid JSON: {err}") from err return source_text, json.dumps(parsed, ensure_ascii=False) def refresh_workflow_runner_image(config: Any, workflow_id: str, runner_image: str) -> dict[str, Any]: if not runner_image: return base.get_workflow(config, workflow_id) workflow = base.get_workflow(config, workflow_id) nodes = workflow.get("nodes", []) changed = False for node in nodes: data = node.get("data") or {} image = str(data.get("image", "")).strip() if "droboticflow-operator-runner:" not in image: continue if image == runner_image: continue data["image"] = runner_image changed = True if changed: workflow = base.update_workflow(config, workflow_id, {"nodes": nodes}) return workflow def build_group_projects(config: Any, timestamp: str) -> OrderedDict[str, dict[str, Any]]: groups: OrderedDict[str, dict[str, Any]] = OrderedDict( [ ( "基础数据与索引", { "description": "真实数据源、目录扫描、模型推理、BOS 扫描、索引、校验与上传回归。", }, ), ( "JSON与逻辑", { "description": "JSON/CSV/分流/合并等逻辑链路回归。", }, ), ( "审核与业务写入", { "description": "人工审核 approve/reject 分支及 Mongo 写入回归。", }, ), ( "视频处理与打包", { "description": "视频质检、过滤、切片、标注、打包、抽检回归。", }, ), ( "Robotics与增量", { "description": "LeRobot / ROSBag 概览和目录级增量循环处理回归。", }, ), ] ) for group_name, payload in groups.items(): payload["project"] = base.create_project( config, f"{config.project_prefix} {timestamp} {group_name}", payload["description"], ) return groups def group_name_for_workflow(workflow_name: str) -> str: if workflow_name.startswith(("00-", "01-")): return "基础数据与索引" if workflow_name.startswith("02-"): return "JSON与逻辑" if workflow_name.startswith(("03-", "04-", "05-")): return "审核与业务写入" if workflow_name.startswith(("06-", "07-", "08-", "10-", "11-")): return "视频处理与打包" if workflow_name.startswith(("09-", "12-")): return "Robotics与增量" raise base.RegressionError(f"unmapped workflow group for {workflow_name}") def clone_temp_workflows( config: Any, temp_project_id: str, temp_workflows: list[dict[str, Any]], grouped_projects: OrderedDict[str, dict[str, Any]], ) -> list[dict[str, Any]]: cloned: list[dict[str, Any]] = [] for item in temp_workflows: original = item["workflow"] target_group = group_name_for_workflow(str(original.get("name", ""))) target_project_id = grouped_projects[target_group]["project"]["id"] copy = base.clone_workflow(config, target_project_id, original["id"], original["name"]) copy = refresh_workflow_runner_image(config, copy["id"], RUNNER_IMAGE_OVERRIDE) cloned_item = dict(item) cloned_item["workflow"] = copy cloned_item["targetGroup"] = target_group cloned.append(cloned_item) base.request(config, "DELETE", f"/api/projects/{temp_project_id}", expected=(204,)) return cloned def create_source_variants_workflow( config: Any, project_id: str, operators_by_name: dict[str, dict[str, Any]], connection_id: str, output_root: str, source_text: str, source_json: str, ) -> dict[str, Any]: nodes = [ base.build_node( operators_by_name["真实数据源"], "source_text", 0, -90, {"SOURCE_TEXT": source_text}, connection_id=connection_id, label="真实数据源-文本", ), base.build_node( operators_by_name["模型推理"], "infer_text", 240, -90, {"COMMAND": base.model_inference_command()}, ), base.upload_node( operators_by_name["结果上传"], connection_id, "upload_text", 480, -90, f"{output_root}/00-source-variants/text", ), base.build_node( operators_by_name["真实数据源"], "source_json", 0, 80, {"SOURCE_JSON": source_json}, connection_id=connection_id, label="真实数据源-JSON", ), base.upload_node( operators_by_name["结果上传"], connection_id, "upload_json", 240, 80, f"{output_root}/00-source-variants/json", ), ] edges = [ base.upload_edge("source_text", "infer_text", "source-text-infer"), base.upload_edge("infer_text", "upload_text", "infer-text-upload"), base.upload_edge("source_json", "upload_json", "source-json-upload"), ] workflow = base.create_workflow( config, project_id, "00-真实数据源多入口", "使用调用方显式提供的真实文本和 JSON payload 覆盖 SOURCE_TEXT / SOURCE_JSON 子功能。", nodes, edges, ) return refresh_workflow_runner_image(config, workflow["id"], RUNNER_IMAGE_OVERRIDE) def create_manual_sampling_workflow( config: Any, project_id: str, operators_by_name: dict[str, dict[str, Any]], connection_id: str, output_root: str, source_path: str, ) -> dict[str, Any]: nodes = [ base.build_node( operators_by_name["BOS 增量扫描"], "source", 0, 0, {"SOURCE_PATH": source_path, "SCAN_MODE": "FULL"}, connection_id=connection_id, ), base.build_node(operators_by_name["X 平台数据集索引"], "index", 220, 0), base.build_node( operators_by_name["采集目录校验"], "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}, ), base.build_node( operators_by_name["Shell 命令执行"], "batch_pick", 660, 0, {"COMMAND": base.cap_duration_command(base.int_config(config.batch_window_seconds), max_items=4, max_duration=500)}, label="人工抽检前抽样", ), base.build_node( operators_by_name["视频质检"], "quality", 880, -60, base.quality_hyperparams(config), ), base.build_node( operators_by_name["视频切片"], "slice", 1100, -60, base.segment_hyperparams(config.batch_segment_duration_seconds), ), base.build_node( operators_by_name["视频标注"], "annotate", 1320, -60, base.annotation_hyperparams(config), ), base.build_node(operators_by_name["成品打包"], "package", 1540, -60), base.build_node( operators_by_name["抽检采样"], "manual_sample", 1760, -60, {"RULES_JSON": config.sampling_rules_json}, label="人工抽检采样", ), base.upload_node( operators_by_name["结果上传"], connection_id, "manual_sample_upload", 1980, -60, f"{output_root}/11b-manual-sampling/sample", ), ] edges = [ base.upload_edge("source", "index", "source-index"), base.upload_edge("index", "validate", "index-validate"), base.upload_edge("validate", "batch_pick", "validate-batch-pick"), base.upload_edge("batch_pick", "quality", "batch-pick-quality"), base.upload_edge("quality", "slice", "quality-slice"), base.upload_edge("slice", "annotate", "slice-annotate"), base.upload_edge("annotate", "package", "annotate-package"), base.upload_edge("package", "manual_sample", "package-manual-sample"), base.upload_edge("manual_sample", "manual_sample_upload", "manual-sample-upload"), ] workflow = base.create_workflow( config, project_id, "11B-人工抽检采样", "使用人工抽检采样算子对成品包执行规则采样回归。", nodes, edges, ) return refresh_workflow_runner_image(config, workflow["id"], RUNNER_IMAGE_OVERRIDE) def create_multi_project_regression( config: Any, source_text: str, source_json: str, ) -> tuple[OrderedDict[str, dict[str, Any]], list[dict[str, Any]], str, str, str]: settings = base.request(config, "GET", "/api/settings") operators = base.request(config, "GET", "/api/operators")["items"] connection = base.ensure_storage_connection(config) mongo_connection = base.find_mongo_connection(config) base.validate_preflight(config, connection, operators, settings, mongo_connection) operators_by_name = {item["name"]: item for item in operators} base.delete_projects(config) timestamp = time.strftime("%Y%m%d%H%M%S") temp_project_id, temp_workflows, mongo_collection, output_root, incremental_source_path = base.create_regression_workflows( config, connection["id"], mongo_connection["id"] if mongo_connection else "", operators_by_name, ) grouped_projects = build_group_projects(config, timestamp) cloned = clone_temp_workflows(config, temp_project_id, temp_workflows, grouped_projects) extras: list[dict[str, Any]] = [] extras.append( { "workflow": create_source_variants_workflow( config, grouped_projects["基础数据与索引"]["project"]["id"], operators_by_name, connection["id"], output_root, source_text, source_json, ), "targetGroup": "基础数据与索引", } ) extras.append( { "workflow": create_manual_sampling_workflow( config, grouped_projects["视频处理与打包"]["project"]["id"], operators_by_name, connection["id"], output_root, config.source_path, ), "targetGroup": "视频处理与打包", } ) return grouped_projects, cloned + extras, mongo_collection, output_root, incremental_source_path def configure_directory_incremental_workflow( config: Any, workflow_id: str, incremental_source_path: str, connection_id: str, interval_seconds: int, ) -> None: scan_bucket, scan_prefix = base.parse_bos_path(incremental_source_path) payload: dict[str, Any] = { "kind": "scan", "maxConcurrentRuns": 1, "incrementalUnit": "DIRECTORY", "directoryDepth": 1, "scanConfig": { "enabled": True, "connectionId": connection_id, "dataSource": "BOS", "sourcePath": incremental_source_path, "bucket": scan_bucket, "prefix": scan_prefix, "scanMode": "INCREMENTAL", }, "type": "streaming", "schedule": { "interval": interval_seconds, }, } base.update_workflow(config, workflow_id, payload) def find_seed_clip_keys(config: Any, connection_id: str, output_root: str) -> list[str]: seed_clip_keys: list[str] = [] for suffix in ["06-medium-video/slice", "07-full-chain/slice", "11-batch-fullchain/slice"]: seed_clip_keys.extend(base.list_uploaded_mp4_keys(config, connection_id, output_root, suffix)) return list(dict.fromkeys([key for key in seed_clip_keys if key])) def preflight_multi_project_regression(config: Any, source_text: str, source_json: str) -> dict[str, Any]: settings = base.request(config, "GET", "/api/settings") operators = base.request(config, "GET", "/api/operators")["items"] connection = base.ensure_storage_connection(config) mongo_connection = base.find_mongo_connection(config) base.validate_preflight(config, connection, operators, settings, mongo_connection) return { "status": "ok", "mode": "preflight-only", "connectionId": connection["id"], "mongoConnectionId": mongo_connection["id"] if mongo_connection else "", "sourcePath": config.source_path, "uploadRootPrefix": config.upload_root_prefix, "sourceTextBytes": len(source_text.encode("utf-8")), "sourceJsonBytes": len(source_json.encode("utf-8")), "roboticsSourcePaths": { "lerobot": config.lerobot_source_path, "rosbag": config.rosbag_source_path, }, } def run_grouped_regression(config: Any, source_text: str, source_json: str) -> dict[str, Any]: grouped_projects, workflows, mongo_collection, output_root, incremental_source_path = create_multi_project_regression( config, source_text, source_json, ) connection = base.ensure_storage_connection(config) mongo_connection = base.find_mongo_connection(config) workflow_items_by_name = {item["workflow"]["name"]: item for item in workflows} execution_order = [ "00-源数据目录推理", "00-真实数据源多入口", "01-索引校验双上传", "02-JSON工具链多分支", "03-人工审核通过", "04-人工审核拒绝", "05-Mongo写入", "06-中大型视频切片", "07-超大视频全链路", "08-多目录批量窗口化", "09-Robotics真实概览", "10-多目录多视频批量质检", "11-多目录多视频批量全链路", "11B-人工抽检采样", ] results: list[dict[str, Any]] = [] mongo_verification: dict[str, Any] | None = None for name in execution_order: item = workflow_items_by_name[name] workflow = item["workflow"] rounds = max(int(item.get("rounds", 1) or 1), 1) for round_index in range(rounds): current_workflow = workflow if round_index > 0: group_name = item["targetGroup"] current_workflow = base.clone_workflow( config, grouped_projects[group_name]["project"]["id"], workflow["id"], f"{workflow['name']} 第{round_index + 1}轮", ) current_workflow = refresh_workflow_runner_image(config, current_workflow["id"], RUNNER_IMAGE_OVERRIDE) finished = base.run_workflow_until_complete( config, current_workflow["id"], config.max_wait_seconds, audit_decision=str(item.get("auditDecision", "")).strip(), ) summary = base.summarize_run(finished) summary["projectId"] = grouped_projects[item["targetGroup"]]["project"]["id"] summary["projectName"] = grouped_projects[item["targetGroup"]]["project"]["name"] if rounds > 1: summary["round"] = round_index + 1 if item.get("auditDecision"): summary["auditDecision"] = item["auditDecision"] results.append(summary) if finished["status"] != "completed": raise base.RegressionError( json.dumps( { "status": "workflow-failed", "workflowName": current_workflow["name"], "run": summary, "runLogsTail": base.get_run_logs(config, finished["id"])[-30:], }, ensure_ascii=False, indent=2, ) ) if item.get("mongoCollection"): documents = base.query_business_collection(config, mongo_connection["id"], str(item["mongoCollection"]), limit=50) mongo_verification = { "collection": item["mongoCollection"], "documentCount": documents.get("total", 0), "sampleDocumentIds": [doc.get("_id") for doc in documents.get("documents", [])[:10]], } if int(mongo_verification["documentCount"] or 0) <= 0: raise base.RegressionError(f"mongo verification failed for collection {item['mongoCollection']}") incremental_item = workflow_items_by_name["12-增量嗅探循环处理"] incremental_workflow = incremental_item["workflow"] seed_clip_keys = find_seed_clip_keys(config, connection["id"], output_root) if not seed_clip_keys: raise base.RegressionError(f"failed to find seed clips under {output_root}") incremental_bucket, incremental_prefix = base.parse_bos_path(incremental_source_path) configure_directory_incremental_workflow( config, incremental_workflow["id"], incremental_source_path, connection["id"], int(incremental_item.get("streamingIntervalSeconds", 20) or 20), ) round_summaries: list[dict[str, Any]] = [] previous_run_id = "" for round_item in incremental_item.get("incrementalRounds", []): created_directories = base.ensure_incremental_seed_directories( config, connection["id"], incremental_prefix, seed_clip_keys, bucket=incremental_bucket, round_name=str(round_item.get("name", "round")).strip() or "round", directory_count=int(round_item.get("directoryCount", 0) or 0), place=config.incremental_place, camera_model=config.incremental_camera_model, video_subdir=config.incremental_video_subdir, ) if round_summaries: time.sleep(2) finished = base.wait_for_next_workflow_run( config, incremental_workflow["id"], previous_run_id, config.max_wait_seconds, ) summary = base.summarize_run(finished) summary["incrementalRound"] = round_item.get("name") summary["createdDirectories"] = created_directories summary["projectId"] = grouped_projects[incremental_item["targetGroup"]]["project"]["id"] summary["projectName"] = grouped_projects[incremental_item["targetGroup"]]["project"]["name"] results.append(summary) round_summaries.append(summary) previous_run_id = str(finished.get("id", "")).strip() if finished["status"] != "completed": raise base.RegressionError( json.dumps( { "status": "incremental-workflow-failed", "workflowName": incremental_workflow["name"], "run": summary, "runLogsTail": base.get_run_logs(config, finished["id"])[-30:], }, ensure_ascii=False, indent=2, ) ) return { "status": "completed", "projects": [ { "name": payload["project"]["name"], "id": payload["project"]["id"], "description": payload["description"], } for payload in grouped_projects.values() ], "mongoCollection": mongo_collection, "mongoVerification": mongo_verification, "incrementalSourcePath": incremental_source_path, "incrementalVerification": { "seedClipKeys": seed_clip_keys, "rounds": round_summaries, }, "sourcePath": config.source_path, "uploadRootPrefix": output_root, "workflows": results, } def main() -> int: config = base.parse_args() base.validate_config(config) source_text, source_json = load_source_variant_payloads() if config.api_token: print("using API token auth for WJZH multi-project regression requests") base.login_if_configured(config) if config.preflight_only: payload = preflight_multi_project_regression(config, source_text, source_json) print(json.dumps(payload, ensure_ascii=False, indent=2)) return 0 payload = run_grouped_regression(config, source_text, source_json) print(json.dumps(payload, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": try: raise SystemExit(main()) except base.RegressionError as err: print(str(err), file=sys.stderr) raise SystemExit(1)