diff --git a/.env.example b/.env.example index 18e5d42c..1375382b 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,15 @@ PORT=3001 WORKER_PORT=3002 -MONGO_URI=mongodb://127.0.0.1:27017 +# For Kubernetes deployment, set this to an external MongoDB URI. +MONGO_URI=mongodb://:@:27017/ MONGO_DB=droboticflow MONGO_CONNECT_RETRIES=30 MONGO_CONNECT_RETRY_DELAY_MS=2000 WORKER_POLL_INTERVAL_MS=2000 -K8S_ENABLED=false K8S_NAMESPACE=droboticflow K8S_IMAGE_PULL_SECRET_NAME=droboticflow-registry +WORKSPACE_PVC_NAME=droboticflow-workspace +WORKSPACE_MOUNT_PATH=/workspace OPERATOR_RUNNER_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:latest VITE_API_BASE_URL= AI_API_BASE_URL=https://coding.dashscope.aliyuncs.com/v1 diff --git a/Dockerfile.api b/Dockerfile.api index 9fdf0219..2aeaa116 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -23,7 +23,7 @@ WORKDIR /app ENV PORT=3001 ENV DIST_DIR=/app/dist -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* COPY --from=web-build /app/dist ./dist COPY --from=go-build /out/droboticflow-api /usr/local/bin/droboticflow-api diff --git a/Dockerfile.operator-runner b/Dockerfile.operator-runner index 8f9fd242..c054caea 100644 --- a/Dockerfile.operator-runner +++ b/Dockerfile.operator-runner @@ -2,6 +2,9 @@ FROM python:3.11-slim AS runtime WORKDIR /app ENV PYTHONUNBUFFERED=1 +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir numpy opencv-python-headless bce-python-sdk pymongo + COPY backend/operator-runner/main.py ./main.py CMD ["python", "/app/main.py"] diff --git a/README.md b/README.md index 74509c38..089621f6 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# DRoboticFlow +# Drobodata -DRoboticFlow is a workflow orchestration platform for embodied AI and vision pipelines. This repository now contains: +Drobodata is a workflow orchestration platform for embodied AI and vision pipelines. This repository now contains: - a React/Vite frontend - a Go-based MongoDB-backed REST API - a Go-based worker service that executes workflow DAGs - a Python-based Kubernetes Job operator runner -- deployment assets for MongoDB, Kubernetes, and Baidu Cloud Container Registry +- deployment assets for Kubernetes and Baidu Cloud Container Registry ## Architecture @@ -65,6 +65,7 @@ DRoboticFlow is a workflow orchestration platform for embodied AI and vision pip - `npm run verify`: lint + test + build - `npm run smoke`: local smoke test against the running API and worker - `npm run reconcile:data -- `: remove legacy demo data, reset settings, and bootstrap built-in operators through the real API +- `python3 scripts/wjzh_operator_regression.py --preflight-only`: preflight the production-style WJZH real-data operator regression against a live API ## Runtime Configuration @@ -77,9 +78,10 @@ Key environment variables: - `MONGO_CONNECT_RETRIES`: startup retry count for API and worker Mongo connections - `MONGO_CONNECT_RETRY_DELAY_MS`: delay between Mongo connection retries - `WORKER_POLL_INTERVAL_MS`: worker polling interval -- `K8S_ENABLED`: enable Kubernetes Job execution in the worker - `K8S_NAMESPACE`: namespace used for Jobs - `K8S_IMAGE_PULL_SECRET_NAME`: image pull secret injected into operator-runner Jobs +- `WORKSPACE_PVC_NAME`: shared PVC used by worker, API, and operator-runner Jobs +- `WORKSPACE_MOUNT_PATH`: shared workspace mount path, default `/workspace` - `OPERATOR_RUNNER_IMAGE`: container image used for node execution Jobs - `AI_API_BASE_URL`: OpenAI-compatible DashScope endpoint used by server-side AI generation, default `https://coding.dashscope.aliyuncs.com/v1` - `AI_MODEL`: server-side model name, default `glm-5` @@ -87,7 +89,9 @@ Key environment variables: - `VITE_API_BASE_URL`: optional frontend API base URL override The frontend never reads the AI key directly. Workflow generation and operator generation are proxied through the Go API so secrets stay on the server side. -The settings page can update K8s access details, the worker defaults, the global `timezone`, `provider`, `baseURL`, `model`, and the server-side AI key. `GET /api/settings` only returns whether secrets exist via `k8s.hasKubeconfig` / `ai.hasAPIKey`; it does not echo plaintext kubeconfig or API keys back to the browser. Timestamps in the UI are rendered in the configured IANA timezone with compact minute-level formatting for cards, lists, and summaries, while logs keep finer granularity when needed. OSS-related bucket and path settings belong on OSS operators in workflows, not in the global system settings page. The OSS browser APIs also now require the current node `SOURCE_PATH` and only list content inside that node-scoped root. +The settings page can update K8s access details, the worker defaults, the global `timezone`, `provider`, `baseURL`, `model`, and the server-side AI key. `GET /api/settings` only returns whether secrets exist via `k8s.hasKubeconfig` / `ai.hasAPIKey`; it does not echo plaintext kubeconfig or API keys back to the browser. Timestamps in the UI are rendered in the configured IANA timezone with compact minute-level formatting for cards, lists, and summaries, while logs keep finer granularity when needed. OSS-related bucket and path settings belong on OSS operators in workflows, not in the global system settings page. Scan workflows can now reference reusable BOS storage connections via `scanConfig.connectionId`, so endpoint and AK/SK are stored once in `/api/storage/connections` and resolved by the API/worker at preview and execution time instead of being duplicated into each workflow. + +The Kubernetes runtime now assumes a shared workspace PVC. Worker writes per-node `input-artifacts.json` files into the shared workspace, operator-runner Jobs read those files through `INPUT_ARTIFACTS_PATH`, then persist `result.json` and `stage-manifest.json` back into the same PVC. Large artifact lists no longer need to travel through giant env vars or stdout payloads. Workflow execution concurrency now has two levels: @@ -117,7 +121,26 @@ Workflow templates now carry the same workflow-level defaults as project workflo Artifact previews are persisted with the run artifacts themselves. JSON artifacts keep their structured payload in `artifact.data`, while non-JSON artifacts can attach a compact `artifact.data.preview` block for text, CSV/TSV, JSONL, images, audio/video metadata, PDF/document hints, SQLite/rosbag summaries, and LeRobot dataset summaries. Artifacts also carry `portId` / `portName`, which lets the worker route multi-output operators by edge handle instead of mixing all outputs together. -The built-in operator catalog now includes real data-processing and robotics helpers on top of the original source/merge/upload/audit flow: `目录扫描`, `CSV 转 JSON`, `JSON 转 CSV`, `JSON 字段提取`, `JSON 过滤`, `JSON 映射重命名`, `文本模板渲染`, `条件分流`, `LeRobot 数据集概览`, `ROSBag 数据概览`, and `Shell 命令执行`. +The built-in operator catalog now includes real data-processing, X-platform video pipeline, and robotics helpers on top of the original source/merge/upload/audit flow: `BOS 增量扫描`, `X 平台数据集索引`, `采集目录校验`, `视频质检`, `数据过滤`, `视频切片`, `视频标注`, `成品打包`, `抽检采样`, `目录扫描`, `CSV 转 JSON`, `JSON 转 CSV`, `JSON 字段提取`, `JSON 过滤`, `JSON 映射重命名`, `文本模板渲染`, `条件分流`, `LeRobot 数据集概览`, `ROSBag 数据概览`, and `Shell 命令执行`. + +`BOS 增量扫描` is a stateful source operator. The workflow trigger still controls whether a workflow runs once, on a timer, or in a streaming loop, but the operator itself persists a per-`workflowId/nodeId/sourceFingerprint` checkpoint. Each run lists the configured BOS prefix, emits only unseen objects, and the worker commits the updated checkpoint only after the run completes successfully. Failed downstream runs do not advance the checkpoint, so the same increment can be retried safely. + +The frontend now exposes a unified `数据资产` module backed by real APIs, while scan-oriented definitions live inside the workflow system project `数据源扫描` instead of a separate first-level page: + +- `源数据`: source collections indexed by completed workflows that produced a real `dataset-index.json` +- `成品数据`: finished clip packages aggregated from `packages-manifest.json` +- `抽检任务`: persisted human review queues synced from `audit-samples.json` + +`数据资产` groups `源数据` and `成品数据` as two tabs instead of two unrelated menu entries: + +- `源数据` shows upstream collected data and its real metadata such as `meta.json` / `video_meta.json` +- `成品数据` shows downstream outputs produced by project workflows, with provenance back to `project / workflow / run` +- both tabs accept project/workflow/run scope filters so project detail and workflow editor can deep-link into the relevant assets + +`抽检任务` and the existing workflow `任务审核` share the same manual-review idea, but they serve different phases: + +- `任务审核`: approval gates during workflow execution +- `抽检任务`: post-processing quality review over completed data packages ## Deployment @@ -135,7 +158,7 @@ Build and push images: ./scripts/build-and-push-images.sh ``` -If the target cluster cannot pull the upstream `mongo:7` image from Docker Hub, mirror it into the same Baidu registry and set `MONGO_IMAGE` during deployment. +Kubernetes deployment requires an explicit external `MONGO_URI`. The main manifest and deploy script no longer create or manage a cluster-local MongoDB instance. Render Kubernetes manifests: @@ -146,6 +169,7 @@ Render Kubernetes manifests: Deploy to Kubernetes: ```bash +export MONGO_URI=mongodb://:@:27017/ ./scripts/deploy-k8s.sh ``` @@ -171,4 +195,13 @@ See the detailed guides: - [`docs/api.md`](/Users/longtaowu/workspace/droboticflow/docs/api.md) - [`docs/deployment.md`](/Users/longtaowu/workspace/droboticflow/docs/deployment.md) +- [`docs/platform-usage-manual.zh-CN.md`](/Users/longtaowu/workspace/droboticflow/docs/platform-usage-manual.zh-CN.md) +- [`docs/custom-operator-manual.zh-CN.md`](/Users/longtaowu/workspace/droboticflow/docs/custom-operator-manual.zh-CN.md) +- [`docs/custom-operators/README.md`](/Users/longtaowu/workspace/droboticflow/docs/custom-operators/README.md) +- [`docs/builtin-operators-manual.zh-CN.md`](/Users/longtaowu/workspace/droboticflow/docs/builtin-operators-manual.zh-CN.md) +- [`docs/operators/README.md`](/Users/longtaowu/workspace/droboticflow/docs/operators/README.md) +- [`docs/operator-authoring.md`](/Users/longtaowu/workspace/droboticflow/docs/operator-authoring.md) - [`docs/operations.md`](/Users/longtaowu/workspace/droboticflow/docs/operations.md) +- [`docs/wjzh-operator-regression.md`](/Users/longtaowu/workspace/droboticflow/docs/wjzh-operator-regression.md) +- [`docs/feature-overview-playwright-validation-2026-04-15.md`](/Users/longtaowu/workspace/droboticflow/docs/feature-overview-playwright-validation-2026-04-15.md) +- [`docs/feature-overview-playwright-validation-2026-04-15.html`](/Users/longtaowu/workspace/droboticflow/docs/feature-overview-playwright-validation-2026-04-15.html) diff --git a/backend/operator-runner/main.py b/backend/operator-runner/main.py index 24198b4e..f4470a32 100644 --- a/backend/operator-runner/main.py +++ b/backend/operator-runner/main.py @@ -2,6 +2,7 @@ from __future__ import annotations import datetime as dt +import contextlib import fnmatch import hashlib import json @@ -10,17 +11,23 @@ import os import pathlib import base64 import csv +import math import re import resource import shutil import sqlite3 import subprocess import sys +import tarfile import time +import zipfile from typing import Any +from urllib import error, request +from urllib.parse import urlparse RESULT_PREFIX = "DROBOTICFLOW_RESULT=" +RESULT_PATH_PREFIX = "DROBOTICFLOW_RESULT_PATH=" TEXT_FILE_SUFFIXES = { ".txt", ".log", @@ -36,11 +43,42 @@ IMAGE_FILE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"} VIDEO_FILE_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"} AUDIO_FILE_SUFFIXES = {".wav", ".mp3", ".aac", ".m4a", ".ogg", ".flac"} ROSBAG_FILE_SUFFIXES = {".bag", ".db3", ".mcap"} +ARCHIVE_FILE_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".zip") +XPLATFORM_VIDEO_META_FILES = ("video_meta.json", "video_metadata.json") +XPLATFORM_INLINE_JSON_FILES = {"meta.json", "intrinsics.json", *XPLATFORM_VIDEO_META_FILES} PREVIEW_TEXT_LIMIT = 12_000 PREVIEW_TEXT_LINE_LIMIT = 240 PREVIEW_TABLE_ROW_LIMIT = 24 PREVIEW_FILE_LIST_LIMIT = 40 PREVIEW_INLINE_BINARY_LIMIT = 512 * 1024 +DATASET_PARTITION_SIZE = 500 +QUALITY_SAMPLE_LIMIT = 48 +QUALITY_ANALYSIS_MAX_DIMENSION = 640 +QUALITY_SKIN_COMPONENT_MIN_RATIO = 0.002 +RESERVED_MONGO_COLLECTIONS = { + "projects", + "operators", + "workflows", + "templates", + "runs", + "sourceStates", + "auditTasks", + "samplingTasks", + "users", + "storageConnections", + "settings", +} +DATASET_PARTITION_KEYS = ( + "items", + "entries", + "rows", + "collections", + "catalogCollections", + "catalogItems", + "catalogEntries", +) +PAYLOAD_FILE_READ_ATTEMPTS = 60 +PAYLOAD_FILE_READ_DELAY_SECONDS = 1.0 def now_iso() -> str: @@ -58,6 +96,275 @@ def load_json_env(name: str, default: Any) -> Any: return json.loads(value) +def load_json_file(path: str, default: Any) -> Any: + raw = str(path or "").strip() + if not raw: + return default + candidate = pathlib.Path(raw) + if not candidate.exists(): + return default + return json.loads(candidate.read_text(encoding="utf-8")) + + +def write_json_file(path: str | pathlib.Path, value: Any) -> pathlib.Path: + candidate = pathlib.Path(path) + candidate.parent.mkdir(parents=True, exist_ok=True) + candidate.write_text(json.dumps(json_compatible(value), ensure_ascii=False), encoding="utf-8") + return candidate + + +def text_value(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value or "") + + +def json_compatible(value: Any) -> Any: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, dict): + return {str(key): json_compatible(item) for key, item in value.items()} + if isinstance(value, list): + return [json_compatible(item) for item in value] + if isinstance(value, tuple): + return [json_compatible(item) for item in value] + return value + + +def load_input_artifacts() -> list[dict[str, Any]]: + input_artifacts_path = os.getenv("INPUT_ARTIFACTS_PATH", "").strip() + if input_artifacts_path: + loaded = load_json_file(input_artifacts_path, []) + if isinstance(loaded, list): + return json_compatible(loaded) + loaded = load_json_env("INPUT_ARTIFACTS_JSON", []) + if isinstance(loaded, list): + return json_compatible(loaded) + return [] + + +def stage_manifest_payload( + run_id: str, + workflow_id: str, + node_id: str, + node_label: str, + output_dir: pathlib.Path, + artifacts: list[dict[str, Any]], +) -> dict[str, Any]: + manifest_artifacts: list[dict[str, Any]] = [] + data_refs: list[dict[str, Any]] = [] + for artifact in artifacts: + item = { + "id": text_value(artifact.get("id")), + "name": text_value(artifact.get("name")), + "type": text_value(artifact.get("type")), + "path": text_value(artifact.get("path")), + "size": text_value(artifact.get("size")), + } + if text_value(artifact.get("portId")): + item["portId"] = text_value(artifact.get("portId")) + if text_value(artifact.get("portName")): + item["portName"] = text_value(artifact.get("portName")) + if isinstance(artifact.get("metadata"), dict) and artifact["metadata"]: + item["metadata"] = json_compatible(artifact["metadata"]) + data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} + data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {} + if data_ref: + item["dataRef"] = json_compatible(data_ref) + data_refs.append( + { + "artifactId": item["id"], + "artifactName": item["name"], + "portId": item.get("portId", ""), + **json_compatible(data_ref), + } + ) + manifest_artifacts.append(item) + return { + "runId": run_id, + "workflowId": workflow_id, + "nodeId": node_id, + "nodeLabel": node_label, + "outputDir": str(output_dir), + "artifactCount": len(manifest_artifacts), + "artifacts": manifest_artifacts, + "dataRefs": data_refs, + "primaryDataRefs": data_refs, + "generatedAt": now_iso(), + } + + +def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path: + return output_dir.parent / "result.json" + + +def default_stage_manifest_path(output_dir: pathlib.Path) -> pathlib.Path: + return output_dir.parent / "stage-manifest.json" + + +def normalize_json_payload(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return json_compatible(value) + if isinstance(value, list): + return {"items": json_compatible(value), "itemCount": len(value)} + return {"value": json_compatible(value)} + + +def bos_prefix(value: str) -> str: + return str(value or "").strip().strip("/") + + +def canonical_bos_path(bucket: str, key: str) -> str: + normalized_bucket = str(bucket or "").strip() + normalized_key = str(key or "").strip().lstrip("/") + if not normalized_bucket: + return "" + if not normalized_key: + return f"bos://{normalized_bucket}" + return f"bos://{normalized_bucket}/{normalized_key}" + + +def parse_bos_path(value: str) -> tuple[str, str]: + raw = str(value or "").strip() + if not raw.startswith("bos://"): + return "", "" + bucket_and_key = raw[len("bos://") :] + if not bucket_and_key: + return "", "" + bucket, _, key = bucket_and_key.partition("/") + return bucket.strip(), bos_prefix(key) + + +def is_xplatform_video_meta_name(value: str) -> bool: + return str(value or "").strip().lower() in XPLATFORM_VIDEO_META_FILES + + +def load_collection_video_meta(collection_dir: pathlib.Path) -> tuple[dict[str, dict[str, Any]], str]: + for candidate in XPLATFORM_VIDEO_META_FILES: + payload = optional_json_file(collection_dir / candidate) + if payload is not None: + return parse_video_meta_rows(payload), candidate + return {}, "" + + +def sampled_frame_positions(frame_count: int, sample_limit: int = QUALITY_SAMPLE_LIMIT) -> list[int]: + if frame_count <= 0: + return [] + if frame_count <= sample_limit: + return list(range(frame_count)) + last_frame = max(frame_count - 1, 0) + positions = { + min(last_frame, max(0, int(round(index * last_frame / max(sample_limit - 1, 1))))) + for index in range(sample_limit) + } + return sorted(positions) + + +def resize_quality_frame(frame: Any, max_dimension: int = QUALITY_ANALYSIS_MAX_DIMENSION) -> Any: + try: + height, width = frame.shape[:2] + except Exception: + return frame + longest_edge = max(width, height) + if longest_edge <= 0 or longest_edge <= max_dimension: + return frame + scale = max_dimension / float(longest_edge) + target_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale)))) + cv2, _ = require_cv2() + return cv2.resize(frame, target_size, interpolation=cv2.INTER_AREA) + + +def build_skin_mask(frame: Any) -> Any: + cv2, np = require_cv2() + ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb) + ycrcb_mask = cv2.inRange( + ycrcb, + np.array((0, 133, 77), dtype=np.uint8), + np.array((255, 173, 127), dtype=np.uint8), + ) + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + hsv_mask = cv2.inRange( + hsv, + np.array((0, 20, 40), dtype=np.uint8), + np.array((25, 220, 255), dtype=np.uint8), + ) + return cv2.bitwise_and(ycrcb_mask, hsv_mask) + + +def largest_skin_component_ratio(mask: Any, min_component_ratio: float = QUALITY_SKIN_COMPONENT_MIN_RATIO) -> float: + cv2, np = require_cv2() + if mask is None: + return 0.0 + total_area = int(mask.shape[0] * mask.shape[1]) if len(mask.shape) >= 2 else 0 + if total_area <= 0: + return 0.0 + kernel = np.ones((5, 5), dtype=np.uint8) + cleaned = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel) + cleaned = cv2.morphologyEx(cleaned, cv2.MORPH_CLOSE, kernel) + component_threshold = max(1, int(total_area * max(min_component_ratio, 0.0))) + count, _, stats, _ = cv2.connectedComponentsWithStats(cleaned) + largest_area = 0 + for index in range(1, count): + area = int(stats[index, cv2.CC_STAT_AREA]) + if area < component_threshold: + continue + if area > largest_area: + largest_area = area + return float(largest_area / total_area) + + +def motion_jitter_ratio(shifts: list[tuple[float, float]], diagonal: float) -> float: + if not shifts: + return 0.0 + _, np = require_cv2() + normalized_diagonal = max(diagonal, 1.0) + dx_values = [item[0] for item in shifts] + dy_values = [item[1] for item in shifts] + median_dx = float(np.median(dx_values)) + median_dy = float(np.median(dy_values)) + jitters = [ + math.sqrt((dx - median_dx) ** 2 + (dy - median_dy) ** 2) / normalized_diagonal + for dx, dy in shifts + ] + return sum(jitters) / max(len(jitters), 1) + + +def bos_key_join(prefix: str, relative_path: str) -> str: + normalized_prefix = bos_prefix(prefix) + normalized_relative = str(relative_path or "").strip().lstrip("/") + if not normalized_prefix: + return normalized_relative + if not normalized_relative: + return normalized_prefix + return f"{normalized_prefix}/{normalized_relative}" + + +def require_bos_sdk(): + try: + import baidubce.protocol as protocol # type: ignore + from baidubce.auth.bce_credentials import BceCredentials # type: ignore + from baidubce.bce_client_configuration import BceClientConfiguration # type: ignore + from baidubce.services.bos.bos_client import BosClient # type: ignore + except Exception as error: # pragma: no cover - runtime environment specific + raise RuntimeError("BOS 扫描需要 bce-python-sdk 运行时") from error + return protocol, BceCredentials, BceClientConfiguration, BosClient + + +def build_bos_client(endpoint: str, access_key_id: str, secret_access_key: str): + protocol_module, BceCredentials, BceClientConfiguration, BosClient = require_bos_sdk() + parsed = urlparse(endpoint if "://" in endpoint else f"https://{endpoint}") + scheme = (parsed.scheme or "https").lower() + host = parsed.netloc or parsed.path + if not host: + raise RuntimeError("BOS endpoint 不能为空") + config = BceClientConfiguration( + credentials=BceCredentials(access_key_id, secret_access_key), + endpoint=host, + protocol=protocol_module.HTTPS if scheme == "https" else protocol_module.HTTP, + ) + return BosClient(config), scheme + + def resource_snapshot() -> tuple[float, int]: usage = resource.getrusage(resource.RUSAGE_SELF) cpu_seconds = usage.ru_utime + usage.ru_stime @@ -91,6 +398,9 @@ def format_size(num_bytes: int) -> str: return f"{num_bytes / (1024 * 1024 * 1024):.2f} GiB" +MAX_SHA256_FILE_BYTES = 64 * 1024 * 1024 + + def sha256_file(path: pathlib.Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -99,6 +409,171 @@ def sha256_file(path: pathlib.Path) -> str: return digest.hexdigest() +def artifact_sha256(path: pathlib.Path, size_bytes: int) -> tuple[str, dict[str, Any]]: + if size_bytes > MAX_SHA256_FILE_BYTES: + return "", {"sha256Skipped": True, "sha256SkipReason": "size-limit", "sha256SkipBytes": size_bytes} + return sha256_file(path), {} + + +def require_cv2() -> tuple[Any, Any]: + try: + import cv2 # type: ignore + import numpy as np # type: ignore + except Exception as error: # pragma: no cover - environment specific + raise RuntimeError("视频算子需要 opencv-python-headless 和 numpy 运行时") from error + return cv2, np + + +def data_url_for_path(path: pathlib.Path, mime_type: str) -> str: + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + return f"data:{mime_type};base64,{encoded}" + + +def decode_data_url(url: str) -> tuple[str, bytes] | None: + if not url.startswith("data:") or "," not in url: + return None + header, encoded = url.split(",", 1) + if ";base64" not in header: + return None + mime_type = header[5:].split(";", 1)[0] or "application/octet-stream" + try: + return mime_type, base64.b64decode(encoded) + except Exception: + return None + + +def download_remote_source_to_path(url: str, path: pathlib.Path) -> pathlib.Path | None: + parsed = urlparse(url) + if parsed.scheme.lower() not in {"http", "https"}: + return None + path.parent.mkdir(parents=True, exist_ok=True) + remote_request = request.Request( + url, + headers={ + "User-Agent": "DroboticFlow operator-runner/1.0", + }, + ) + try: + with request.urlopen(remote_request, timeout=300) as response, path.open("wb") as handle: + shutil.copyfileobj(response, handle, length=1024 * 1024) + except error.HTTPError as exc: + raise RuntimeError(f"下载远程输入失败({exc.code}): {path.name}") from exc + except error.URLError as exc: + raise RuntimeError(f"下载远程输入失败: {path.name}") from exc + return path + + +def download_remote_source(url: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None: + path = output_dir / "_materialized_sources" / default_name + return download_remote_source_to_path(url, path) + + +def workspace_cached_bos_source_path(raw_path: str, output_dir: pathlib.Path) -> pathlib.Path | None: + bucket, key = parse_bos_path(raw_path) + if not bucket or not key: + return None + workspace_root = str(os.getenv("WORKSPACE_ROOT", "")).strip() + if workspace_root: + root = pathlib.Path(workspace_root) + if root.is_absolute(): + return root / "_bos_cache" / bucket / key + return output_dir / "_materialized_sources" / bucket / key + + +def download_bos_source(raw_path: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None: + target = workspace_cached_bos_source_path(raw_path, output_dir) + if target is None: + return None + if target.exists() and target.stat().st_size > 0: + return target + bucket, key = parse_bos_path(raw_path) + bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip() + bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip() + bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip() + if not bucket or not key or not bos_endpoint or not bos_access_key_id or not bos_secret_access_key: + return None + target.parent.mkdir(parents=True, exist_ok=True) + temp_target = target.with_name(target.name + ".part") + with contextlib.suppress(FileNotFoundError): + temp_target.unlink() + try: + client, _ = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key) + client.get_object_to_file(bucket, key, str(temp_target)) + temp_target.replace(target) + except Exception as exc: + with contextlib.suppress(FileNotFoundError): + temp_target.unlink() + raise RuntimeError(f"下载 BOS 输入失败: {default_name}") from exc + return target + + +def signed_bos_source_url(raw_path: str) -> str: + bucket, key = parse_bos_path(raw_path) + if not bucket or not key: + return "" + bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip() + bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip() + bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip() + if not bos_endpoint or not bos_access_key_id or not bos_secret_access_key: + return "" + client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key) + return text_value( + client.generate_pre_signed_url( + bucket, + key, + expiration_in_seconds=3600, + protocol=scheme, + ) + ).strip() + + +def remote_video_url_for_source(source: dict[str, Any]) -> str: + preview_url = str(source.get("previewURL", "")).strip() + parsed_preview = urlparse(preview_url) + if parsed_preview.scheme.lower() in {"http", "https"}: + return preview_url + raw_path = str(source.get("path", "")).strip() + if raw_path.startswith("bos://"): + return signed_bos_source_url(raw_path) + return "" + + +def is_archive_path(path: pathlib.Path) -> bool: + name = path.name.lower() + return any(name.endswith(suffix) for suffix in ARCHIVE_FILE_SUFFIXES) + + +def safe_extract_archive(path: pathlib.Path, destination: pathlib.Path) -> list[str]: + destination.mkdir(parents=True, exist_ok=True) + destination_resolved = destination.resolve() + extracted_files: list[str] = [] + + def ensure_child(member_name: str) -> None: + target = (destination / member_name).resolve() + if target != destination_resolved and destination_resolved not in target.parents: + raise RuntimeError(f"archive entry escapes target directory: {member_name}") + + if path.name.lower().endswith(".zip"): + with zipfile.ZipFile(path) as archive: + for member in archive.infolist(): + if not member.filename or member.is_dir(): + continue + ensure_child(member.filename) + extracted_files.append(member.filename) + archive.extractall(destination) + return extracted_files + + with tarfile.open(path, "r:*") as archive: + members = archive.getmembers() + for member in members: + if not member.name or member.isdir(): + continue + ensure_child(member.name) + extracted_files.append(member.name) + archive.extractall(destination) + return extracted_files + + def artifact_slug(value: str) -> str: normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value).strip("-") return normalized or "artifact" @@ -116,23 +591,35 @@ def infer_port_type(path: pathlib.Path, fallback: str = "file") -> str: def clean_metadata(metadata: dict[str, Any]) -> dict[str, Any] | None: - cleaned = {key: value for key, value in metadata.items() if value not in ("", None, [], {})} + cleaned = {key: json_compatible(value) for key, value in metadata.items() if value not in ("", None, [], {})} return cleaned or None def load_payload_from_file(path: pathlib.Path) -> dict[str, Any] | None: - if not path.exists() or path.suffix.lower() != ".json": + if path.suffix.lower() != ".json": return None - try: - with path.open("r", encoding="utf-8") as handle: - value = json.load(handle) - except Exception: - return None - if isinstance(value, dict): - return value - if isinstance(value, list): - return {"items": value, "itemCount": len(value)} - return {"value": value} + for attempt in range(PAYLOAD_FILE_READ_ATTEMPTS): + if not path.exists(): + if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS: + time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS) + continue + return None + try: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle) + except (FileNotFoundError, OSError, json.JSONDecodeError): + if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS: + time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS) + continue + return None + except Exception: + return None + if isinstance(value, dict): + return value + if isinstance(value, list): + return {"items": value, "itemCount": len(value)} + return {"value": value} + return None def output_binding(node_data: dict[str, Any], index: int = 0, name: str = "") -> dict[str, Any] | None: @@ -254,6 +741,30 @@ def preview_jsonl_payload(path: pathlib.Path) -> dict[str, Any]: } +def inspect_video_file(path: pathlib.Path) -> dict[str, Any]: + cv2, _ = require_cv2() + capture = cv2.VideoCapture(str(path)) + if not capture.isOpened(): + return {} + try: + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) + frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0 + payload = clean_metadata( + { + "frameRate": round(fps, 3) if fps > 0 else None, + "frameCount": frame_count or None, + "resolution": f"{width}x{height}" if width > 0 and height > 0 else None, + "duration": round(duration, 3) if duration > 0 else None, + } + ) + return payload or {} + finally: + capture.release() + + def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]: payload: dict[str, Any] = { "preview": { @@ -263,11 +774,23 @@ def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]: } size_bytes = path.stat().st_size if path.exists() else 0 if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type: - encoded = base64.b64encode(path.read_bytes()).decode("ascii") - payload["preview"]["url"] = f"data:{mime_type};base64,{encoded}" + payload["preview"]["url"] = data_url_for_path(path, mime_type) return payload +def preview_media_payload(path: pathlib.Path, mime_type: str, kind: str) -> dict[str, Any]: + preview: dict[str, Any] = { + "kind": kind, + "format": path.suffix.lower().lstrip(".") or kind, + } + if kind == "video": + preview.update(inspect_video_file(path)) + size_bytes = path.stat().st_size if path.exists() else 0 + if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type: + preview["url"] = data_url_for_path(path, mime_type) + return {"preview": preview} + + def sample_file_entries(paths: list[pathlib.Path], root: pathlib.Path) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] for candidate in paths[:PREVIEW_FILE_LIST_LIMIT]: @@ -534,6 +1057,1119 @@ def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any } +def parse_float_param(params: dict[str, str], key: str, default: float) -> float: + raw = params.get(key, "").strip() + if not raw: + return default + try: + return float(raw) + except Exception: + return default + + +def float_value(value: Any, default: float) -> float: + if value is None: + return default + if isinstance(value, str) and not value.strip(): + return default + try: + return float(value) + except Exception: + return default + + +def parse_int_param(params: dict[str, str], key: str, default: int) -> int: + raw = params.get(key, "").strip() + if not raw: + return default + try: + return int(raw) + except Exception: + return default + + +def parse_bool_param(params: dict[str, str], key: str, default: bool = False) -> bool: + raw = params.get(key, "").strip().lower() + if not raw: + return default + return raw in {"1", "true", "yes", "on"} + + +def first_payload_with_keys(inputs: list[dict[str, Any]], *keys: str) -> dict[str, Any] | None: + for artifact in inputs: + payload = load_artifact_payload(artifact) + if isinstance(payload, dict) and all(key in payload for key in keys): + return payload + return None + + +def common_input_root(paths: list[str]) -> str: + filtered = [item for item in paths if item] + if not filtered: + return "" + if len(filtered) == 1: + return str(pathlib.Path(filtered[0]).parent) + try: + return os.path.commonpath(filtered) + except Exception: + return str(pathlib.Path(filtered[0]).parent) + + +def artifact_preview_url(artifact: dict[str, Any]) -> str: + preview = artifact_preview_payload(artifact) or {} + value = preview.get("url") + return value if isinstance(value, str) else "" + + +def source_ref_from_artifact(artifact: dict[str, Any], relative_path: str = "") -> dict[str, Any]: + metadata = artifact.get("metadata") if isinstance(artifact.get("metadata"), dict) else {} + return clean_metadata( + { + "name": str(artifact.get("name", "")).strip(), + "path": str(artifact.get("path", "")).strip(), + "relativePath": relative_path, + "previewURL": artifact_preview_url(artifact), + "mimeType": str(metadata.get("mimeType", "")).strip(), + } + ) or {} + + +def materialize_artifact_to_path(artifact: dict[str, Any], destination: pathlib.Path) -> bool: + raw_path = str(artifact.get("path", "")).strip() + destination.parent.mkdir(parents=True, exist_ok=True) + if raw_path and pathlib.Path(raw_path).exists(): + shutil.copy2(raw_path, destination) + return True + preview_url = artifact_preview_url(artifact) + decoded = decode_data_url(preview_url) if preview_url else None + if decoded is not None: + _, payload = decoded + destination.write_bytes(payload) + return True + payload = artifact.get("data") + if isinstance(payload, dict) and destination.suffix.lower() == ".json": + destination.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + return True + preview = artifact_preview_payload(artifact) or {} + if preview.get("kind") == "text": + destination.write_text(str(preview.get("text", "")), encoding="utf-8") + return True + return False + + +def upload_relative_path(artifact: dict[str, Any], input_root: str, fallback_name: str) -> str: + explicit_relative_path = str(artifact.get("relativePath", "")).strip() + if explicit_relative_path: + return pathlib.PurePosixPath(explicit_relative_path).as_posix().lstrip("/") + payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} + payload_relative_path = str(payload.get("relativePath", "")).strip() + if payload_relative_path: + return pathlib.PurePosixPath(payload_relative_path).as_posix().lstrip("/") + raw_path = str(artifact.get("path", "")).strip() + relative_path = str(fallback_name or artifact.get("name", "")).strip() or "artifact" + if raw_path and input_root: + try: + relative_path = pathlib.Path(raw_path).relative_to(input_root).as_posix() + except Exception: + relative_path = pathlib.Path(raw_path).name or relative_path + if not relative_path: + relative_path = pathlib.Path(fallback_name or "artifact").name or "artifact" + return pathlib.PurePosixPath(relative_path).as_posix().lstrip("/") + + +def materialized_upload_source(artifact: dict[str, Any], staging_dir: pathlib.Path, relative_path: str) -> pathlib.Path | None: + raw_path = str(artifact.get("path", "")).strip() + if raw_path and os.path.exists(raw_path): + return pathlib.Path(raw_path) + destination = staging_dir / "_materialized_upload_inputs" / relative_path + if destination.exists(): + return destination + if materialize_artifact_to_path(artifact, destination): + return destination + return None + + +def package_config_payload(item: dict[str, Any], manifest_payload: dict[str, Any]) -> dict[str, Any]: + return { + "collectionId": item.get("collectionId", ""), + "meta": item.get("meta", {}), + "segmentDurationSeconds": manifest_payload.get("segmentDurationSeconds"), + "sourceVideo": item.get("sourceVideo", ""), + "startSecond": item.get("startSecond"), + "endSecond": item.get("endSecond"), + "duration": item.get("duration") or item.get("clipDuration"), + "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, + } + + +def package_related_artifact_names(manifest_payload: dict[str, Any]) -> set[str]: + names: set[str] = set() + for item in manifest_payload.get("items", []): + if not isinstance(item, dict): + continue + clip_id = str(item.get("clipId", "")).strip() + clip_name = str(item.get("clipName", "")).strip() + annotation_name = str(item.get("annotationFile", "")).strip() or (f"{clip_id}_annotation.json" if clip_id else "") + quality_name = f"{clip_id}_quality.json" if clip_id else "" + config_name = f"{clip_id}_config.json" if clip_id else "" + for candidate in (clip_name, annotation_name, quality_name, config_name): + if candidate: + names.add(candidate) + return names + + +def materialize_package_artifacts_from_manifest( + run_id: str, + node_id: str, + staging_dir: pathlib.Path, + manifest_payload: dict[str, Any], + node_data: dict[str, Any], +) -> list[dict[str, Any]]: + archive_sources = manifest_payload.get("archiveSources", {}) if isinstance(manifest_payload.get("archiveSources"), dict) else {} + binding = output_binding(node_data) + synthetic_artifacts: list[dict[str, Any]] = [] + materialized_root = staging_dir / "_materialized_upload_inputs" + for item in manifest_payload.get("items", []): + if not isinstance(item, dict): + continue + clip_id = str(item.get("clipId", "")).strip() + if not clip_id: + continue + package_name = str(item.get("packageName", "")).strip() or f"{clip_id}_package" + clip_name = str(item.get("clipName", "")).strip() or f"{clip_id}.mp4" + package_relative_dir = pathlib.PurePosixPath("packages") / package_name + package_dir = materialized_root / package_relative_dir + package_dir.mkdir(parents=True, exist_ok=True) + + clip_destination = package_dir / clip_name + direct_clip_path = str(item.get("clipPath", "")).strip() + clip_source: pathlib.Path | None = pathlib.Path(direct_clip_path) if direct_clip_path and pathlib.Path(direct_clip_path).exists() else None + if clip_source is None: + clip_source = materialize_clip_from_source(item, staging_dir, archive_sources) + if clip_source is not None and clip_source.exists(): + clip_size = clip_source.stat().st_size + clip_mime_type = mimetypes.guess_type(clip_name)[0] or "video/mp4" + clip_artifact = { + "id": f"{run_id}-{node_id}-{package_name}-{clip_name}", + "runId": run_id, + "nodeId": node_id, + "name": clip_name, + "type": "video", + "size": format_size(clip_size), + "path": str(clip_source), + "metadata": {"mimeType": clip_mime_type}, + "data": { + "relativePath": (package_relative_dir / clip_name).as_posix(), + "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, + }, + } + clip_artifact["relativePath"] = (package_relative_dir / clip_name).as_posix() + synthetic_artifacts.append(clip_artifact) + + annotation_name = str(item.get("annotationFile", "")).strip() or f"{clip_id}_annotation.json" + annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {} + annotation_artifact = write_json_artifact(run_id, node_id, package_dir, annotation_name, annotation_payload, output=binding) + annotation_artifact["relativePath"] = (package_relative_dir / annotation_name).as_posix() + synthetic_artifacts.append(annotation_artifact) + + quality_name = f"{clip_id}_quality.json" + quality_payload = { + "clip_id": clip_id, + "source_video": item.get("sourceVideo", ""), + **(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}), + } + quality_artifact = write_json_artifact(run_id, node_id, package_dir, quality_name, quality_payload, output=binding) + quality_artifact["relativePath"] = (package_relative_dir / quality_name).as_posix() + synthetic_artifacts.append(quality_artifact) + + config_name = f"{clip_id}_config.json" + config_artifact = write_json_artifact( + run_id, + node_id, + package_dir, + config_name, + package_config_payload(item, manifest_payload), + output=binding, + ) + config_artifact["relativePath"] = (package_relative_dir / config_name).as_posix() + synthetic_artifacts.append(config_artifact) + return synthetic_artifacts + + +def materialize_input_tree( + inputs: list[dict[str, Any]], + output_dir: pathlib.Path, +) -> tuple[pathlib.Path | None, dict[str, dict[str, Any]]]: + paths = [str(item.get("path", "")).strip() for item in inputs if str(item.get("path", "")).strip()] + root = common_input_root(paths) + if not root: + return None, {} + target_root = output_dir / "_materialized_inputs" + if target_root.exists(): + shutil.rmtree(target_root) + target_root.mkdir(parents=True, exist_ok=True) + source_refs: dict[str, dict[str, Any]] = {} + for index, artifact in enumerate(inputs): + raw_path = str(artifact.get("path", "")).strip() + relative_path = str(artifact.get("name", "")).strip() or f"artifact-{index}" + if raw_path: + try: + relative_path = pathlib.Path(raw_path).relative_to(root).as_posix() + except Exception: + relative_path = pathlib.Path(raw_path).name or relative_path + destination = target_root / relative_path + if materialize_artifact_to_path(artifact, destination): + source_refs[relative_path] = source_ref_from_artifact(artifact, relative_path) + if destination.is_file() and is_archive_path(destination): + extracted_files = safe_extract_archive(destination, target_root) + archive_preview_url = artifact_preview_url(artifact) + archive_key = artifact_slug(hashlib.sha256(archive_preview_url.encode("utf-8")).hexdigest()[:16]) if archive_preview_url else artifact_slug(relative_path) + for extracted_relative_path in extracted_files: + normalized_relative_path = pathlib.PurePosixPath(extracted_relative_path).as_posix().lstrip("/") + source_refs[normalized_relative_path] = clean_metadata( + { + "name": pathlib.PurePosixPath(normalized_relative_path).name, + "path": str(target_root / normalized_relative_path), + "relativePath": normalized_relative_path, + "archiveKey": archive_key, + "archivePreviewURL": archive_preview_url, + "archiveMemberPath": normalized_relative_path, + "archiveName": destination.name, + "mimeType": mimetypes.guess_type(normalized_relative_path)[0] or "", + } + ) or {} + return target_root, source_refs + + +def detect_xplatform_collections(root: pathlib.Path) -> list[pathlib.Path]: + if not root.exists(): + return [] + candidates: list[pathlib.Path] = [] + if (root / "meta.json").exists(): + return [root] + for child in sorted(root.iterdir()): + if not child.is_dir(): + continue + if (child / "meta.json").exists(): + candidates.append(child) + continue + if any(candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES for candidate in child.rglob("*")): + candidates.append(child) + return candidates + + +def normalize_collection_relative_path(value: pathlib.PurePath | str) -> str: + text = value.as_posix() if isinstance(value, pathlib.PurePath) else str(value or "") + normalized = text.strip().strip("/") + if normalized in {"", "."}: + return "" + return normalized + + +def collection_name_from_source_path(source_path: str, fallback_name: str = "collection") -> str: + candidate = pathlib.PurePosixPath(str(source_path or "").rstrip("/")).name + return candidate or fallback_name + + +def parse_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]: + rows: dict[str, dict[str, Any]] = {} + if not isinstance(value, list): + return rows + for item in value: + if not isinstance(item, dict): + continue + filename = str(item.get("filename", "")).strip() + if not filename: + continue + rows[filename] = item + return rows + + +def collection_file_entry( + root: pathlib.Path, + collection_dir: pathlib.Path, + candidate: pathlib.Path, + collection_id: str, + meta: dict[str, Any], + video_meta_rows: dict[str, dict[str, Any]], +) -> dict[str, Any]: + relative_path = candidate.relative_to(root).as_posix() + collection_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root)) + collection_relative_file_path = candidate.relative_to(collection_dir).as_posix() + mime_type = mimetypes.guess_type(candidate.name)[0] or "" + entry: dict[str, Any] = { + "collectionId": collection_id, + "collectionName": collection_dir.name, + "collectionPath": str(collection_dir), + "collectionRelativePath": collection_relative_path, + "meta": meta, + "relativePath": relative_path, + "path": str(candidate), + "sizeBytes": candidate.stat().st_size, + "mimeType": mime_type, + "source": {"path": str(candidate), "relativePath": relative_path, "mimeType": mime_type}, + } + if candidate.suffix.lower() == ".json": + payload = optional_json_file(candidate) + if payload is not None: + entry["data"] = payload + if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES: + matched_meta = video_meta_rows.get(candidate.name, {}) + entry.update( + { + "videoFolder": collection_relative_file_path.split("/", 1)[0] if "/" in collection_relative_file_path else "", + "filename": candidate.name, + "codec": matched_meta.get("codec"), + "resolution": matched_meta.get("resolution"), + "frameRate": matched_meta.get("frameRate"), + "duration": matched_meta.get("duration"), + "startTime": matched_meta.get("startTime"), + } + ) + return entry + + +def collection_summary( + root: pathlib.Path, + collection_dir: pathlib.Path, + meta: dict[str, Any], + video_meta_rows: dict[str, dict[str, Any]], +) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]: + files = [candidate for candidate in sorted(collection_dir.rglob("*")) if candidate.is_file()] + videos = [candidate for candidate in files if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES] + total_bytes = sum(candidate.stat().st_size for candidate in files) + required = ["meta.json", "video_meta.json"] + missing_required = [] + if not (collection_dir / "meta.json").exists(): + missing_required.append("meta.json") + if not any((collection_dir / candidate).exists() for candidate in XPLATFORM_VIDEO_META_FILES): + missing_required.append("video_meta.json") + has_intrinsics = (collection_dir / "intrinsics.json").exists() + collection_root_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root)) + collection_id = collection_key(collection_root_relative_path, collection_dir.name) + items: list[dict[str, Any]] = [] + entries: list[dict[str, Any]] = [] + for candidate in files: + entry = collection_file_entry(root, collection_dir, candidate, collection_id, meta, video_meta_rows) + entries.append(entry) + if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES: + items.append(dict(entry)) + summary = { + "id": collection_id, + "name": collection_dir.name, + "path": str(collection_dir), + "relativePath": collection_root_relative_path, + "meta": meta, + "hasIntrinsics": has_intrinsics, + "hasVideoMeta": bool(video_meta_rows), + "requiredFiles": required, + "missingRequiredFiles": missing_required, + "videoCount": len(videos), + "fileCount": len(files), + "totalBytes": total_bytes, + "valid": len(missing_required) == 0 and len(videos) > 0, + } + return summary, items, entries + + +def source_path_join(root: str, relative_path: str) -> str: + if not root: + return relative_path + return root.rstrip("/") + "/" + relative_path.lstrip("/") + + +def manifest_source_ref(entry: dict[str, Any], relative_path: str, source_path: str) -> dict[str, Any]: + source = entry.get("source") if isinstance(entry.get("source"), dict) else {} + storage_path = str(source.get("path", "") or entry.get("storagePath", "") or source_path_join(source_path, relative_path)).strip() + return clean_metadata( + { + "path": storage_path, + "relativePath": relative_path, + "previewURL": str(source.get("previewURL", "")).strip(), + "mimeType": str(entry.get("mimeType", "")).strip(), + } + ) or {} + + +def manifest_entry_json_payload(entry: Any, source_path: str = "") -> Any: + if not isinstance(entry, dict): + return None + inline_data = entry.get("data") + if inline_data is not None: + return inline_data + source = entry.get("source") if isinstance(entry.get("source"), dict) else {} + candidates = [str(source.get("path", "")).strip(), str(entry.get("storagePath", "")).strip(), str(entry.get("path", "")).strip()] + for raw_candidate in candidates: + if not raw_candidate: + continue + if "://" in raw_candidate and not raw_candidate.startswith("file://"): + continue + candidate_text = raw_candidate[len("file://") :] if raw_candidate.startswith("file://") else raw_candidate + candidate_path = pathlib.Path(candidate_text) + if not candidate_path.is_absolute(): + normalized_source_path = str(source_path or "").strip() + if normalized_source_path and "://" not in normalized_source_path: + candidate_path = pathlib.Path(normalized_source_path) / candidate_text + if candidate_path.is_file(): + return optional_json_file(candidate_path) + return None + + +def manifest_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]: + if isinstance(value, dict) and isinstance(value.get("items"), list): + return parse_video_meta_rows(value.get("items")) + return parse_video_meta_rows(value) + + +def collection_key(relative_path: str, fallback_name: str = "collection") -> str: + normalized = relative_path.strip().strip("/") + if not normalized: + return artifact_slug(fallback_name) + return artifact_slug(normalized) + + +def fallback_manifest_collections(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]: + grouped: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + relative_path = str(entry.get("path", "")).strip().lstrip("/") + if not relative_path: + continue + parts = pathlib.PurePosixPath(relative_path).parts + if len(parts) < 2: + continue + grouped.setdefault(parts[0], []).append({**entry, "path": relative_path}) + + collection_summaries: list[dict[str, Any]] = [] + items: list[dict[str, Any]] = [] + indexed_entries: list[dict[str, Any]] = [] + for collection_relative_path in sorted(grouped.keys()): + collection_entries = grouped[collection_relative_path] + collection_name = pathlib.PurePosixPath(collection_relative_path).name + collection_id = collection_key(collection_relative_path, collection_name) + meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/meta.json"), None) + intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/intrinsics.json"), None) + video_meta_entry = next( + ( + entry + for entry in collection_entries + if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name) + ), + None, + ) + meta_payload = manifest_entry_json_payload(meta_entry, source_path) + video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path) + meta = meta_payload if isinstance(meta_payload, dict) else {} + video_meta_rows = manifest_video_meta_rows(video_meta_payload) + videos = [ + entry + for entry in collection_entries + if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES + ] + total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries) + required = ["meta.json", "video_meta.json"] + missing_required = [] + if meta_entry is None: + missing_required.append("meta.json") + if video_meta_entry is None: + missing_required.append("video_meta.json") + collection_path = source_path_join(source_path, collection_relative_path) + collection_summaries.append( + { + "id": collection_id, + "name": collection_name, + "path": collection_path, + "relativePath": collection_relative_path, + "meta": meta, + "hasIntrinsics": intrinsics_entry is not None, + "hasVideoMeta": bool(video_meta_rows), + "requiredFiles": required, + "missingRequiredFiles": missing_required, + "videoCount": len(videos), + "fileCount": len(collection_entries), + "totalBytes": total_bytes, + "valid": len(missing_required) == 0 and len(videos) > 0, + } + ) + for collection_entry in collection_entries: + relative_path = str(collection_entry.get("path", "")).strip() + entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or "" + indexed_entry = { + "collectionId": collection_id, + "collectionName": collection_name, + "collectionPath": collection_path, + "collectionRelativePath": collection_relative_path, + "meta": meta, + "relativePath": relative_path, + "path": source_path_join(source_path, relative_path), + "sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0), + "mimeType": entry_mime_type, + "source": manifest_source_ref(collection_entry, relative_path, source_path), + } + if "data" in collection_entry: + indexed_entry["data"] = normalize_json_payload(collection_entry.get("data")) + if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES: + collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path)) + matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {}) + indexed_entry.update( + { + "videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "", + "filename": pathlib.PurePosixPath(relative_path).name, + "codec": matched_meta.get("codec"), + "resolution": matched_meta.get("resolution"), + "frameRate": matched_meta.get("frameRate"), + "duration": matched_meta.get("duration"), + "startTime": matched_meta.get("startTime"), + } + ) + indexed_entries.append(indexed_entry) + for video_entry in videos: + relative_path = str(video_entry.get("path", "")).strip() + collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path)) + matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {}) + items.append( + { + "collectionId": collection_id, + "collectionName": collection_name, + "collectionPath": collection_path, + "collectionRelativePath": collection_relative_path, + "meta": meta, + "videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "", + "filename": pathlib.PurePosixPath(relative_path).name, + "relativePath": relative_path, + "path": source_path_join(source_path, relative_path), + "sizeBytes": int(video_entry.get("sizeBytes", 0) or 0), + "codec": matched_meta.get("codec"), + "resolution": matched_meta.get("resolution"), + "frameRate": matched_meta.get("frameRate"), + "duration": matched_meta.get("duration"), + "startTime": matched_meta.get("startTime"), + "source": manifest_source_ref(video_entry, relative_path, source_path), + } + ) + return { + "datasetType": "x-platform", + "rootPath": source_path, + "collectionCount": len(collection_summaries), + "videoCount": len(items), + "collections": collection_summaries, + "items": items, + "entries": indexed_entries, + } + + +def scan_xplatform_manifest_entries(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]: + normalized_entries: list[dict[str, Any]] = [] + meta_roots: set[str] = set() + auxiliary_roots: set[str] = set() + has_root_metadata = False + for entry in entries: + relative_path = str(entry.get("path", "")).strip().lstrip("/") + if not relative_path: + continue + normalized_entry = {**entry, "path": relative_path} + normalized_entries.append(normalized_entry) + path_obj = pathlib.PurePosixPath(relative_path) + parent = path_obj.parent.as_posix() + if parent in {".", ""}: + parent = "" + file_name = path_obj.name.lower() + if file_name == "meta.json": + if parent: + meta_roots.add(parent) + else: + has_root_metadata = True + elif is_xplatform_video_meta_name(file_name) or file_name == "intrinsics.json": + if parent: + auxiliary_roots.add(parent) + else: + has_root_metadata = True + + root_candidates = meta_roots or auxiliary_roots + if has_root_metadata: + root_candidates = set(root_candidates) + root_candidates.add("") + if not root_candidates: + return fallback_manifest_collections(normalized_entries, source_path) + + ordered_roots = sorted(root_candidates, key=lambda value: (-len(pathlib.PurePosixPath(value).parts), value)) + grouped: dict[str, list[dict[str, Any]]] = {root: [] for root in ordered_roots} + unmatched_entries: list[dict[str, Any]] = [] + for entry in normalized_entries: + relative_path = str(entry.get("path", "")).strip() + matched_root = next( + ( + root + for root in ordered_roots + if ( + (not root) + or relative_path == root + or relative_path.startswith(root + "/") + ) + ), + None, + ) + if matched_root is not None: + grouped.setdefault(matched_root, []).append(entry) + else: + unmatched_entries.append(entry) + + collection_summaries: list[dict[str, Any]] = [] + items: list[dict[str, Any]] = [] + indexed_entries: list[dict[str, Any]] = [] + for collection_relative_path in sorted(grouped.keys()): + collection_entries = grouped[collection_relative_path] + if not collection_entries: + continue + collection_name = ( + pathlib.PurePosixPath(collection_relative_path).name + if collection_relative_path + else collection_name_from_source_path(source_path) + ) + collection_id = collection_key(collection_relative_path, collection_name) + meta_path = f"{collection_relative_path}/meta.json" if collection_relative_path else "meta.json" + intrinsics_path = f"{collection_relative_path}/intrinsics.json" if collection_relative_path else "intrinsics.json" + meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == meta_path), None) + intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == intrinsics_path), None) + video_meta_entry = next( + ( + entry + for entry in collection_entries + if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name) + ), + None, + ) + meta_payload = manifest_entry_json_payload(meta_entry, source_path) + video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path) + meta = meta_payload if isinstance(meta_payload, dict) else {} + video_meta_rows = manifest_video_meta_rows(video_meta_payload) + videos = [ + entry + for entry in collection_entries + if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES + ] + total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries) + required = ["meta.json", "video_meta.json"] + missing_required = [] + if meta_entry is None: + missing_required.append("meta.json") + if video_meta_entry is None: + missing_required.append("video_meta.json") + collection_path = source_path if not collection_relative_path else source_path_join(source_path, collection_relative_path) + collection_summaries.append( + { + "id": collection_id, + "name": collection_name, + "path": collection_path, + "relativePath": collection_relative_path, + "meta": meta, + "hasIntrinsics": intrinsics_entry is not None, + "hasVideoMeta": bool(video_meta_rows), + "requiredFiles": required, + "missingRequiredFiles": missing_required, + "videoCount": len(videos), + "fileCount": len(collection_entries), + "totalBytes": total_bytes, + "valid": len(missing_required) == 0 and len(videos) > 0, + } + ) + for collection_entry in collection_entries: + relative_path = str(collection_entry.get("path", "")).strip() + entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or "" + indexed_entry = { + "collectionId": collection_id, + "collectionName": collection_name, + "collectionPath": collection_path, + "collectionRelativePath": collection_relative_path, + "meta": meta, + "relativePath": relative_path, + "path": source_path_join(source_path, relative_path), + "sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0), + "mimeType": entry_mime_type, + "source": manifest_source_ref(collection_entry, relative_path, source_path), + } + if "data" in collection_entry: + indexed_entry["data"] = normalize_json_payload(collection_entry.get("data")) + if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES: + collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path)) + matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {}) + indexed_entry.update( + { + "videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "", + "filename": pathlib.PurePosixPath(relative_path).name, + "codec": matched_meta.get("codec"), + "resolution": matched_meta.get("resolution"), + "frameRate": matched_meta.get("frameRate"), + "duration": matched_meta.get("duration"), + "startTime": matched_meta.get("startTime"), + } + ) + indexed_entries.append(indexed_entry) + for video_entry in videos: + relative_path = str(video_entry.get("path", "")).strip() + collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path)) + matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {}) + items.append( + { + "collectionId": collection_id, + "collectionName": collection_name, + "collectionPath": collection_path, + "collectionRelativePath": collection_relative_path, + "meta": meta, + "videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "", + "filename": pathlib.PurePosixPath(relative_path).name, + "relativePath": relative_path, + "path": source_path_join(source_path, relative_path), + "sizeBytes": int(video_entry.get("sizeBytes", 0) or 0), + "codec": matched_meta.get("codec"), + "resolution": matched_meta.get("resolution"), + "frameRate": matched_meta.get("frameRate"), + "duration": matched_meta.get("duration"), + "startTime": matched_meta.get("startTime"), + "source": manifest_source_ref(video_entry, relative_path, source_path), + } + ) + if unmatched_entries: + fallback_payload = fallback_manifest_collections(unmatched_entries, source_path) + collection_summaries.extend(fallback_payload.get("collections", [])) + items.extend(fallback_payload.get("items", [])) + indexed_entries.extend(fallback_payload.get("entries", [])) + return { + "datasetType": "x-platform", + "rootPath": source_path, + "collectionCount": len(collection_summaries), + "videoCount": len(items), + "collections": collection_summaries, + "items": items, + "entries": indexed_entries, + } + + +def with_xplatform_source_context(payload: dict[str, Any], source_manifest: dict[str, Any]) -> dict[str, Any]: + for key in ( + "dataSource", + "sourcePath", + "scanMode", + "listedCount", + "newCount", + "sourceFingerprint", + "bucket", + "prefix", + "pattern", + "endpoint", + ): + if key in source_manifest: + payload[key] = source_manifest.get(key) + return payload + + +def attach_xplatform_catalog_payload(payload: dict[str, Any], catalog_payload: dict[str, Any]) -> dict[str, Any]: + payload["catalogCollectionCount"] = int(catalog_payload.get("collectionCount", 0) or 0) + payload["catalogVideoCount"] = int(catalog_payload.get("videoCount", 0) or 0) + payload["catalogCollections"] = catalog_payload.get("collections", []) + payload["catalogItems"] = catalog_payload.get("items", []) + payload["catalogEntries"] = catalog_payload.get("entries", []) + return payload + + +def apply_source_refs_to_items( + items: list[dict[str, Any]], + source_refs: dict[str, dict[str, Any]], +) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: + if not source_refs: + return items, {} + archive_sources: dict[str, dict[str, Any]] = {} + for item in items: + relative_path = str(item.get("relativePath", "")).strip() + if not relative_path: + continue + source_ref = source_refs.get(relative_path) + if source_ref: + archive_key = str(source_ref.get("archiveKey", "")).strip() + archive_preview_url = str(source_ref.get("archivePreviewURL", "")).strip() + if archive_key and archive_preview_url and archive_key not in archive_sources: + archive_sources[archive_key] = clean_metadata( + { + "name": str(source_ref.get("archiveName", "")).strip() or "archive", + "previewURL": archive_preview_url, + } + ) or {} + item_source = {**source_ref, "path": item.get("path", "")} + item_source.pop("archivePreviewURL", None) + item_source.pop("archiveName", None) + item["source"] = item_source + return items, archive_sources + + +def scan_xplatform_dataset(root: pathlib.Path, source_refs: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]: + collections = detect_xplatform_collections(root) + collection_summaries: list[dict[str, Any]] = [] + items: list[dict[str, Any]] = [] + entries: list[dict[str, Any]] = [] + for collection_dir in collections: + meta = optional_json_file(collection_dir / "meta.json") + meta = meta if isinstance(meta, dict) else {} + video_meta_rows, _ = load_collection_video_meta(collection_dir) + summary, collection_items, collection_entries = collection_summary(root, collection_dir, meta, video_meta_rows) + collection_summaries.append(summary) + items.extend(collection_items) + entries.extend(collection_entries) + items, archive_sources = apply_source_refs_to_items(items, source_refs or {}) + entries, entry_archive_sources = apply_source_refs_to_items(entries, source_refs or {}) + archive_sources.update(entry_archive_sources) + payload = { + "datasetType": "x-platform", + "rootPath": str(root), + "collectionCount": len(collection_summaries), + "videoCount": len(items), + "collections": collection_summaries, + "items": items, + "entries": entries, + } + if archive_sources: + payload["archiveSources"] = archive_sources + return payload + + +def load_xplatform_index_payload( + params: dict[str, str], + inputs: list[dict[str, Any]], + output_dir: pathlib.Path, +) -> dict[str, Any]: + source_manifest = first_payload_with_keys(inputs, "entries") + if source_manifest is not None: + entries = source_manifest.get("entries") + if isinstance(entries, list): + manifest_source_path = str(source_manifest.get("sourcePath", "")).strip() + payload = scan_xplatform_manifest_entries(entries, manifest_source_path) + catalog_entries = source_manifest.get("catalogEntries") + if isinstance(catalog_entries, list): + catalog_payload = scan_xplatform_manifest_entries(catalog_entries, manifest_source_path) + attach_xplatform_catalog_payload(payload, catalog_payload) + return with_xplatform_source_context(payload, source_manifest) + manifest_source_path = str(source_manifest.get("sourcePath", "")).strip() + if manifest_source_path and pathlib.Path(manifest_source_path).exists(): + root = pathlib.Path(manifest_source_path) + return scan_xplatform_dataset(root if root.is_dir() else root.parent) + for artifact in inputs: + name = str(artifact.get("name", "")).strip().lower() + data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} + if name != "source-manifest.json" and not any(key in data for key in ("entriesCount", "catalogEntriesCount", "sourcePath")): + continue + raise RuntimeError("X 平台数据集索引未能读取 source-manifest 的 entries,拒绝回退到空目录扫描") + source_path = resolve_source_path(params, inputs) + if source_path is not None: + root = source_path if source_path.is_dir() else source_path.parent + return scan_xplatform_dataset(root) + materialized_root, source_refs = materialize_input_tree(inputs, output_dir) + if materialized_root is not None: + return scan_xplatform_dataset(materialized_root, source_refs) + raise RuntimeError("X 平台数据集索引需要 SOURCE_PATH 或上游输入提供真实数据目录") + + +def normalize_date_value(value: Any) -> str: + text = str(value or "").strip() + digits = "".join(character for character in text if character.isdigit()) + if len(digits) >= 8: + return digits[:8] + return "" + + +def date_in_range(value: Any, start: str, end: str) -> bool: + normalized = normalize_date_value(value) + if not normalized: + return False + if start and normalized < start: + return False + if end and normalized > end: + return False + return True + + +def materialize_archive_member( + source: dict[str, Any], + archive_sources: dict[str, Any], + default_name: str, + output_dir: pathlib.Path, +) -> pathlib.Path | None: + archive_key = str(source.get("archiveKey", "")).strip() + archive_source = archive_sources.get(archive_key) if isinstance(archive_sources, dict) else {} + archive_source = archive_source if isinstance(archive_source, dict) else {} + preview_url = str(archive_source.get("previewURL", "")).strip() + member_path = str(source.get("archiveMemberPath", "")).strip() + decoded = decode_data_url(preview_url) if preview_url and member_path else None + if decoded is None: + return None + _, payload = decoded + archive_name = pathlib.PurePosixPath(str(archive_source.get("name", "")).strip() or "archive").name + archive_path = output_dir / "_materialized_sources" / archive_name + archive_path.parent.mkdir(parents=True, exist_ok=True) + archive_path.write_bytes(payload) + destination = output_dir / "_materialized_sources" / default_name + destination.parent.mkdir(parents=True, exist_ok=True) + if archive_name.lower().endswith(".zip"): + with zipfile.ZipFile(archive_path) as archive: + with archive.open(member_path) as handle: + destination.write_bytes(handle.read()) + return destination + with tarfile.open(archive_path, "r:*") as archive: + extracted = archive.extractfile(member_path) + if extracted is None: + return None + with extracted: + destination.write_bytes(extracted.read()) + return destination + + +def source_path_from_item( + item: dict[str, Any], + default_name: str, + output_dir: pathlib.Path, + archive_sources: dict[str, Any] | None = None, +) -> pathlib.Path: + source = item.get("source") if isinstance(item.get("source"), dict) else {} + direct = str(source.get("path", "") or item.get("path", "")).strip() + if direct and pathlib.Path(direct).exists(): + return pathlib.Path(direct) + if direct.startswith("bos://"): + cached_target = workspace_cached_bos_source_path(direct, output_dir) + if cached_target is not None and cached_target.exists() and cached_target.stat().st_size > 0: + return cached_target + cached_bos_source = download_bos_source(direct, default_name, output_dir) + if cached_bos_source is not None: + return cached_bos_source + preview_url = str(source.get("previewURL", "")).strip() + if cached_target is not None and preview_url: + cached_remote_source = download_remote_source_to_path(preview_url, cached_target) + if cached_remote_source is not None: + return cached_remote_source + archive_member = materialize_archive_member(source, archive_sources or {}, default_name, output_dir) + if archive_member is not None: + return archive_member + preview_url = str(source.get("previewURL", "")).strip() + remote_path = download_remote_source(preview_url, default_name, output_dir) if preview_url else None + if remote_path is not None: + return remote_path + decoded = decode_data_url(preview_url) if preview_url else None + if decoded is None: + raise RuntimeError(f"无法获取真实输入文件: {default_name}") + _, payload = decoded + path = output_dir / "_materialized_sources" / default_name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + return path + + +def clip_segment_bounds(item: dict[str, Any]) -> tuple[float, float]: + start_second = float(item.get("startSecond") or 0.0) + end_second = float(item.get("endSecond") or 0.0) + duration = float(item.get("duration") or 0.0) + if end_second <= start_second and duration > 0: + end_second = start_second + duration + if end_second <= start_second: + end_second = start_second + 0.1 + return start_second, end_second + + +def materialize_clip_from_source( + item: dict[str, Any], + output_dir: pathlib.Path, + archive_sources: dict[str, Any] | None = None, +) -> pathlib.Path | None: + source = item.get("source") if isinstance(item.get("source"), dict) else {} + source_name = ( + str(item.get("sourceVideo", "")).strip() + or pathlib.PurePosixPath(str(source.get("path", "")).strip()).name + or "source.mp4" + ) + clip_name = str(item.get("clipName", "")).strip() or f"{pathlib.Path(source_name).stem}_clip.mp4" + destination = output_dir / "_materialized_clips" / clip_name + if destination.exists(): + return destination + start_second, end_second = clip_segment_bounds(item) + remote_source_url = remote_video_url_for_source(source) if source else "" + if remote_source_url: + try: + write_video_segment_from_remote_url(remote_source_url, destination, start_second, end_second) + return destination + except RuntimeError as exc: + log(f"[operator-runner][warn] fallback to temporary source materialization for {clip_name}: {exc}") + if not source: + return None + source_path = source_path_from_item({"source": source}, source_name, output_dir, archive_sources) + write_video_segment(source_path, destination, start_second, end_second) + return destination + + +def analyze_video_quality_metrics(path: pathlib.Path, clarity_scale: float) -> dict[str, Any]: + cv2, np = require_cv2() + capture = cv2.VideoCapture(str(path)) + if not capture.isOpened(): + raise RuntimeError(f"无法打开视频文件: {path}") + try: + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) + frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + positions = sampled_frame_positions(frame_count) + if not positions: + positions = [0] + clarities: list[float] = [] + shifts: list[tuple[float, float]] = [] + skin_ratios: list[float] = [] + previous_gray = None + previous_position: int | None = None + analysis_diagonal = math.sqrt((width or 1) ** 2 + (height or 1) ** 2) + for position in positions: + if frame_count > 0: + capture.set(cv2.CAP_PROP_POS_FRAMES, position) + ok, frame = capture.read() + if not ok: + continue + analysis_frame = resize_quality_frame(frame) + analysis_height, analysis_width = analysis_frame.shape[:2] + analysis_diagonal = math.sqrt((analysis_width or 1) ** 2 + (analysis_height or 1) ** 2) + gray = cv2.cvtColor(analysis_frame, cv2.COLOR_BGR2GRAY) + laplacian = cv2.Laplacian(gray, cv2.CV_64F).var() + clarities.append(float(laplacian)) + if previous_gray is not None: + gap = max(position - previous_position, 1) if previous_position is not None else 1 + shift, _ = cv2.phaseCorrelate(np.float32(previous_gray), np.float32(gray)) + shifts.append((float(shift[0]) / gap, float(shift[1]) / gap)) + mask = build_skin_mask(analysis_frame) + skin_ratios.append(largest_skin_component_ratio(mask)) + previous_gray = gray + previous_position = position + clarity_raw = sum(clarities) / max(len(clarities), 1) + clarity = 1.0 - math.exp(-max(clarity_raw, 0.0) / max(clarity_scale, 1.0)) + shake = motion_jitter_ratio(shifts, analysis_diagonal) if shifts else 0.0 + hand_occlusion = sum(skin_ratios) / max(len(skin_ratios), 1) if skin_ratios else 0.0 + duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0 + quality_score = max(0.0, min(1.0, (clarity + (1.0 - min(shake, 1.0)) + (1.0 - min(hand_occlusion, 1.0))) / 3.0)) + return { + "codec": "opencv", + "resolution": f"{width}x{height}" if width > 0 and height > 0 else "", + "frameRate": round(fps, 3) if fps > 0 else None, + "duration": round(duration, 3) if duration > 0 else None, + "clarity": round(clarity, 4), + "shake": round(shake, 4), + "handOcclusion": round(hand_occlusion, 4), + "qualityScore": round(quality_score, 4), + "sampleCount": len(clarities), + } + finally: + capture.release() + + +def stable_sample_hit(value: str, percent: float) -> bool: + if percent >= 100: + return True + if percent <= 0: + return False + digest = hashlib.sha256(value.encode("utf-8")).hexdigest() + score = int(digest[:8], 16) / 0xFFFFFFFF + return score <= (percent / 100.0) + + def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) -> dict[str, Any] | None: suffix = path.suffix.lower() if suffix in TABULAR_FILE_SUFFIXES: @@ -550,9 +2186,12 @@ def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) if suffix == ".pdf": return {"preview": {"kind": "document", "format": "pdf"}} if artifact_type == "video" or suffix in VIDEO_FILE_SUFFIXES: - return {"preview": {"kind": "video", "format": suffix.lstrip(".") or "video"}} + return preview_media_payload(path, mime_type or "video/mp4", "video") if suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"): - return {"preview": {"kind": "audio", "format": suffix.lstrip(".") or "audio"}} + return preview_media_payload(path, mime_type or "audio/mpeg", "audio") + size_bytes = path.stat().st_size if path.exists() else 0 + if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type: + return {"preview": {"kind": "binary", "format": suffix.lstrip(".") or "file", "url": data_url_for_path(path, mime_type)}} return None @@ -566,14 +2205,19 @@ def build_artifact( output: dict[str, Any] | None = None, ) -> dict[str, Any]: mime_type, _ = mimetypes.guess_type(path.name) + size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8")) + sha256_value = "" + sha256_metadata: dict[str, Any] = {} + if path.exists(): + sha256_value, sha256_metadata = artifact_sha256(path, size_bytes) artifact_metadata = clean_metadata( { **(metadata or {}), - "sha256": sha256_file(path) if path.exists() else "", + **sha256_metadata, + "sha256": sha256_value, "mimeType": mime_type or "", } ) - size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8")) artifact = { "id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else path.stem)}-{artifact_slug(path.name)}", "runId": run_id, @@ -595,6 +2239,160 @@ def build_artifact( return artifact +def build_virtual_artifact( + run_id: str, + node_id: str, + name: str, + path: str, + artifact_type: str, + size_bytes: int, + data: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + output: dict[str, Any] | None = None, +) -> dict[str, Any]: + artifact = { + "id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else name)}-{artifact_slug(name)}", + "runId": run_id, + "nodeId": node_id, + "name": name, + "type": artifact_type, + "size": format_size(max(0, size_bytes)), + "path": path, + } + if output: + if str(output.get("id", "")).strip(): + artifact["portId"] = str(output.get("id", "")).strip() + if str(output.get("name", "")).strip(): + artifact["portName"] = str(output.get("name", "")).strip() + if metadata: + cleaned = clean_metadata(metadata) + if cleaned: + artifact["metadata"] = cleaned + if data: + artifact["data"] = data + return artifact + + +def dataset_partition_keys(payload: dict[str, Any]) -> list[str]: + keys: list[str] = [] + for key in DATASET_PARTITION_KEYS: + if isinstance(payload.get(key), list): + keys.append(key) + return keys + + +def should_externalize_structured_payload(payload: dict[str, Any]) -> bool: + if dataset_partition_keys(payload): + return True + dataset_type = str(payload.get("datasetType", "")).strip() + if dataset_type: + return True + return False + + +def dataset_payload_summary(payload: dict[str, Any]) -> dict[str, Any]: + summary: dict[str, Any] = {} + for key in ( + "datasetType", + "count", + "itemCount", + "collectionCount", + "entryCount", + "videoCount", + "clipCount", + "annotationCount", + "packageCount", + "sampledCount", + "passedCount", + "failedCount", + "validCollectionCount", + "invalidCollectionCount", + "dataSource", + "sourcePath", + "bucket", + "prefix", + "endpoint", + "pattern", + "scanMode", + "sourceFingerprint", + "listedCount", + "newCount", + "rootPath", + "segmentDurationSeconds", + ): + if key in payload: + summary[key] = json_compatible(payload[key]) + if "preview" in payload: + summary["preview"] = json_compatible(payload.get("preview")) + for key in dataset_partition_keys(payload): + value = payload.get(key) + if isinstance(value, list): + summary[f"{key}Count"] = len(value) + if "preview" not in summary: + summary["preview"] = { + "kind": "dataset", + "format": str(payload.get("datasetType", "")).strip() or "json", + "counts": {key: summary[key] for key in summary.keys() if key.endswith("Count") or key == "count"}, + } + return summary + + +def chunk_list(values: list[Any], chunk_size: int = DATASET_PARTITION_SIZE) -> list[list[Any]]: + if chunk_size <= 0: + chunk_size = DATASET_PARTITION_SIZE + return [values[index : index + chunk_size] for index in range(0, len(values), chunk_size)] or [[]] + + +def externalize_structured_payload(path: pathlib.Path, payload: dict[str, Any]) -> dict[str, Any]: + normalized_payload = json_compatible(payload) + dataset_dir = path.parent / "_datasets" / artifact_slug(path.stem or path.name) + partitions_dir = dataset_dir / "partitions" + partitions_dir.mkdir(parents=True, exist_ok=True) + partition_files: dict[str, list[str]] = {} + partition_count = 0 + partition_keys = dataset_partition_keys(normalized_payload) + for key in partition_keys: + values = normalized_payload.get(key) + if not isinstance(values, list): + continue + files: list[str] = [] + for index, chunk in enumerate(chunk_list(values), start=1): + partition_path = partitions_dir / f"{key}-p{index:06d}.json" + write_json_file(partition_path, chunk) + files.append(str(partition_path)) + partition_files[key] = files + partition_count += len(files) + manifest = { + "kind": "dataset", + "datasetType": str(normalized_payload.get("datasetType", "")).strip() or "json", + "payloadPath": str(path), + "partitionDir": str(partitions_dir), + "partitionKeys": partition_keys, + "partitionFiles": partition_files, + "summary": dataset_payload_summary(normalized_payload), + "generatedAt": now_iso(), + } + manifest_path = write_json_file(dataset_dir / "manifest.json", manifest) + data_ref = { + "kind": "dataset", + "datasetType": manifest["datasetType"], + "manifestPath": str(manifest_path), + "payloadPath": str(path), + "partitionDir": str(partitions_dir), + "partitionCount": partition_count, + } + item_count = 0 + for key in partition_keys: + values = normalized_payload.get(key) + if isinstance(values, list): + item_count += len(values) + if item_count > 0: + data_ref["itemCount"] = item_count + summary = dataset_payload_summary(normalized_payload) + summary["dataRef"] = data_ref + return summary + + def write_json_artifact( run_id: str, node_id: str, @@ -604,11 +2402,15 @@ def write_json_artifact( artifact_type: str = "json", output: dict[str, Any] | None = None, ) -> dict[str, Any]: + normalized_payload = json_compatible(payload) path = output_dir / filename path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: - json.dump(payload, handle, ensure_ascii=False, indent=2) - return build_artifact(run_id, node_id, path, artifact_type, data=payload, output=output) + json.dump(normalized_payload, handle, ensure_ascii=False, indent=2) + artifact_data: dict[str, Any] | None = normalized_payload + if should_externalize_structured_payload(normalized_payload): + artifact_data = externalize_structured_payload(path, normalized_payload) + return build_artifact(run_id, node_id, path, artifact_type, data=artifact_data, output=output) def summarize_json_like(value: Any) -> dict[str, Any]: @@ -621,6 +2423,13 @@ def summarize_json_like(value: Any) -> dict[str, Any]: def load_artifact_payload(artifact: dict[str, Any]) -> dict[str, Any]: data = artifact.get("data") + if isinstance(data, dict): + data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {} + payload_path = str(data_ref.get("payloadPath", "")).strip() + if payload_path: + payload = load_payload_from_file(pathlib.Path(payload_path)) + if isinstance(payload, dict): + return payload if isinstance(data, dict): return data path_value = artifact.get("path") @@ -675,6 +2484,250 @@ def list_source_entries(base_path: pathlib.Path, pattern: str) -> list[dict[str, return entries +def list_bos_entries( + bucket: str, + prefix: str, + pattern: str, + endpoint: str, + access_key_id: str, + secret_access_key: str, +) -> list[dict[str, Any]]: + client, scheme = build_bos_client(endpoint, access_key_id, secret_access_key) + normalized_prefix = bos_prefix(prefix) + prefix_with_slash = f"{normalized_prefix}/" if normalized_prefix else "" + entries: list[dict[str, Any]] = [] + for candidate in client.list_all_objects(bucket, prefix=normalized_prefix or None): + key = text_value(getattr(candidate, "key", "")).strip().lstrip("/") + if not key or key.endswith("/"): + continue + if normalized_prefix: + if key == normalized_prefix: + continue + if not key.startswith(prefix_with_slash): + continue + relative_path = key[len(prefix_with_slash):] + else: + relative_path = key + if not relative_path: + continue + if pattern and not fnmatch.fnmatch(relative_path, pattern): + continue + size_bytes = int(getattr(candidate, "size", 0) or 0) + mime_type = mimetypes.guess_type(relative_path)[0] or "" + storage_path = canonical_bos_path(bucket, key) + entry: dict[str, Any] = { + "path": relative_path, + "storagePath": storage_path, + "sizeBytes": size_bytes, + "mimeType": mime_type, + "etag": text_value(getattr(candidate, "e_tag", "") or getattr(candidate, "etag", "")), + "versionId": text_value(getattr(candidate, "version_id", "") or getattr(candidate, "versionId", "")), + "lastModified": text_value(getattr(candidate, "last_modified", "") or getattr(candidate, "lastModified", "")), + "source": {"path": storage_path}, + } + file_name = pathlib.PurePosixPath(relative_path).name.lower() + should_inline_json = relative_path.lower().endswith(".json") and ( + file_name in XPLATFORM_INLINE_JSON_FILES or size_bytes <= PREVIEW_INLINE_BINARY_LIMIT + ) + if should_inline_json: + raw = client.get_object_as_string(bucket, key) + payload_text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw) + try: + entry["data"] = json.loads(payload_text) + except Exception: + entry["data"] = {"value": payload_text} + elif pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"): + entry["source"]["previewURL"] = client.generate_pre_signed_url( + bucket, + key, + expiration_in_seconds=3600, + protocol=scheme, + ) + entries.append(clean_metadata(entry) or {}) + return entries + + +def source_entry_fingerprint(data_source: str, bucket: str, entry: dict[str, Any]) -> str: + normalized_source = (data_source or "BOS").strip().upper() or "BOS" + relative_path = str(entry.get("path", "")).strip() + version_id = str(entry.get("versionId", "")).strip() + sha256 = str(entry.get("sha256", "")).strip() + etag = str(entry.get("etag", "")).strip() + size_bytes = str(entry.get("sizeBytes", "")).strip() + last_modified = str(entry.get("lastModified", "")).strip() + parts = [normalized_source, str(bucket or "").strip(), relative_path] + if version_id: + parts.append(f"version:{version_id}") + elif sha256: + parts.append(f"sha256:{sha256}") + else: + parts.extend([f"etag:{etag}", f"size:{size_bytes}", f"lastModified:{last_modified}"]) + raw = "\n".join(parts).encode("utf-8") + return hashlib.sha256(raw).hexdigest() + + +def incremental_unit(params: dict[str, str]) -> str: + return "DIRECTORY" if (params.get("INCREMENTAL_UNIT") or "").strip().upper() == "DIRECTORY" else "OBJECT" + + +def incremental_directory_depth(params: dict[str, str]) -> int: + raw = (params.get("INCREMENTAL_DIRECTORY_DEPTH") or "").strip() + try: + value = int(raw) + except Exception: + value = 1 + return value if value > 0 else 1 + + +def directory_incremental_key(relative_path: str, depth: int) -> str: + clean_relative = str(relative_path or "").strip().lstrip("/") + if not clean_relative: + return "__root__" + parts = [segment for segment in pathlib.PurePosixPath(clean_relative).parts if segment not in ("", ".")] + if len(parts) <= 1: + return f"FILE::{clean_relative}" + directory_parts = parts[:-1] + take = min(max(depth, 1), len(directory_parts)) + return "/".join(directory_parts[:take]) + + +def incremental_source_state(params: dict[str, str]) -> dict[str, Any]: + raw = params.get("SOURCE_STATE_JSON", "").strip() + if not raw: + return {"seenFingerprints": []} + try: + parsed = json.loads(raw) + except Exception: + return {"seenFingerprints": []} + if not isinstance(parsed, dict): + return {"seenFingerprints": []} + seen = parsed.get("seenFingerprints") + if not isinstance(seen, list): + parsed["seenFingerprints"] = [] + else: + parsed["seenFingerprints"] = [str(item).strip() for item in seen if str(item).strip()] + parsed["incrementalUnit"] = "DIRECTORY" if str(parsed.get("incrementalUnit", "")).strip().upper() == "DIRECTORY" else "OBJECT" + try: + parsed["directoryDepth"] = int(parsed.get("directoryDepth", 1) or 1) + except Exception: + parsed["directoryDepth"] = 1 + if parsed["directoryDepth"] <= 0: + parsed["directoryDepth"] = 1 + return parsed + + +def apply_incremental_source_filter( + entries: list[dict[str, Any]], + data_source: str, + bucket: str, + params: dict[str, str], + source_path: str, + prefix: str, + pattern: str, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + state = incremental_source_state(params) + seen = {str(item).strip() for item in state.get("seenFingerprints", []) if str(item).strip()} + scan_started_at = now_iso() + mode = incremental_unit(params) + depth = incremental_directory_depth(params) + emitted: list[dict[str, Any]] = [] + updated_seen = set(seen) + if mode == "DIRECTORY": + grouped_entries: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + group_key = directory_incremental_key(str(entry.get("path", "")).strip(), depth) + grouped_entries.setdefault(group_key, []).append(entry) + for group_key in sorted(grouped_entries.keys()): + group_entries = grouped_entries[group_key] + if group_key in updated_seen: + continue + updated_seen.add(group_key) + for entry in group_entries: + enriched = dict(entry) + enriched["fingerprint"] = source_entry_fingerprint(data_source, bucket, entry) + enriched["incrementalKey"] = group_key + emitted.append(enriched) + else: + for entry in entries: + fingerprint = source_entry_fingerprint(data_source, bucket, entry) + enriched = dict(entry) + enriched["fingerprint"] = fingerprint + enriched["incrementalKey"] = fingerprint + if fingerprint in updated_seen: + continue + updated_seen.add(fingerprint) + emitted.append(enriched) + scan_completed_at = now_iso() + checkpoint = { + "sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(), + "scanMode": "INCREMENTAL", + "incrementalUnit": mode, + "directoryDepth": depth, + "dataSource": data_source, + "sourcePath": source_path, + "bucket": bucket, + "prefix": prefix, + "pattern": pattern, + "seenFingerprints": sorted(updated_seen), + "listedCount": len(entries), + "newCount": len(emitted), + "scanStartedAt": scan_started_at, + "scanCompletedAt": scan_completed_at, + } + return emitted, checkpoint + + +def preview_payload_for_remote_entry(relative_path: str, mime_type: str, preview_url: str) -> dict[str, Any] | None: + normalized_preview_url = text_value(preview_url).strip() + if not normalized_preview_url: + return None + suffix = pathlib.PurePosixPath(relative_path).suffix.lower() + kind = "binary" + if suffix in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"): + kind = "video" + elif suffix in IMAGE_FILE_SUFFIXES or mime_type.startswith("image/"): + kind = "image" + elif suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"): + kind = "audio" + return {"preview": {"kind": kind, "format": suffix.lstrip(".") or kind, "url": normalized_preview_url}} + + +def source_entry_artifact( + run_id: str, + node_id: str, + entry: dict[str, Any], + source_path: str, + output: dict[str, Any] | None = None, +) -> dict[str, Any]: + relative_path = str(entry.get("path", "")).strip().lstrip("/") + source = manifest_source_ref(entry, relative_path, source_path) + storage_path = str(source.get("path", "")).strip() or source_path_join(source_path, relative_path) + mime_type = str(entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or "" + artifact_type = infer_port_type(pathlib.Path(relative_path)) + payload: dict[str, Any] | None = None + if "data" in entry: + payload = normalize_json_payload(entry.get("data")) + else: + preview_payload = preview_payload_for_remote_entry(relative_path, mime_type, str(source.get("previewURL", "")).strip()) + payload = preview_payload if preview_payload else None + if payload is not None: + payload["relativePath"] = relative_path + if source: + payload["source"] = source + metadata = clean_metadata({"mimeType": mime_type, "sha256": entry.get("sha256", "")}) + return build_virtual_artifact( + run_id, + node_id, + pathlib.PurePosixPath(relative_path).name or "artifact", + storage_path, + artifact_type, + int(entry.get("sizeBytes", 0) or 0), + data=payload, + metadata=metadata, + output=output, + ) + + def collect_output_artifacts( run_id: str, node_id: str, @@ -737,6 +2790,37 @@ def collect_items(payloads: list[dict[str, Any]]) -> list[Any]: return items +def normalize_mongo_document(item: Any) -> dict[str, Any]: + if isinstance(item, dict): + if set(item.keys()) == {"value"} and isinstance(item.get("value"), dict): + return json_compatible(item["value"]) + return json_compatible(item) + return {"value": json_compatible(item)} + + +def documents_from_inputs(payloads: list[dict[str, Any]], document_path: str) -> list[dict[str, Any]]: + raw_items: list[Any] = [] + path = str(document_path or "").strip() + if path: + for payload in payloads: + value = get_field_value(payload, path) + if isinstance(value, list): + raw_items.extend(value) + elif value is not None: + raw_items.append(value) + else: + raw_items = collect_items(payloads) + return [normalize_mongo_document(item) for item in raw_items] + + +def build_mongo_client(uri: str): + try: + from pymongo import MongoClient + except Exception as exc: + raise RuntimeError("pymongo is required for MongoDB 写入算子") from exc + return MongoClient(uri, serverSelectionTimeoutMS=5000) + + def get_field_value(value: Any, field_path: str) -> Any: current = value for token in [part for part in field_path.split(".") if part]: @@ -871,6 +2955,287 @@ def write_csv_artifact( return build_artifact(run_id, node_id, path, "file", data=preview, output=output) +def write_video_segment(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]: + cv2, _ = require_cv2() + capture = cv2.VideoCapture(str(source_path)) + if not capture.isOpened(): + raise RuntimeError(f"无法打开视频文件: {source_path}") + try: + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0 + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64 + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64 + frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + start_frame = max(0, int(start_second * fps)) + end_frame = max(start_frame + 1, int(math.ceil(end_second * fps))) + if frame_count > 0: + end_frame = min(end_frame, frame_count) + capture.set(cv2.CAP_PROP_POS_FRAMES, start_frame) + destination.parent.mkdir(parents=True, exist_ok=True) + writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) + if not writer.isOpened(): + raise RuntimeError(f"无法创建视频切片: {destination}") + written = 0 + try: + for _ in range(start_frame, end_frame): + ok, frame = capture.read() + if not ok: + break + writer.write(frame) + written += 1 + finally: + writer.release() + duration = written / fps if fps > 0 else 0.0 + return { + "frameRate": round(fps, 3), + "frameCount": written, + "resolution": f"{width}x{height}", + "duration": round(duration, 3), + } + finally: + capture.release() + + +def ffmpeg_available() -> bool: + return shutil.which("ffmpeg") is not None + + +def run_ffmpeg(args: list[str]) -> None: + completed = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if completed.returncode == 0: + return + stderr = (completed.stderr or "").strip() + stdout = (completed.stdout or "").strip() + message = stderr or stdout or "unknown ffmpeg error" + raise RuntimeError(f"ffmpeg execution failed: {message}") + + +def write_video_segment_with_ffmpeg(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]: + if not ffmpeg_available(): + raise RuntimeError("ffmpeg is not available in operator-runner image") + duration = max(float(end_second) - float(start_second), 0.001) + destination.parent.mkdir(parents=True, exist_ok=True) + copy_args = [ + "ffmpeg", + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-ss", + f"{max(float(start_second), 0.0):.3f}", + "-i", + str(source_path), + "-t", + f"{duration:.3f}", + "-map", + "0:v:0", + "-map", + "0:a?", + "-c", + "copy", + str(destination), + ] + reencode_args = [ + "ffmpeg", + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-ss", + f"{max(float(start_second), 0.0):.3f}", + "-i", + str(source_path), + "-t", + f"{duration:.3f}", + "-map", + "0:v:0", + "-map", + "0:a?", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + str(destination), + ] + try: + run_ffmpeg(copy_args) + except RuntimeError: + run_ffmpeg(reencode_args) + metrics = inspect_video_file(destination) + return { + "frameRate": metrics.get("frameRate"), + "frameCount": metrics.get("frameCount"), + "resolution": metrics.get("resolution"), + "duration": metrics.get("duration", round(duration, 3)), + } + + +def write_video_segments( + source_path: pathlib.Path | str, + segments: list[dict[str, Any]], +) -> list[dict[str, Any]]: + cv2, _ = require_cv2() + capture = cv2.VideoCapture(str(source_path)) + if not capture.isOpened(): + raise RuntimeError(f"无法打开视频文件: {source_path}") + try: + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0 + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64 + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64 + frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + normalized_segments: list[dict[str, Any]] = [] + for raw in segments: + destination = pathlib.Path(raw.get("destination", "")) + start_second = float(raw.get("startSecond") or 0.0) + end_second = float(raw.get("endSecond") or 0.0) + start_frame = max(0, int(start_second * fps)) + end_frame = max(start_frame + 1, int(math.ceil(end_second * fps))) + if frame_count > 0: + end_frame = min(end_frame, frame_count) + normalized_segments.append( + { + **raw, + "destination": destination, + "startFrame": start_frame, + "endFrame": end_frame, + } + ) + normalized_segments.sort(key=lambda item: (int(item["startFrame"]), str(item["destination"]))) + current_frame = 0 + if normalized_segments: + initial_frame = int(normalized_segments[0]["startFrame"]) + if initial_frame > 0: + capture.set(cv2.CAP_PROP_POS_FRAMES, initial_frame) + current_frame = initial_frame + results: list[dict[str, Any]] = [] + for segment in normalized_segments: + start_frame = int(segment["startFrame"]) + end_frame = int(segment["endFrame"]) + destination = pathlib.Path(segment["destination"]) + if current_frame < start_frame: + while current_frame < start_frame: + ok, _ = capture.read() + if not ok: + current_frame = end_frame + break + current_frame += 1 + destination.parent.mkdir(parents=True, exist_ok=True) + writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) + if not writer.isOpened(): + raise RuntimeError(f"无法创建视频切片: {destination}") + written = 0 + try: + while current_frame < end_frame: + ok, frame = capture.read() + if not ok: + break + writer.write(frame) + written += 1 + current_frame += 1 + finally: + writer.release() + results.append( + { + **segment, + "frameRate": round(fps, 3), + "frameCount": written, + "resolution": f"{width}x{height}", + "duration": round(written / fps, 3) if fps > 0 else 0.0, + } + ) + return results + finally: + capture.release() + + +def write_video_segment_from_remote_url(source_url: str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]: + return write_video_segment(source_url, destination, start_second, end_second) + + +def annotate_video_clip(path: pathlib.Path, frame_interval_seconds: float) -> dict[str, Any]: + cv2, np = require_cv2() + capture = cv2.VideoCapture(str(path)) + if not capture.isOpened(): + raise RuntimeError(f"无法打开视频切片: {path}") + try: + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0 + frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0 + step = max(1, int(round(max(frame_interval_seconds, 0.1) * fps))) + previous_gray = None + samples: list[tuple[float, float]] = [] + objects: set[str] = set() + frame_index = 0 + while True: + ok, frame = capture.read() + if not ok: + break + if frame_index % step != 0: + frame_index += 1 + continue + timestamp = frame_index / fps if fps > 0 else 0.0 + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + motion = 0.0 + if previous_gray is not None: + diff = cv2.absdiff(previous_gray, gray) + motion = float(diff.mean() / 255.0) + previous_gray = gray + ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb) + mask = cv2.inRange(ycrcb, np.array((0, 133, 77), dtype=np.uint8), np.array((255, 173, 127), dtype=np.uint8)) + if float(mask.mean() / 255.0) > 0.1: + objects.add("手部") + _, threshold = cv2.threshold(gray, 24, 255, cv2.THRESH_BINARY) + contours, _ = cv2.findContours(threshold, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + if any(cv2.contourArea(contour) >= 64 for contour in contours): + objects.add("前景目标") + samples.append((timestamp, motion)) + frame_index += 1 + if not samples: + samples.append((0.0, 0.0)) + + def label_for_motion(motion: float) -> tuple[str, float]: + if motion < 0.015: + return "静止观察", 0.88 + if motion < 0.045: + return "平稳活动", 0.79 + return "快速移动", 0.71 + + actions: list[dict[str, Any]] = [] + last_timestamp = duration + for timestamp, motion in samples: + label, confidence = label_for_motion(motion) + last_timestamp = timestamp + if not actions: + actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)}) + continue + current = actions[-1] + if current["label"] != label: + current["end"] = round(timestamp, 3) + actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)}) + else: + current["end"] = round(timestamp, 3) + current["confidence"] = round(min(float(current["confidence"]), confidence), 3) + if actions: + actions[-1]["end"] = round(max(duration, last_timestamp), 3) + min_confidence = min(float(item["confidence"]) for item in actions) if actions else 0.0 + return { + "duration": round(duration, 3), + "actions": actions, + "objects": sorted(objects), + "minConfidence": round(min_confidence, 3), + "frameIntervalSeconds": frame_interval_seconds, + } + finally: + capture.release() + + def builtin_source( run_id: str, node_id: str, @@ -879,17 +3244,29 @@ def builtin_source( inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) + inline_source = bool(params.get("SOURCE_JSON") or params.get("SOURCE_TEXT")) + data_source = (params.get("DATA_SOURCE") or os.getenv("DATA_SOURCE", "")).strip().upper() + if not data_source: + data_source = "INLINE" if inline_source else "BOS" source_path = params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "").strip() prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip() bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip() pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip() + bos_endpoint = params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "").strip() + bos_access_key_id = params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "").strip() + bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "").strip() + scan_mode = (params.get("SCAN_MODE") or "").strip().upper() + unit = incremental_unit(params) + depth = incremental_directory_depth(params) + source_bucket, source_prefix = parse_bos_path(source_path) + if not bucket and source_bucket: + bucket = source_bucket + if not prefix and source_prefix: + prefix = source_prefix entries: list[dict[str, Any]] = [] - if source_path: - entries = list_source_entries(pathlib.Path(source_path), pattern) - elif prefix and pathlib.Path(prefix).exists(): - entries = list_source_entries(pathlib.Path(prefix), pattern) - elif params.get("SOURCE_JSON"): + resolved_source_path = source_path + if params.get("SOURCE_JSON"): parsed = json.loads(params["SOURCE_JSON"]) if isinstance(parsed, list): entries = [{"value": item} for item in parsed] @@ -897,17 +3274,78 @@ def builtin_source( entries = [{"value": parsed}] elif params.get("SOURCE_TEXT"): entries = [{"text": params["SOURCE_TEXT"]}] + elif data_source == "BOS" or (bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key): + cleaned_prefix = bos_prefix(prefix) + entries = list_bos_entries( + bucket, + cleaned_prefix, + pattern, + bos_endpoint, + bos_access_key_id, + bos_secret_access_key, + ) + resolved_source_path = canonical_bos_path(bucket, cleaned_prefix) + elif source_path: + entries = list_source_entries(pathlib.Path(source_path), pattern) + elif prefix and pathlib.Path(prefix).exists(): + entries = list_source_entries(pathlib.Path(prefix), pattern) elif inputs: entries = [load_artifact_payload(item) for item in inputs] + listed_entries = list(entries) + checkpoint_payload = None + if scan_mode == "INCREMENTAL": + entries, checkpoint_payload = apply_incremental_source_filter( + listed_entries, + data_source, + bucket, + params, + resolved_source_path, + prefix, + pattern, + ) + payload = { + "dataSource": data_source, "bucket": bucket, "prefix": prefix, + "endpoint": bos_endpoint, "pattern": pattern, - "sourcePath": source_path, + "scanMode": scan_mode or "FULL", + "incrementalUnit": unit, + "directoryDepth": depth, + "sourcePath": resolved_source_path, + "sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(), + "listedCount": len(listed_entries), + "newCount": len(entries) if checkpoint_payload is not None else len(entries), "entries": entries, + "catalogEntries": listed_entries, } - return [write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data))] + artifacts = [ + write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data)) + ] + if checkpoint_payload is not None: + artifacts.append(write_json_artifact(run_id, node_id, output_dir, "source-checkpoint.json", checkpoint_payload)) + + if data_source == "BOS": + for entry in entries: + artifacts.append(source_entry_artifact(run_id, node_id, entry, resolved_source_path, output=output_binding(node_data))) + elif source_path and pathlib.Path(source_path).exists(): + source_root = pathlib.Path(source_path) + staged_root = output_dir / "_source_tree" + for entry in entries: + relative_path = str(entry.get("path", "")).strip() + if not relative_path: + continue + candidate = source_root / relative_path if source_root.is_dir() else source_root + if not candidate.exists() or not candidate.is_file(): + continue + destination = staged_root / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(candidate, destination) + artifacts.extend(collect_output_artifacts(run_id, node_id, staged_root, node_data)) + + return artifacts def builtin_merge( @@ -986,12 +3424,166 @@ def builtin_upload( node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: + params = hyperparams(node_data) + upload_path = params.get("OUTPUT_PATH", "").strip() or params.get("SOURCE_PATH", "").strip() + bucket = str(node_data.get("bucket", "")).strip() + prefix = str(node_data.get("prefix", "")).strip() + parsed_bucket, parsed_prefix = parse_bos_path(upload_path) + if parsed_bucket: + bucket = parsed_bucket + if parsed_prefix or upload_path.endswith("/"): + prefix = parsed_prefix + data_source = (params.get("DATA_SOURCE", "") or ("BOS" if bucket and upload_path.startswith("bos://") else "")).strip().upper() + bos_endpoint = params.get("BOS_ENDPOINT", "").strip() + bos_access_key_id = params.get("BOS_ACCESS_KEY_ID", "").strip() + bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY", "").strip() copied_paths: list[str] = [] bundle_inputs: list[dict[str, Any]] = [] staging_dir = output_dir / "uploaded" staging_dir.mkdir(parents=True, exist_ok=True) + package_manifest_payload = first_payload_with_keys(inputs, "packageCount", "items") + upload_inputs = list(inputs) + if package_manifest_payload is not None: + package_names = package_related_artifact_names(package_manifest_payload) + upload_inputs = [ + artifact + for artifact in inputs + if str(artifact.get("name", "")).strip() not in package_names + ] + upload_inputs.extend( + materialize_package_artifacts_from_manifest( + run_id, + node_id, + staging_dir, + package_manifest_payload, + node_data, + ) + ) - for artifact in inputs: + if data_source == "BOS" and bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key: + client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key) + temp_paths = [ + str(item.get("path", "")).strip() + for item in upload_inputs + if str(item.get("path", "")).strip() and "://" not in str(item.get("path", "")).strip() + ] + input_root = common_input_root(temp_paths) + uploaded_objects: list[dict[str, Any]] = [] + remote_artifacts: list[dict[str, Any]] = [] + + for index, artifact in enumerate(upload_inputs): + payload = load_artifact_payload(artifact) + bundle_inputs.append(payload) + fallback_name = str(artifact.get("name", "")).strip() or f"artifact-{index}" + relative_path = upload_relative_path(artifact, input_root, fallback_name) + temp_source = materialized_upload_source(artifact, staging_dir, relative_path) + if temp_source is None or not temp_source.exists(): + continue + + mime_type = ( + str((artifact.get("metadata") or {}).get("mimeType", "")).strip() + if isinstance(artifact.get("metadata"), dict) + else "" + ) or mimetypes.guess_type(temp_source.name)[0] or "" + object_key = bos_key_join(prefix, relative_path) + put_kwargs = {"content_type": mime_type} if mime_type else {} + try: + size_bytes = temp_source.stat().st_size + except OSError: + size_bytes = 0 + log( + f"[operator-runner][info] upload starts: {temp_source} -> {canonical_bos_path(bucket, object_key)} " + f"({size_bytes} bytes)" + ) + client.put_object_from_file(bucket, object_key, str(temp_source), **put_kwargs) + remote_path = canonical_bos_path(bucket, object_key) + log( + f"[operator-runner][info] upload finished: {temp_source} -> {remote_path} " + f"({size_bytes} bytes)" + ) + preview_url = "" + if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"): + preview_url = text_value( + client.generate_pre_signed_url( + bucket, + object_key, + expiration_in_seconds=3600, + protocol=scheme, + ) + ).strip() + + uploaded_objects.append( + clean_metadata( + { + "name": pathlib.PurePosixPath(relative_path).name, + "relativePath": relative_path, + "path": remote_path, + "previewURL": preview_url, + "sizeBytes": temp_source.stat().st_size, + "mimeType": mime_type, + } + ) + or {} + ) + + artifact_type = str(artifact.get("type", "")).strip() or infer_port_type(temp_source) + artifact_payload: dict[str, Any] | None = None + if artifact_type == "json": + loaded = load_payload_from_file(temp_source) + if loaded is not None: + artifact_payload = normalize_json_payload(loaded) + else: + artifact_payload = preview_payload_for_remote_entry(relative_path, mime_type, preview_url) + if artifact_payload is None: + artifact_payload = extract_preview_data(temp_source, artifact_type, mime_type) + if artifact_payload is not None: + artifact_payload["relativePath"] = relative_path + artifact_payload["source"] = clean_metadata( + { + "path": remote_path, + "previewURL": preview_url, + "relativePath": relative_path, + "mimeType": mime_type, + } + ) or {} + + source_size_bytes = temp_source.stat().st_size + source_sha256, source_sha256_metadata = artifact_sha256(temp_source, source_size_bytes) + + remote_artifacts.append( + build_virtual_artifact( + run_id, + node_id, + pathlib.PurePosixPath(relative_path).name, + remote_path, + artifact_type, + source_size_bytes, + data=artifact_payload, + metadata={ + **source_sha256_metadata, + "sha256": source_sha256, + "mimeType": mime_type, + }, + output=output_binding(node_data), + ) + ) + + payload = { + "copiedPaths": copied_paths, + "uploadedCount": len(uploaded_objects), + "uploadedObjects": uploaded_objects, + "items": bundle_inputs, + "inputCount": len(inputs), + "dataSource": "BOS", + "bucket": bucket, + "prefix": prefix, + "destinationPath": canonical_bos_path(bucket, prefix), + } + artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))] + artifacts.extend(remote_artifacts) + return artifacts + + for artifact in upload_inputs: payload = load_artifact_payload(artifact) bundle_inputs.append(payload) source_path = artifact.get("path") @@ -1062,6 +3654,100 @@ def builtin_json_map( return [write_json_artifact(run_id, node_id, output_dir, "mapped.json", payload, output=output_binding(node_data))] +def builtin_mongodb_write( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + mongo_target = params.get("MONGO_TARGET", "").strip().lower() or "business" + explicit_mongo_uri = params.get("MONGO_URI", "").strip() or os.getenv("MONGO_URI", "").strip() + explicit_mongo_db = params.get("MONGO_DB", "").strip() or os.getenv("MONGO_DB", "").strip() + collection_name = params.get("MONGO_COLLECTION", "").strip() + mode = params.get("MONGO_MODE", "insert_many").strip().lower() or "insert_many" + upsert_key = params.get("UPSERT_KEY", "_id").strip() or "_id" + document_path = params.get("DOCUMENT_PATH", "").strip() + + mongo_uri = explicit_mongo_uri + mongo_db = explicit_mongo_db + if not mongo_uri: + if mongo_target == "platform": + mongo_uri = os.getenv("PLATFORM_MONGO_URI", "").strip() + mongo_db = mongo_db or os.getenv("PLATFORM_MONGO_DB", "").strip() + else: + mongo_uri = os.getenv("BUSINESS_MONGO_URI", "").strip() + mongo_db = mongo_db or os.getenv("BUSINESS_MONGO_DB", "").strip() + + if not mongo_uri: + raise RuntimeError(f"MongoDB 写入缺少可用连接:target={mongo_target}") + if not mongo_db: + raise RuntimeError(f"MongoDB 写入缺少数据库名:target={mongo_target}") + if not collection_name: + raise RuntimeError("MONGO_COLLECTION is required for MongoDB 写入") + if mongo_target == "platform" and collection_name.lower() in {item.lower() for item in RESERVED_MONGO_COLLECTIONS}: + raise RuntimeError(f"collection {collection_name} is reserved and cannot be written by workflow operators") + if mode not in {"insert_many", "upsert"}: + raise RuntimeError("MONGO_MODE must be insert_many or upsert") + + documents = documents_from_inputs(payloads_from_inputs(inputs), document_path) + if not documents: + raise RuntimeError("MongoDB 写入未收到可写入的文档") + + client = build_mongo_client(mongo_uri) + try: + collection = client[mongo_db][collection_name] + summary: dict[str, Any] = { + "target": mongo_target, + "database": mongo_db, + "collection": collection_name, + "mode": mode, + "documentCount": len(documents), + } + if document_path: + summary["documentPath"] = document_path + if mode == "insert_many": + result = collection.insert_many(documents, ordered=False) + summary["insertedCount"] = len(getattr(result, "inserted_ids", []) or []) + else: + processed = 0 + matched_count = 0 + modified_count = 0 + upserted_ids: list[Any] = [] + for document in documents: + key_value = get_field_value(document, upsert_key) + if key_value is None: + raise RuntimeError(f"document is missing UPSERT_KEY={upsert_key}") + result = collection.replace_one({upsert_key: key_value}, document, upsert=True) + processed += 1 + matched_count += int(getattr(result, "matched_count", 0) or 0) + modified_count += int(getattr(result, "modified_count", 0) or 0) + upserted_id = getattr(result, "upserted_id", None) + if upserted_id is not None: + upserted_ids.append(json_compatible(upserted_id)) + summary["upsertKey"] = upsert_key + summary["upsertedCount"] = processed + summary["matchedCount"] = matched_count + summary["modifiedCount"] = modified_count + if upserted_ids: + summary["upsertedIds"] = upserted_ids + return [ + write_json_artifact( + run_id, + node_id, + output_dir, + "mongodb-write-result.json", + summary, + output=output_binding(node_data), + ) + ] + finally: + close = getattr(client, "close", None) + if callable(close): + close() + + def builtin_text_template( run_id: str, node_id: str, @@ -1178,6 +3864,483 @@ def builtin_directory_scan( return [write_json_artifact(run_id, node_id, output_dir, "directory-scan.json", payload, output=output_binding(node_data))] +def builtin_xplatform_index( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + payload = load_xplatform_index_payload(hyperparams(node_data), inputs, output_dir) + return [write_json_artifact(run_id, node_id, output_dir, "dataset-index.json", payload, output=output_binding(node_data))] + + +def builtin_collection_validate( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + require_intrinsics = parse_bool_param(params, "REQUIRE_INTRINSICS", False) + payload = first_payload_with_keys(inputs, "collections", "items") + if payload is None: + payload = load_xplatform_index_payload(params, inputs, output_dir) + collections = [] + valid_collection_count = 0 + for collection in payload.get("collections", []): + if not isinstance(collection, dict): + continue + missing_required = list(collection.get("missingRequiredFiles", [])) + if require_intrinsics and not collection.get("hasIntrinsics", False): + missing_required.append("intrinsics.json") + valid = len(missing_required) == 0 and int(collection.get("videoCount", 0) or 0) > 0 + if valid: + valid_collection_count += 1 + collections.append({**collection, "missingRequiredFiles": missing_required, "valid": valid}) + items = [] + for item in payload.get("items", []): + if not isinstance(item, dict): + continue + collection_id = str(item.get("collectionId", "")).strip() + items.append({**item, "collectionValid": any(c.get("id") == collection_id and c.get("valid", False) for c in collections)}) + result = { + **payload, + "datasetType": "x-platform-validation", + "collections": collections, + "items": items, + "validCollectionCount": valid_collection_count, + "invalidCollectionCount": max(len(collections) - valid_collection_count, 0), + } + return [write_json_artifact(run_id, node_id, output_dir, "dataset-validation.json", result, output=output_binding(node_data))] + + +def builtin_video_quality( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + min_clarity = parse_float_param(params, "MIN_CLARITY", 0.8) + max_shake = parse_float_param(params, "MAX_SHAKE", 0.1) + max_hand_occlusion = parse_float_param(params, "MAX_HAND_OCCLUSION", 0.1) + clarity_scale = parse_float_param(params, "CLARITY_SCALE", 35.0) + payload = first_payload_with_keys(inputs, "items") + if payload is None: + payload = load_xplatform_index_payload(params, inputs, output_dir) + archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {} + items = [] + passed_count = 0 + for item in payload.get("items", []): + if not isinstance(item, dict): + continue + source_path = source_path_from_item(item, str(item.get("filename", "video.mp4")), output_dir, archive_sources) + metrics = analyze_video_quality_metrics(source_path, clarity_scale) + passed = ( + float_value(metrics.get("clarity"), 0.0) >= min_clarity + and float_value(metrics.get("shake"), 1.0) <= max_shake + and float_value(metrics.get("handOcclusion"), 1.0) <= max_hand_occlusion + ) + if passed: + passed_count += 1 + items.append( + { + **item, + "quality": { + **metrics, + "passed": passed, + "thresholds": { + "minClarity": min_clarity, + "maxShake": max_shake, + "maxHandOcclusion": max_hand_occlusion, + }, + }, + } + ) + result = { + **payload, + "datasetType": "x-platform-quality", + "items": items, + "videoCount": len(items), + "passedCount": passed_count, + "failedCount": max(len(items) - passed_count, 0), + } + return [write_json_artifact(run_id, node_id, output_dir, "quality-report.json", result, output=output_binding(node_data))] + + +def builtin_dataset_filter( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + payload = first_payload_with_keys(inputs, "items") + if payload is None: + raise RuntimeError("数据过滤需要上游数据清单输入") + date_start = normalize_date_value(params.get("DATE_START", "")) + date_end = normalize_date_value(params.get("DATE_END", "")) + place = params.get("PLACE", "").strip() + collection_type = params.get("TYPE", "").strip() + min_clarity = parse_float_param(params, "MIN_CLARITY", 0.0) + max_shake = parse_float_param(params, "MAX_SHAKE", 1.0) + max_hand_occlusion = parse_float_param(params, "MAX_HAND_OCCLUSION", 1.0) + filtered = [] + for item in payload.get("items", []): + if not isinstance(item, dict): + continue + meta = item.get("meta") if isinstance(item.get("meta"), dict) else {} + quality = item.get("quality") if isinstance(item.get("quality"), dict) else {} + if place and str(meta.get("place", "")).strip() != place: + continue + if collection_type and str(meta.get("type", "")).strip() != collection_type: + continue + if date_start or date_end: + if not date_in_range(meta.get("date", ""), date_start, date_end): + continue + if float_value(quality.get("clarity"), 0.0) < min_clarity: + continue + if float_value(quality.get("shake"), 1.0) > max_shake: + continue + if float_value(quality.get("handOcclusion"), 1.0) > max_hand_occlusion: + continue + if not bool(quality.get("passed", False)): + continue + filtered.append(item) + result = { + "datasetType": "x-platform-filtered", + "criteria": { + "dateStart": date_start, + "dateEnd": date_end, + "place": place, + "type": collection_type, + "minClarity": min_clarity, + "maxShake": max_shake, + "maxHandOcclusion": max_hand_occlusion, + }, + "count": len(filtered), + "items": filtered, + } + if isinstance(payload.get("archiveSources"), dict): + result["archiveSources"] = payload["archiveSources"] + return [write_json_artifact(run_id, node_id, output_dir, "filtered-videos.json", result, output=output_binding(node_data))] + + +def builtin_video_slicing( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + payload = first_payload_with_keys(inputs, "items") + if payload is None: + raise RuntimeError("视频切片需要上游过滤结果输入") + segment_duration = max(1, parse_int_param(params, "SEGMENT_DURATION_SECONDS", 300)) + archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {} + clips: list[dict[str, Any]] = [] + for item in payload.get("items", []): + if not isinstance(item, dict): + continue + source_filename = str(item.get("filename", "")).strip() or "video.mp4" + source_path = source_path_from_item(item, source_filename, output_dir, archive_sources) + metrics = inspect_video_file(source_path) + duration = float(item.get("duration") or metrics.get("duration") or 0.0) + if duration <= 0: + duration = float(metrics.get("duration") or 0.0) + if duration <= 0: + duration = float(segment_duration) + clip_count = max(1, int(math.ceil(duration / segment_duration))) + stem = pathlib.Path(source_filename).stem + clip_specs: list[dict[str, Any]] = [] + for clip_index in range(clip_count): + clip_start = clip_index * segment_duration + clip_end = min(duration, clip_start + segment_duration) + clip_name = f"{stem}_clip{clip_index + 1:03d}.mp4" + clip_path = output_dir / "clips" / clip_name + clip_specs.append( + { + "clipIndex": clip_index, + "clipStart": clip_start, + "clipEnd": clip_end, + "clipName": clip_name, + "clipPath": clip_path, + "destination": clip_path, + "startSecond": clip_start, + "endSecond": clip_end, + } + ) + clip_metrics_list: list[dict[str, Any]] | None = None + if ffmpeg_available(): + try: + log(f"[operator-runner][info] video_slicing uses ffmpeg for {source_filename} ({clip_count} clip(s))") + clip_metrics_list = [] + for clip_spec in clip_specs: + clip_metrics_list.append( + { + **clip_spec, + **write_video_segment_with_ffmpeg( + source_path, + pathlib.Path(clip_spec["clipPath"]), + float(clip_spec["clipStart"]), + float(clip_spec["clipEnd"]), + ), + } + ) + except RuntimeError as exc: + log(f"[operator-runner][warn] ffmpeg video slicing failed for {source_filename}, fallback to opencv: {exc}") + clip_metrics_list = None + if clip_metrics_list is None: + if len(clip_specs) > 1: + clip_metrics_list = write_video_segments(source_path, clip_specs) + else: + only = clip_specs[0] + clip_metrics_list = [ + { + **only, + **write_video_segment(source_path, only["clipPath"], float(only["clipStart"]), float(only["clipEnd"])), + } + ] + for clip_spec in clip_metrics_list: + clip_start = float(clip_spec["clipStart"]) + clip_end = float(clip_spec["clipEnd"]) + clip_name = str(clip_spec["clipName"]) + clip_path = pathlib.Path(clip_spec["clipPath"]) + clips.append( + { + "clipId": pathlib.Path(clip_name).stem, + "clipName": clip_name, + "clipPath": str(clip_path), + "sourceVideo": source_filename, + "sourcePath": str(source_path), + "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, + "collectionId": item.get("collectionId", ""), + "meta": item.get("meta", {}), + "quality": item.get("quality", {}), + "startSecond": round(clip_start, 3), + "endSecond": round(clip_end, 3), + "duration": clip_spec.get("duration", round(max(clip_end - clip_start, 0.0), 3)), + "resolution": clip_spec.get("resolution"), + "frameRate": clip_spec.get("frameRate"), + } + ) + manifest = { + "datasetType": "x-platform-clips", + "segmentDurationSeconds": segment_duration, + "clipCount": len(clips), + "items": clips, + } + artifacts = [write_json_artifact(run_id, node_id, output_dir, "clips-manifest.json", manifest, output=output_binding(node_data))] + artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "clips", node_data=node_data, output=output_binding(node_data))) + return artifacts + + +def builtin_video_annotation( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + payload = first_payload_with_keys(inputs, "clipCount", "items") + if payload is None: + raise RuntimeError("视频标注需要上游切片结果输入") + frame_interval = max(0.1, parse_float_param(params, "FRAME_INTERVAL_SECONDS", 1.0)) + clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")} + archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {} + items = [] + for clip in payload.get("items", []): + if not isinstance(clip, dict): + continue + clip_name = str(clip.get("clipName", "")).strip() or pathlib.Path(str(clip.get("clipPath", ""))).name + artifact = clip_artifacts.get(clip_name) + clip_path: pathlib.Path | None = None + if artifact is not None: + try: + clip_path = source_path_from_item( + {"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")}, + clip_name, + output_dir, + archive_sources, + ) + except RuntimeError: + clip_path = None + if clip_path is None: + direct_clip_path = str(clip.get("clipPath", "")).strip() + if direct_clip_path and pathlib.Path(direct_clip_path).exists(): + clip_path = pathlib.Path(direct_clip_path) + if clip_path is None: + clip_path = materialize_clip_from_source(clip, output_dir, archive_sources) + if clip_path is None or not clip_path.exists(): + raise RuntimeError(f"无法获取真实输入文件: {clip_name}") + annotation = annotate_video_clip(clip_path, frame_interval) + annotation_filename = f"{pathlib.Path(clip_name).stem}_annotation.json" + annotation_payload = { + "clipId": clip.get("clipId", pathlib.Path(clip_name).stem), + "sourceVideo": clip.get("sourceVideo", ""), + **annotation, + } + items.append({**clip, "annotation": annotation_payload, "annotationFile": annotation_filename}) + write_json_artifact( + run_id, + node_id, + output_dir / "annotations", + annotation_filename, + annotation_payload, + output=output_binding(node_data), + ) + manifest = { + "datasetType": "x-platform-annotations", + "annotationCount": len(items), + "items": items, + } + artifacts = [write_json_artifact(run_id, node_id, output_dir, "annotations-manifest.json", manifest, output=output_binding(node_data))] + artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "annotations", node_data=node_data, output=output_binding(node_data))) + return artifacts + + +def builtin_result_package( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + payload = first_payload_with_keys(inputs, "annotationCount", "items") + if payload is None: + raise RuntimeError("成品打包需要上游标注结果输入") + clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")} + archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {} + packages = [] + for item in payload.get("items", []): + if not isinstance(item, dict): + continue + clip_name = str(item.get("clipName", "")).strip() + clip_id = str(item.get("clipId", "")).strip() or pathlib.Path(clip_name).stem + package_name = f"{clip_id}_package" + package_dir = output_dir / "packages" / package_name + package_dir.mkdir(parents=True, exist_ok=True) + artifact = clip_artifacts.get(clip_name) + clip_path: pathlib.Path | None = None + if artifact is not None: + try: + clip_path = source_path_from_item( + {"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")}, + clip_name, + output_dir, + archive_sources, + ) + except RuntimeError: + clip_path = None + if clip_path is None: + direct_clip_path = str(item.get("clipPath", "")).strip() + if direct_clip_path and pathlib.Path(direct_clip_path).exists(): + clip_path = pathlib.Path(direct_clip_path) + if clip_path is None: + clip_path = materialize_clip_from_source(item, output_dir, archive_sources) + if clip_path is not None and clip_path.exists(): + shutil.copy2(clip_path, package_dir / clip_name) + quality_payload = { + "clip_id": clip_id, + "source_video": item.get("sourceVideo", ""), + **(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}), + } + annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {} + config_payload = package_config_payload(item, payload) + write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_quality.json", quality_payload, output=output_binding(node_data)) + write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_annotation.json", annotation_payload, output=output_binding(node_data)) + write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_config.json", config_payload, output=output_binding(node_data)) + package_files = [candidate for candidate in package_dir.rglob("*") if candidate.is_file()] + packages.append( + { + "packageName": package_name, + "clipId": clip_id, + "clipName": clip_name, + "sourceVideo": item.get("sourceVideo", ""), + "clipDuration": item.get("duration", 0), + "duration": item.get("duration", 0), + "startSecond": item.get("startSecond"), + "endSecond": item.get("endSecond"), + "collectionId": item.get("collectionId", ""), + "meta": item.get("meta", {}), + "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, + "fileCount": len(package_files), + "pipelineTags": ["quality_v1", "slicing_v1", "annotation_v1"], + "storagePath": str(package_dir), + "quality": quality_payload, + "annotation": annotation_payload, + } + ) + manifest = { + "datasetType": "x-platform-packages", + "packageCount": len(packages), + "items": packages, + } + artifacts = [write_json_artifact(run_id, node_id, output_dir, "packages-manifest.json", manifest, output=output_binding(node_data))] + artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "packages", node_data=node_data, output=output_binding(node_data))) + return artifacts + + +def builtin_audit_sampling( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + payload = first_payload_with_keys(inputs, "packageCount", "items") + if payload is None: + raise RuntimeError("抽检采样需要上游成品数据输入") + rules = parse_json_param(params, "RULES_JSON", []) + sampled_items = [] + summaries = [] + for rule in rules if isinstance(rules, list) else []: + if not isinstance(rule, dict): + continue + rule_id = str(rule.get("id", "rule")).strip() or "rule" + field_path = str(rule.get("field", "")).strip() + operator = str(rule.get("operator", "lt")).strip() or "lt" + value = rule.get("value") + percent = float(rule.get("samplePercent", 10) or 0) + matched = [] + sampled = [] + for item in payload.get("items", []): + if not isinstance(item, dict): + continue + if not compare_values(get_field_value(item, field_path), operator, value): + continue + matched.append(item) + package_name = str(item.get("packageName", "")).strip() or str(item.get("clipId", "")).strip() + if stable_sample_hit(f"{rule_id}:{package_name}", percent): + sampled.append(item) + sampled_items.append({**item, "sampleRuleId": rule_id}) + summaries.append( + { + "id": rule_id, + "field": field_path, + "operator": operator, + "value": value, + "samplePercent": percent, + "matchedCount": len(matched), + "sampledCount": len(sampled), + } + ) + manifest = { + "datasetType": "x-platform-audit-samples", + "rules": summaries, + "sampledCount": len(sampled_items), + "items": sampled_items, + } + return [write_json_artifact(run_id, node_id, output_dir, "audit-samples.json", manifest, output=output_binding(node_data))] + + def builtin_condition_router( run_id: str, node_id: str, @@ -1350,6 +4513,24 @@ def execute_command( def select_handler(node_label: str) -> str: normalized = node_label.lower() + if any(token in normalized for token in ("mongodb", "mongo 写入", "mongo write", "写入 mongo")): + return "mongodb_write" + if any(token in normalized for token in ("x 平台数据集索引", "x-platform", "数据集索引", "dataset index")): + return "xplatform_index" + if any(token in normalized for token in ("采集目录校验", "目录校验", "validate collection")): + return "collection_validate" + if any(token in normalized for token in ("视频质检", "quality check", "quality")): + return "video_quality" + if any(token in normalized for token in ("数据过滤", "dataset filter", "video filter")): + return "dataset_filter" + if any(token in normalized for token in ("视频切片", "video slicing", "slice")): + return "video_slicing" + if any(token in normalized for token in ("视频标注", "annotation", "annotate")): + return "video_annotation" + if any(token in normalized for token in ("成品打包", "package output", "package result")): + return "result_package" + if any(token in normalized for token in ("抽检采样", "audit sample", "sampling")): + return "audit_sampling" if "lerobot" in normalized: return "lerobot" if any(token in normalized for token in ("rosbag", "mcap", "ros bag")): @@ -1378,7 +4559,7 @@ def select_handler(node_label: str) -> str: return "distribution" if any(token in normalized for token in ("upload", "export", "上传", "导出")): return "upload" - if any(token in normalized for token in ("source", "scanner", "oss", "scan", "扫描")): + if any(token in normalized for token in ("source", "scanner", "oss", "scan", "扫描", "数据源")): return "source" return "generic" @@ -1392,7 +4573,7 @@ def main() -> int: output_dir.mkdir(parents=True, exist_ok=True) node_data = load_json_env("NODE_DATA_JSON", {}) - inputs = load_json_env("INPUT_ARTIFACTS_JSON", []) + inputs = load_input_artifacts() started_at = now_iso() start_cpu, start_memory = resource_snapshot() @@ -1428,6 +4609,8 @@ def main() -> int: artifacts = builtin_json_filter(run_id, node_id, output_dir, node_data, inputs) elif handler == "json_map": artifacts = builtin_json_map(run_id, node_id, output_dir, node_data, inputs) + elif handler == "mongodb_write": + artifacts = builtin_mongodb_write(run_id, node_id, output_dir, node_data, inputs) elif handler == "text_template": artifacts = builtin_text_template(run_id, node_id, output_dir, node_data, inputs) elif handler == "csv_to_json": @@ -1436,6 +4619,22 @@ def main() -> int: artifacts = builtin_json_to_csv(run_id, node_id, output_dir, node_data, inputs) elif handler == "directory_scan": artifacts = builtin_directory_scan(run_id, node_id, output_dir, node_data, inputs) + elif handler == "xplatform_index": + artifacts = builtin_xplatform_index(run_id, node_id, output_dir, node_data, inputs) + elif handler == "collection_validate": + artifacts = builtin_collection_validate(run_id, node_id, output_dir, node_data, inputs) + elif handler == "video_quality": + artifacts = builtin_video_quality(run_id, node_id, output_dir, node_data, inputs) + elif handler == "dataset_filter": + artifacts = builtin_dataset_filter(run_id, node_id, output_dir, node_data, inputs) + elif handler == "video_slicing": + artifacts = builtin_video_slicing(run_id, node_id, output_dir, node_data, inputs) + elif handler == "video_annotation": + artifacts = builtin_video_annotation(run_id, node_id, output_dir, node_data, inputs) + elif handler == "result_package": + artifacts = builtin_result_package(run_id, node_id, output_dir, node_data, inputs) + elif handler == "audit_sampling": + artifacts = builtin_audit_sampling(run_id, node_id, output_dir, node_data, inputs) elif handler == "condition_router": artifacts = builtin_condition_router(run_id, node_id, output_dir, node_data, inputs) elif handler == "lerobot": @@ -1453,7 +4652,20 @@ def main() -> int: "artifacts": artifacts, "resourceSeries": resource_series(start_cpu, start_memory, started_at), } - print(RESULT_PREFIX + json.dumps(result, ensure_ascii=False), flush=True) + stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip() + if stage_manifest_path: + write_json_file(stage_manifest_path, stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts)) + else: + write_json_file(default_stage_manifest_path(output_dir), stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts)) + + result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip() + if result_output_path: + written_path = write_json_file(result_output_path, result) + print(RESULT_PATH_PREFIX + str(written_path), flush=True) + return 0 + + default_result_path = write_json_file(default_result_output_path(output_dir), result) + print(RESULT_PATH_PREFIX + str(default_result_path), flush=True) return 0 diff --git a/backend/operator-runner/test_main.py b/backend/operator-runner/test_main.py new file mode 100644 index 00000000..829f151f --- /dev/null +++ b/backend/operator-runner/test_main.py @@ -0,0 +1,1795 @@ +import contextlib +import importlib.util +import io +import json +import os +import pathlib +import tempfile +import unittest +from unittest import mock + + +MODULE_PATH = pathlib.Path(__file__).with_name("main.py") +SPEC = importlib.util.spec_from_file_location("operator_runner_main", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class _FakeHTTPResponse: + def __init__(self, payload: bytes): + self._payload = payload + self._offset = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self, *_args, **_kwargs): + if self._offset >= len(self._payload): + return b"" + chunk = self._payload[self._offset:] + self._offset = len(self._payload) + return chunk + + +class _FakeBosClient: + def __init__(self): + self.uploads = [] + self.downloads = [] + + def put_object_from_file(self, bucket, key, file_name, **kwargs): + self.uploads.append( + { + "bucket": bucket, + "key": key, + "file_name": file_name, + "content_type": kwargs.get("content_type"), + } + ) + + def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs): + return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}" + + def get_object_to_file(self, bucket, key, file_name, **_kwargs): + self.downloads.append( + { + "bucket": bucket, + "key": key, + "file_name": file_name, + } + ) + pathlib.Path(file_name).write_bytes(f"{bucket}/{key}".encode("utf-8")) + + +class _FakeBosClientBytesURL(_FakeBosClient): + def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs): + return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}".encode("utf-8") + + +class _FakeBosListClient(_FakeBosClient): + def __init__(self, objects, object_payloads): + super().__init__() + self._objects = objects + self._object_payloads = object_payloads + self.downloaded_payload_keys = [] + + def list_all_objects(self, bucket, prefix=None): + return list(self._objects) + + def get_object_as_string(self, bucket, key): + self.downloaded_payload_keys.append((bucket, key)) + return self._object_payloads[key] + + +class SourcePathFromItemTests(unittest.TestCase): + def test_prefers_workspace_bos_cache_for_bos_source_paths(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) / "node-output" + workspace_root = pathlib.Path(temp_dir) / "workspace" + output_dir.mkdir(parents=True) + workspace_root.mkdir(parents=True) + item = { + "source": { + "path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D.MP4", + "previewURL": "https://example.com/should-not-download.mp4", + } + } + fake_client = _FakeBosClient() + with mock.patch.dict( + os.environ, + { + "WORKSPACE_ROOT": str(workspace_root), + "BOS_ENDPOINT": "https://bj.bcebos.com", + "BOS_ACCESS_KEY_ID": "ak", + "BOS_SECRET_ACCESS_KEY": "sk", + }, + clear=False, + ), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")) as build_client, mock.patch.object( + MODULE.request, + "urlopen", + ) as urlopen: + first = MODULE.source_path_from_item(item, "demo.mp4", output_dir) + second = MODULE.source_path_from_item(item, "demo.mp4", output_dir) + + self.assertTrue(first.exists()) + self.assertEqual(first, second) + self.assertIn(str(workspace_root), str(first)) + self.assertEqual(len(fake_client.downloads), 1) + build_client.assert_called() + urlopen.assert_not_called() + + def test_downloads_http_preview_url_to_local_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + item = { + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/demo.mp4", + "previewURL": "https://example.com/demo.mp4?signature=abc", + } + } + with mock.patch.object( + MODULE.request, + "urlopen", + return_value=_FakeHTTPResponse(b"real-video-bytes"), + ) as urlopen: + resolved = MODULE.source_path_from_item(item, "demo.mp4", output_dir) + self.assertTrue(resolved.exists()) + self.assertEqual(resolved.read_bytes(), b"real-video-bytes") + urlopen.assert_called_once() + + def test_prefers_workspace_cache_path_for_preview_url_when_bos_credentials_missing(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) / "node-output" + workspace_root = pathlib.Path(temp_dir) / "workspace" + output_dir.mkdir(parents=True) + workspace_root.mkdir(parents=True) + item = { + "source": { + "path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0000/DJI_001/demo.mp4", + "previewURL": "https://example.com/demo.mp4?signature=abc", + } + } + with mock.patch.dict( + os.environ, + { + "WORKSPACE_ROOT": str(workspace_root), + "BOS_ENDPOINT": "", + "BOS_ACCESS_KEY_ID": "", + "BOS_SECRET_ACCESS_KEY": "", + }, + clear=False, + ), mock.patch.object( + MODULE.request, + "urlopen", + return_value=_FakeHTTPResponse(b"real-video-bytes"), + ) as urlopen: + first = MODULE.source_path_from_item(item, "demo.mp4", output_dir) + second = MODULE.source_path_from_item(item, "demo.mp4", output_dir) + self.assertTrue(first.exists()) + self.assertEqual(first, second) + self.assertEqual(first.read_bytes(), b"real-video-bytes") + self.assertIn(str(workspace_root / "_bos_cache"), str(first)) + urlopen.assert_called_once() + + def test_remote_video_url_for_source_signs_bos_path_when_preview_missing(self): + source = { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_0001_D.MP4", + "previewURL": "", + } + with mock.patch.object(MODULE, "signed_bos_source_url", return_value="https://signed.example.com/video.mp4") as signer: + resolved = MODULE.remote_video_url_for_source(source) + self.assertEqual(resolved, "https://signed.example.com/video.mp4") + signer.assert_called_once_with(source["path"]) + + +class RuntimeFileContractTests(unittest.TestCase): + def test_load_input_artifacts_prefers_manifest_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + manifest_path = pathlib.Path(temp_dir) / "input-artifacts.json" + manifest_path.write_text('[{"id":"artifact-from-file"}]', encoding="utf-8") + + with mock.patch.dict( + os.environ, + { + "INPUT_ARTIFACTS_PATH": str(manifest_path), + "INPUT_ARTIFACTS_JSON": '[{"id":"artifact-inline"}]', + }, + clear=False, + ): + loaded = MODULE.load_input_artifacts() + + self.assertEqual(len(loaded), 1) + self.assertEqual(loaded[0]["id"], "artifact-from-file") + + def test_main_writes_result_and_stage_manifest_files(self): + with tempfile.TemporaryDirectory() as temp_dir: + node_root = pathlib.Path(temp_dir) + output_dir = node_root / "outputs" + output_dir.mkdir(parents=True) + input_artifacts_path = node_root / "input-artifacts.json" + input_artifacts_path.write_text("[]", encoding="utf-8") + result_path = node_root / "result.json" + stage_manifest_path = node_root / "stage-manifest.json" + artifacts = [ + { + "id": "artifact-1", + "runId": "run-1", + "nodeId": "node-1", + "name": "report.json", + "type": "json", + "size": "12 B", + "path": str(output_dir / "report.json"), + } + ] + + with mock.patch.dict( + os.environ, + { + "RUN_ID": "run-1", + "WORKFLOW_ID": "wf-1", + "NODE_ID": "node-1", + "NODE_LABEL": "通用处理", + "OUTPUT_DIR": str(output_dir), + "NODE_DATA_JSON": '{"label":"通用处理","hyperparams":[]}', + "INPUT_ARTIFACTS_PATH": str(input_artifacts_path), + "RESULT_OUTPUT_PATH": str(result_path), + "STAGE_MANIFEST_PATH": str(stage_manifest_path), + }, + clear=False, + ), mock.patch.object(MODULE, "builtin_generic", return_value=artifacts), mock.patch.object( + MODULE, + "resource_series", + return_value=[], + ): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = MODULE.main() + + self.assertEqual(exit_code, 0) + self.assertIn(MODULE.RESULT_PATH_PREFIX + str(result_path), stdout.getvalue()) + result_payload = json.loads(result_path.read_text(encoding="utf-8")) + self.assertEqual(result_payload["artifacts"][0]["id"], "artifact-1") + stage_manifest = json.loads(stage_manifest_path.read_text(encoding="utf-8")) + self.assertEqual(stage_manifest["artifactCount"], 1) + self.assertEqual(stage_manifest["nodeLabel"], "通用处理") + + def test_write_json_artifact_externalizes_dataset_payload_to_data_ref(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + artifact = MODULE.write_json_artifact( + "run-1", + "node-1", + output_dir, + "dataset-index.json", + { + "datasetType": "x-platform-index", + "collectionCount": 1, + "collections": [{"id": "collection-1"}], + "items": [{"id": "item-1"}, {"id": "item-2"}], + }, + ) + + data_ref = artifact["data"]["dataRef"] + self.assertEqual(data_ref["kind"], "dataset") + self.assertTrue(pathlib.Path(data_ref["manifestPath"]).exists()) + self.assertTrue(pathlib.Path(data_ref["payloadPath"]).exists()) + manifest = json.loads(pathlib.Path(data_ref["manifestPath"]).read_text(encoding="utf-8")) + self.assertEqual(manifest["datasetType"], "x-platform-index") + self.assertIn("items", manifest["partitionFiles"]) + self.assertIn("collections", manifest["partitionFiles"]) + + def test_load_artifact_payload_reads_externalized_payload_from_data_ref(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + artifact = MODULE.write_json_artifact( + "run-1", + "node-1", + output_dir, + "filtered-videos.json", + { + "datasetType": "x-platform-filtered", + "count": 2, + "items": [{"id": "v1"}, {"id": "v2"}], + }, + ) + + loaded = MODULE.load_artifact_payload(artifact) + + self.assertEqual(loaded["datasetType"], "x-platform-filtered") + self.assertEqual(len(loaded["items"]), 2) + + def test_load_payload_from_file_retries_transient_json_decode_error(self): + with tempfile.TemporaryDirectory() as temp_dir: + payload_path = pathlib.Path(temp_dir) / "payload.json" + payload_path.write_text('{"items":[{"id":"v1"}]}', encoding="utf-8") + decode_error = json.JSONDecodeError("transient", "", 0) + + with mock.patch.object( + MODULE.json, + "load", + side_effect=[decode_error, {"datasetType": "x-platform", "items": [{"id": "v1"}]}], + ) as json_load, mock.patch.object(MODULE.time, "sleep") as sleep_mock: + loaded = MODULE.load_payload_from_file(payload_path) + + self.assertEqual(loaded["datasetType"], "x-platform") + self.assertEqual(len(loaded["items"]), 1) + self.assertEqual(json_load.call_count, 2) + sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS) + + def test_load_payload_from_file_retries_until_payload_path_exists(self): + with tempfile.TemporaryDirectory() as temp_dir: + payload_path = pathlib.Path(temp_dir) / "payload.json" + payload_path.write_text('{"datasetType":"x-platform","items":[{"id":"v1"}]}', encoding="utf-8") + original_exists = pathlib.Path.exists + seen = {"count": 0} + + def fake_exists(path_obj: pathlib.Path): + if path_obj == payload_path and seen["count"] == 0: + seen["count"] += 1 + return False + return original_exists(path_obj) + + with mock.patch.object(pathlib.Path, "exists", autospec=True, side_effect=fake_exists), mock.patch.object( + MODULE.time, + "sleep", + ) as sleep_mock: + loaded = MODULE.load_payload_from_file(payload_path) + + self.assertEqual(loaded["datasetType"], "x-platform") + self.assertEqual(len(loaded["items"]), 1) + sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS) + + def test_stage_manifest_includes_data_refs(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + artifact = MODULE.write_json_artifact( + "run-1", + "node-1", + output_dir, + "packages-manifest.json", + { + "datasetType": "x-platform-packages", + "packageCount": 1, + "items": [{"packageName": "pkg-1"}], + }, + ) + + manifest = MODULE.stage_manifest_payload("run-1", "wf-1", "node-1", "成品打包", output_dir, [artifact]) + + self.assertEqual(len(manifest["dataRefs"]), 1) + self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages") + + +class XPlatformIndexingTests(unittest.TestCase): + def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) / "outputs" + output_dir.mkdir(parents=True) + manifest_path = pathlib.Path(temp_dir) / "source-manifest.json" + manifest_path.write_text("{}", encoding="utf-8") + + payload = MODULE.load_xplatform_index_payload( + {}, + [ + { + "path": str(manifest_path), + "data": { + "sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration", + "entries": [ + { + "path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json", + "sizeBytes": 10, + "mimeType": "application/json", + "data": { + "type": "工厂", + "place": "绿控传动", + "date": "20260330", + }, + }, + { + "path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json", + "sizeBytes": 10, + "mimeType": "application/json", + "data": [ + { + "filename": "DJI_20260327132547_0003_D.MP4", + "codec": "hevc", + "resolution": "3840x2880", + "frameRate": 29.97, + "duration": 0.6, + } + ], + }, + { + "path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4", + "sizeBytes": 123, + "mimeType": "video/mp4", + "source": { + "path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4", + }, + }, + ], + }, + } + ], + output_dir, + ) + + self.assertEqual(payload["collectionCount"], 1) + self.assertEqual(payload["videoCount"], 1) + self.assertEqual(payload["collections"][0]["meta"]["place"], "绿控传动") + + def test_load_index_payload_raises_when_source_manifest_summary_has_no_entries(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) / "outputs" + output_dir.mkdir(parents=True) + manifest_path = pathlib.Path(temp_dir) / "source-manifest.json" + manifest_path.write_text("{}", encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "source-manifest"): + MODULE.load_xplatform_index_payload( + {}, + [ + { + "name": "source-manifest.json", + "path": str(manifest_path), + "data": { + "sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration", + "entriesCount": 59, + "dataRef": {"payloadPath": str(manifest_path)}, + }, + } + ], + output_dir, + ) + + def test_manifest_entries_support_video_metadata_alias(self): + payload = MODULE.scan_xplatform_manifest_entries( + [ + { + "path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json", + "sizeBytes": 10, + "mimeType": "application/json", + "data": { + "type": "工厂", + "place": "绿控传动", + "date": "20260330", + }, + }, + { + "path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json", + "sizeBytes": 10, + "mimeType": "application/json", + "data": [ + { + "filename": "DJI_20260327132547_0003_D.MP4", + "codec": "hevc", + "resolution": "3840x2880", + "frameRate": 29.97, + "duration": 0.6, + } + ], + }, + { + "path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4", + "sizeBytes": 123, + "mimeType": "video/mp4", + "source": { + "path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4", + }, + }, + ], + "bos://wjzh-data-migration/wjzh-0330-Data-migration", + ) + + self.assertEqual(payload["collectionCount"], 1) + collection = payload["collections"][0] + self.assertTrue(collection["hasVideoMeta"]) + self.assertEqual(collection["missingRequiredFiles"], []) + self.assertEqual(payload["items"][0]["codec"], "hevc") + + def test_manifest_entries_load_local_meta_when_inline_data_is_missing(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) / "algo" / "X-platform-test" + collection = root / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43" + video_dir = collection / "DJI_001" + video_dir.mkdir(parents=True) + meta_path = collection / "meta.json" + video_meta_path = collection / "video_meta.json" + video_path = video_dir / "DJI_0001_D.mp4" + meta_path.write_text('{"type":"家庭","place":"蓟门里小区","date":"20260319"}', encoding="utf-8") + video_meta_path.write_text( + '[{"filename":"DJI_0001_D.mp4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]', + encoding="utf-8", + ) + video_path.write_bytes(b"fake-video") + + payload = MODULE.scan_xplatform_manifest_entries( + [ + { + "path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json", + "sizeBytes": meta_path.stat().st_size, + "mimeType": "application/json", + "source": {"path": str(meta_path)}, + }, + { + "path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json", + "sizeBytes": video_meta_path.stat().st_size, + "mimeType": "application/json", + "source": {"path": str(video_meta_path)}, + }, + { + "path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4", + "sizeBytes": video_path.stat().st_size, + "mimeType": "video/mp4", + "source": {"path": str(video_path)}, + }, + ], + str(root), + ) + + self.assertEqual(payload["collectionCount"], 1) + self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区") + self.assertTrue(payload["collections"][0]["hasVideoMeta"]) + self.assertEqual(payload["items"][0]["codec"], "hevc") + + def test_manifest_entries_support_root_level_meta_and_video_meta(self): + payload = MODULE.scan_xplatform_manifest_entries( + [ + { + "path": "meta.json", + "sizeBytes": 10, + "mimeType": "application/json", + "data": { + "type": "家庭", + "place": "蓟门里小区", + "date": "20260319", + }, + }, + { + "path": "video_meta.json", + "sizeBytes": 10, + "mimeType": "application/json", + "data": [ + { + "filename": "DJI_20260318112834_0001_D.MP4", + "codec": "hevc", + "resolution": "3840x2160", + "frameRate": 29.97, + "duration": 41.8, + } + ], + }, + { + "path": "DJI_001/DJI_20260318112834_0001_D.MP4", + "sizeBytes": 123, + "mimeType": "video/mp4", + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260318112834_0001_D.MP4", + "previewURL": "https://example.com/video.mp4", + }, + }, + ], + "bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43", + ) + + self.assertEqual(payload["collectionCount"], 1) + collection = payload["collections"][0] + self.assertEqual(collection["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43") + self.assertEqual(collection["relativePath"], "") + self.assertEqual(collection["videoCount"], 1) + self.assertTrue(collection["hasVideoMeta"]) + self.assertEqual(collection["missingRequiredFiles"], []) + self.assertEqual(collection["meta"]["place"], "蓟门里小区") + + item = payload["items"][0] + self.assertEqual(item["collectionName"], collection["name"]) + self.assertEqual(item["collectionRelativePath"], "") + self.assertEqual(item["videoFolder"], "DJI_001") + self.assertEqual(item["codec"], "hevc") + self.assertEqual(item["meta"]["type"], "家庭") + + def test_local_dataset_root_with_meta_is_treated_as_collection_root(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43" + video_dir = root / "DJI_001" + video_dir.mkdir(parents=True) + (root / "meta.json").write_text('{"type":"家庭","place":"蓟门里小区"}', encoding="utf-8") + (root / "video_meta.json").write_text( + '[{"filename":"DJI_20260318112834_0001_D.MP4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]', + encoding="utf-8", + ) + (video_dir / "DJI_20260318112834_0001_D.MP4").write_bytes(b"fake-video") + + payload = MODULE.scan_xplatform_dataset(root) + + self.assertEqual(payload["collectionCount"], 1) + collection = payload["collections"][0] + self.assertEqual(collection["name"], root.name) + self.assertEqual(collection["relativePath"], "") + self.assertEqual(collection["missingRequiredFiles"], []) + item = payload["items"][0] + self.assertEqual(item["collectionName"], root.name) + self.assertEqual(item["collectionRelativePath"], "") + self.assertEqual(item["codec"], "hevc") + + def test_local_dataset_root_supports_video_metadata_alias(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) / "SZ_000_0005_OsmoNano_2026-03-30_16-15-11" + video_dir = root / "DJI_001" + video_dir.mkdir(parents=True) + (root / "meta.json").write_text('{"type":"工厂","place":"绿控传动"}', encoding="utf-8") + (root / "video_metadata.json").write_text( + '[{"filename":"DJI_20260327132547_0003_D.MP4","codec":"hevc","resolution":"3840x2880","frameRate":29.97,"duration":0.6}]', + encoding="utf-8", + ) + (video_dir / "DJI_20260327132547_0003_D.MP4").write_bytes(b"fake-video") + + payload = MODULE.scan_xplatform_dataset(root) + + self.assertEqual(payload["collectionCount"], 1) + collection = payload["collections"][0] + self.assertEqual(collection["missingRequiredFiles"], []) + self.assertTrue(collection["hasVideoMeta"]) + self.assertEqual(payload["items"][0]["codec"], "hevc") + + +class SourceOperatorTests(unittest.TestCase): + def test_builtin_source_defaults_to_inline_when_source_json_is_present(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + node_data = { + "label": "真实数据源-默认内联", + "hyperparams": [ + {"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)}, + ], + } + with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")): + artifacts = MODULE.builtin_source("run-test", "source-inline-default", output_dir, node_data, []) + manifest = next(item for item in artifacts if item["name"] == "source-manifest.json") + payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8")) + self.assertEqual(payload["dataSource"], "INLINE") + self.assertEqual(payload["listedCount"], 1) + self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic") + + def test_builtin_source_prefers_source_json_over_bos_env(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + node_data = { + "label": "真实数据源-JSON", + "hyperparams": [ + {"key": "DATA_SOURCE", "value": "BOS"}, + {"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)}, + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "ak"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"}, + ], + } + with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")): + artifacts = MODULE.builtin_source("run-test", "source-json", output_dir, node_data, []) + manifest = next(item for item in artifacts if item["name"] == "source-manifest.json") + payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8")) + self.assertEqual(payload["listedCount"], 1) + self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic") + + def test_builtin_source_prefers_source_text_over_bos_env(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + node_data = { + "label": "真实数据源-文本", + "hyperparams": [ + {"key": "DATA_SOURCE", "value": "BOS"}, + {"key": "SOURCE_TEXT", "value": "alpha\nbeta\ngamma"}, + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "ak"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"}, + ], + } + with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")): + artifacts = MODULE.builtin_source("run-test", "source-text", output_dir, node_data, []) + manifest = next(item for item in artifacts if item["name"] == "source-manifest.json") + payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8")) + self.assertEqual(payload["listedCount"], 1) + self.assertEqual(payload["entries"][0]["text"], "alpha\nbeta\ngamma") + + def test_directory_incremental_source_filter_emits_full_new_directory(self): + entries = [ + {"path": "batch-a/meta.json", "sizeBytes": 10, "sha256": "sha-a-meta"}, + {"path": "batch-a/DJI_001/clip-a.mp4", "sizeBytes": 20, "sha256": "sha-a-clip"}, + {"path": "batch-b/meta.json", "sizeBytes": 30, "sha256": "sha-b-meta"}, + {"path": "batch-b/DJI_001/clip-b.mp4", "sizeBytes": 40, "sha256": "sha-b-clip"}, + ] + emitted, checkpoint = MODULE.apply_incremental_source_filter( + entries, + "BOS", + "wjzh-data-migration", + { + "SOURCE_STATE_JSON": json.dumps( + { + "seenFingerprints": ["batch-a"], + "incrementalUnit": "DIRECTORY", + "directoryDepth": 1, + } + ), + "SOURCE_FINGERPRINT": "fp-directory", + "INCREMENTAL_UNIT": "DIRECTORY", + "INCREMENTAL_DIRECTORY_DEPTH": "1", + }, + "bos://wjzh-data-migration/wjzh-0330-Data-migration", + "wjzh-0330-Data-migration", + "", + ) + + self.assertEqual([item["path"] for item in emitted], ["batch-b/meta.json", "batch-b/DJI_001/clip-b.mp4"]) + self.assertEqual([item["incrementalKey"] for item in emitted], ["batch-b", "batch-b"]) + self.assertEqual(checkpoint["incrementalUnit"], "DIRECTORY") + self.assertEqual(checkpoint["directoryDepth"], 1) + self.assertEqual(checkpoint["newCount"], 2) + self.assertEqual(sorted(checkpoint["seenFingerprints"]), ["batch-a", "batch-b"]) + + def test_list_bos_entries_skips_large_non_metadata_json_payloads(self): + object_meta = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/meta.json", "size": 128, "e_tag": "meta"})() + object_imu = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json", "size": MODULE.PREVIEW_INLINE_BINARY_LIMIT + 1, "e_tag": "imu"})() + fake_client = _FakeBosListClient( + [object_meta, object_imu], + { + "wjzh-0330-Data-migration/SZ_000_0005/meta.json": b'{"type":"\xe5\xb7\xa5\xe5\x8e\x82"}', + "wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json": b'{"data_points":[1,2,3]}', + }, + ) + with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")): + entries = MODULE.list_bos_entries( + "wjzh-data-migration", + "wjzh-0330-Data-migration", + "", + "https://bj.bcebos.com", + "ak", + "sk", + ) + + self.assertEqual(len(entries), 2) + meta_entry = next(item for item in entries if item["path"].endswith("meta.json")) + imu_entry = next(item for item in entries if item["path"].endswith("_imu.json")) + self.assertIn("data", meta_entry) + self.assertNotIn("data", imu_entry) + self.assertEqual( + fake_client.downloaded_payload_keys, + [("wjzh-data-migration", "wjzh-0330-Data-migration/SZ_000_0005/meta.json")], + ) + + def test_bos_source_parses_bucket_and_prefix_from_source_path(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + node_data = { + "label": "BOS 增量扫描", + "hyperparams": [ + {"key": "DATA_SOURCE", "value": "BOS"}, + {"key": "SOURCE_PATH", "value": "bos://wjzh-data-migration/wjzh-0330-Data-migration"}, + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "ak"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"}, + {"key": "SCAN_MODE", "value": "FULL"}, + ], + } + fake_entries = [ + { + "path": "SZ_000_0005/meta.json", + "storagePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json", + "sizeBytes": 10, + "mimeType": "application/json", + "data": {"type": "工厂"}, + "source": {"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json"}, + } + ] + with mock.patch.object(MODULE, "list_bos_entries", return_value=fake_entries) as list_bos_entries, mock.patch.object( + MODULE, + "source_entry_artifact", + return_value={ + "id": "artifact-1", + "runId": "run-1", + "nodeId": "source", + "name": "meta.json", + "type": "json", + "path": str(output_dir / "meta.json"), + }, + ): + artifacts = MODULE.builtin_source("run-1", "source", output_dir, node_data, []) + + self.assertGreaterEqual(len(artifacts), 1) + list_bos_entries.assert_called_once_with( + "wjzh-data-migration", + "wjzh-0330-Data-migration", + "", + "https://bj.bcebos.com", + "ak", + "sk", + ) + + +class QualityMetricHelperTests(unittest.TestCase): + def test_build_skin_mask_detects_skin_tone_patch(self): + _, np = MODULE.require_cv2() + frame = np.zeros((80, 80, 3), dtype=np.uint8) + frame[20:60, 20:60] = (90, 130, 190) + mask = MODULE.build_skin_mask(frame) + self.assertGreater(int(mask.sum()), 0) + + def test_build_skin_mask_rejects_neutral_background(self): + _, np = MODULE.require_cv2() + frame = np.full((80, 80, 3), 128, dtype=np.uint8) + mask = MODULE.build_skin_mask(frame) + self.assertEqual(int(mask.sum()), 0) + + def test_motion_jitter_ratio_stays_low_for_smooth_motion(self): + smooth = [(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)] + jitter = MODULE.motion_jitter_ratio(smooth, 100.0) + self.assertLess(jitter, 0.01) + + def test_motion_jitter_ratio_increases_for_erratic_motion(self): + smooth = MODULE.motion_jitter_ratio([(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)], 100.0) + erratic = MODULE.motion_jitter_ratio([(0.0, 0.0), (12.0, -4.0), (-10.0, 8.0), (15.0, -6.0)], 100.0) + self.assertGreater(erratic, smooth) + self.assertGreater(erratic, 0.05) + + def test_largest_skin_component_ratio_ignores_small_noise(self): + _, np = MODULE.require_cv2() + mask = np.zeros((100, 100), dtype=np.uint8) + mask[10:13, 10:13] = 255 + ratio = MODULE.largest_skin_component_ratio(mask) + self.assertEqual(ratio, 0.0) + + def test_largest_skin_component_ratio_returns_large_component_area(self): + _, np = MODULE.require_cv2() + mask = np.zeros((100, 100), dtype=np.uint8) + mask[20:50, 30:70] = 255 + ratio = MODULE.largest_skin_component_ratio(mask) + self.assertGreater(ratio, 0.1) + self.assertLess(ratio, 0.13) + + +class QualityOperatorTests(unittest.TestCase): + def test_video_quality_accepts_zero_hand_occlusion(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + node_data = { + "label": "视频质检", + "hyperparams": [ + {"key": "MIN_CLARITY", "value": "0.8"}, + {"key": "MAX_SHAKE", "value": "0.1"}, + {"key": "MAX_HAND_OCCLUSION", "value": "0.1"}, + ], + } + inputs = [ + { + "data": { + "items": [ + { + "filename": "demo.mp4", + "source": {"path": "/tmp/demo.mp4"}, + } + ] + } + } + ] + + with mock.patch.object(MODULE, "source_path_from_item", return_value=pathlib.Path("/tmp/demo.mp4")), mock.patch.object( + MODULE, + "analyze_video_quality_metrics", + return_value={ + "clarity": 1.0, + "shake": 0.0, + "handOcclusion": 0.0, + "qualityScore": 1.0, + }, + ): + artifacts = MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, inputs) + + payload = MODULE.load_artifact_payload(artifacts[0]) + self.assertEqual(payload["passedCount"], 1) + self.assertTrue(payload["items"][0]["quality"]["passed"]) + + def test_dataset_filter_keeps_zero_shake_and_hand_occlusion(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + node_data = { + "label": "数据过滤", + "hyperparams": [ + {"key": "DATE_START", "value": "2026-03-01"}, + {"key": "DATE_END", "value": "2026-03-31"}, + {"key": "PLACE", "value": "蓟门里小区"}, + {"key": "TYPE", "value": "家庭"}, + {"key": "MIN_CLARITY", "value": "0.8"}, + {"key": "MAX_SHAKE", "value": "0.1"}, + {"key": "MAX_HAND_OCCLUSION", "value": "0.1"}, + ], + } + inputs = [ + { + "data": { + "items": [ + { + "meta": { + "date": "20260319", + "place": "蓟门里小区", + "type": "家庭", + }, + "quality": { + "clarity": 0.95, + "shake": 0.0, + "handOcclusion": 0.0, + "passed": True, + }, + } + ] + } + } + ] + + artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs) + + payload = MODULE.load_artifact_payload(artifacts[0]) + self.assertEqual(payload["count"], 1) + self.assertEqual(len(payload["items"]), 1) + + +class UploadOperatorTests(unittest.TestCase): + def test_build_artifact_skips_sha256_for_large_files(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) + clip_path = root / "clip001.mp4" + clip_path.write_bytes(b"video-bytes") + + with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object( + MODULE, + "sha256_file", + side_effect=AssertionError("sha256_file should not be called for files above size limit"), + ): + artifact = MODULE.build_artifact("run-1", "slice", clip_path, "video") + + metadata = artifact.get("metadata") or {} + self.assertEqual(metadata.get("sha256", ""), "") + self.assertTrue(metadata.get("sha256Skipped")) + self.assertEqual(metadata.get("sha256SkipReason"), "size-limit") + + def test_uploads_nested_package_outputs_to_bos(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) + package_root = root / "package" + package_dir = package_root / "packages" / "clip001_package" + package_dir.mkdir(parents=True) + + manifest_path = package_root / "packages-manifest.json" + manifest_path.write_text('{"packageCount":1}', encoding="utf-8") + clip_path = package_dir / "clip001.mp4" + clip_path.write_bytes(b"video-bytes") + annotation_path = package_dir / "clip001_annotation.json" + annotation_path.write_text('{"label":"walking"}', encoding="utf-8") + + output_dir = root / "upload" + node_data = { + "label": "结果上传", + "bucket": "datastorage", + "prefix": "factory/output/demo", + "hyperparams": [ + {"key": "DATA_SOURCE", "value": "BOS"}, + {"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"}, + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"}, + ], + } + inputs = [ + MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}), + MODULE.build_artifact("run-1", "package", clip_path, "video"), + MODULE.build_artifact("run-1", "package", annotation_path, "json", data={"label": "walking"}), + ] + fake_client = _FakeBosClient() + + with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")): + artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs) + + uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json") + uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary) + uploaded_names = {item["name"] for item in artifacts} + uploaded_keys = {item["key"] for item in fake_client.uploads} + + self.assertEqual(len(fake_client.uploads), 3) + self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys) + self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys) + self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys) + self.assertEqual(uploaded_summary_payload["uploadedCount"], 3) + self.assertEqual(uploaded_summary_payload["bucket"], "datastorage") + self.assertEqual(uploaded_summary_payload["prefix"], "factory/output/demo") + + def test_upload_skips_sha256_for_large_remote_artifacts(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) + clip_path = root / "clip001.mp4" + clip_path.write_bytes(b"video-bytes") + + output_dir = root / "upload" + node_data = { + "label": "结果上传", + "bucket": "datastorage", + "prefix": "factory/output/demo", + "hyperparams": [ + {"key": "DATA_SOURCE", "value": "BOS"}, + {"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"}, + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"}, + ], + } + inputs = [MODULE.build_artifact("run-1", "slice", clip_path, "video")] + fake_client = _FakeBosClient() + + with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object( + MODULE, + "sha256_file", + side_effect=AssertionError("sha256_file should not be called for files above size limit"), + ), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")): + artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs) + + remote_video = next(item for item in artifacts if item.get("type") == "video" and str(item.get("path", "")).startswith("bos://")) + metadata = remote_video.get("metadata") or {} + self.assertEqual(metadata.get("sha256", ""), "") + self.assertTrue(metadata.get("sha256Skipped")) + self.assertEqual(metadata.get("sha256SkipReason"), "size-limit") + + +class _FakeInsertManyResult: + def __init__(self, inserted_ids): + self.inserted_ids = inserted_ids + + +class _FakeUpdateResult: + def __init__(self, matched_count=0, modified_count=0, upserted_id=None): + self.matched_count = matched_count + self.modified_count = modified_count + self.upserted_id = upserted_id + + +class _FakeMongoCollection: + def __init__(self): + self.insert_many_calls = [] + self.replace_one_calls = [] + + def insert_many(self, documents, ordered=False): + self.insert_many_calls.append({"documents": documents, "ordered": ordered}) + return _FakeInsertManyResult([f"id-{index + 1}" for index in range(len(documents))]) + + def replace_one(self, filter_document, replacement, upsert=False): + self.replace_one_calls.append( + { + "filter": filter_document, + "replacement": replacement, + "upsert": upsert, + } + ) + return _FakeUpdateResult(matched_count=1, modified_count=1, upserted_id=None) + + +class _FakeMongoDatabase: + def __init__(self, collection): + self._collection = collection + self.requested_collections = [] + + def __getitem__(self, name): + self.requested_collections.append(name) + return self._collection + + +class _FakeMongoClient: + def __init__(self, database): + self._database = database + self.requested_databases = [] + self.closed = False + + def __getitem__(self, name): + self.requested_databases.append(name) + return self._database + + def close(self): + self.closed = True + + +class MongoWriteOperatorTests(unittest.TestCase): + def test_inserts_documents_into_default_business_database(self): + collection = _FakeMongoCollection() + database = _FakeMongoDatabase(collection) + client = _FakeMongoClient(database) + node_data = { + "label": "MongoDB 写入", + "outputs": [{"id": "result", "name": "result", "type": "json"}], + "hyperparams": [ + {"key": "MONGO_COLLECTION", "value": "qualityReports"}, + {"key": "MONGO_MODE", "value": "insert_many"}, + ], + } + inputs = [ + { + "data": { + "items": [ + {"clipId": "clip-001", "qualityScore": 0.91}, + {"clipId": "clip-002", "qualityScore": 0.87}, + ] + } + } + ] + + with tempfile.TemporaryDirectory() as temp_dir: + with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False): + with mock.patch.object(MODULE, "build_mongo_client", return_value=client): + artifacts = MODULE.builtin_mongodb_write("run-1", "mongo-node", pathlib.Path(temp_dir), node_data, inputs) + + self.assertEqual(database.requested_collections, ["qualityReports"]) + self.assertEqual(client.requested_databases, ["business-data"]) + self.assertEqual(len(collection.insert_many_calls), 1) + self.assertEqual(collection.insert_many_calls[0]["documents"][0]["clipId"], "clip-001") + summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json") + self.assertEqual(summary["data"]["collection"], "qualityReports") + self.assertEqual(summary["data"]["mode"], "insert_many") + self.assertEqual(summary["data"]["target"], "business") + self.assertEqual(summary["data"]["insertedCount"], 2) + self.assertTrue(client.closed) + + def test_upserts_documents_by_key(self): + collection = _FakeMongoCollection() + database = _FakeMongoDatabase(collection) + client = _FakeMongoClient(database) + node_data = { + "label": "MongoDB 写入", + "outputs": [{"id": "result", "name": "result", "type": "json"}], + "hyperparams": [ + {"key": "MONGO_COLLECTION", "value": "annotations"}, + {"key": "MONGO_MODE", "value": "upsert"}, + {"key": "UPSERT_KEY", "value": "clipId"}, + ], + } + inputs = [ + {"data": {"clipId": "clip-001", "label": "walking"}}, + {"data": {"clipId": "clip-002", "label": "standing"}}, + ] + + with tempfile.TemporaryDirectory() as temp_dir: + with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False): + with mock.patch.object(MODULE, "build_mongo_client", return_value=client): + artifacts = MODULE.builtin_mongodb_write("run-2", "mongo-node", pathlib.Path(temp_dir), node_data, inputs) + + self.assertEqual(len(collection.replace_one_calls), 2) + self.assertEqual(collection.replace_one_calls[0]["filter"], {"clipId": "clip-001"}) + self.assertTrue(collection.replace_one_calls[0]["upsert"]) + summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json") + self.assertEqual(summary["data"]["mode"], "upsert") + self.assertEqual(summary["data"]["upsertKey"], "clipId") + self.assertEqual(summary["data"]["target"], "business") + self.assertEqual(summary["data"]["upsertedCount"], 2) + + def test_unwraps_single_value_dict_documents_for_business_writes(self): + collection = _FakeMongoCollection() + database = _FakeMongoDatabase(collection) + client = _FakeMongoClient(database) + node_data = { + "label": "MongoDB 写入", + "outputs": [{"id": "result", "name": "result", "type": "json"}], + "hyperparams": [ + {"key": "MONGO_COLLECTION", "value": "businessWriteValidation"}, + {"key": "MONGO_MODE", "value": "insert_many"}, + ], + } + inputs = [ + { + "data": { + "items": [ + { + "value": { + "runLabel": "business-mongo-test", + "source": "workflow", + "createdAt": "2026-04-14T00:00:00Z", + "value": 1, + } + } + ] + } + } + ] + + with tempfile.TemporaryDirectory() as temp_dir: + with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False): + with mock.patch.object(MODULE, "build_mongo_client", return_value=client): + MODULE.builtin_mongodb_write("run-unwrap", "mongo-node", pathlib.Path(temp_dir), node_data, inputs) + + self.assertEqual(len(collection.insert_many_calls), 1) + inserted = collection.insert_many_calls[0]["documents"][0] + self.assertEqual(inserted["runLabel"], "business-mongo-test") + self.assertEqual(inserted["source"], "workflow") + self.assertEqual(inserted["value"], 1) + self.assertNotEqual(set(inserted.keys()), {"value"}) + + def test_can_target_platform_database_explicitly(self): + collection = _FakeMongoCollection() + database = _FakeMongoDatabase(collection) + client = _FakeMongoClient(database) + node_data = { + "label": "MongoDB 写入", + "outputs": [{"id": "result", "name": "result", "type": "json"}], + "hyperparams": [ + {"key": "MONGO_COLLECTION", "value": "qualityReports"}, + {"key": "MONGO_TARGET", "value": "platform"}, + ], + } + + with tempfile.TemporaryDirectory() as temp_dir: + with mock.patch.dict(MODULE.os.environ, {"PLATFORM_MONGO_URI": "mongodb://platform-mongo:27017", "PLATFORM_MONGO_DB": "droboticflow"}, clear=False): + with mock.patch.object(MODULE, "build_mongo_client", return_value=client): + artifacts = MODULE.builtin_mongodb_write("run-3", "mongo-node", pathlib.Path(temp_dir), node_data, [{"data": {"clipId": "clip-001"}}]) + + self.assertEqual(client.requested_databases, ["droboticflow"]) + summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json") + self.assertEqual(summary["data"]["target"], "platform") + + def test_rebuilds_package_files_from_manifest_when_temp_paths_are_missing(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) + source_path = root / "source.mp4" + source_path.write_bytes(b"source-video") + output_dir = root / "upload" + node_data = { + "label": "结果上传", + "bucket": "datastorage", + "prefix": "factory/output/demo", + "hyperparams": [ + {"key": "DATA_SOURCE", "value": "BOS"}, + {"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"}, + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"}, + ], + } + package_manifest = { + "packageCount": 1, + "items": [ + { + "packageName": "clip001_package", + "clipId": "clip001", + "clipName": "clip001.mp4", + "sourceVideo": "DJI_20260318112834_0001_D.MP4", + "collectionId": "collection-1", + "meta": {"type": "家庭"}, + "startSecond": 0, + "endSecond": 41.8, + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4", + "previewURL": "https://example.com/source.mp4", + }, + "quality": {"clarity": 0.9, "passed": True}, + "annotation": {"objects": ["人"], "minConfidence": 0.92}, + } + ], + } + inputs = [ + {"name": "packages-manifest.json", "type": "json", "data": package_manifest}, + { + "name": "clip001.mp4", + "type": "video", + "path": "/tmp/non-existent-clip001.mp4", + "data": {"preview": {"kind": "video", "format": "mp4"}}, + }, + { + "name": "clip001_annotation.json", + "type": "json", + "path": "/tmp/non-existent-clip001_annotation.json", + "data": {"objects": ["人"], "minConfidence": 0.92}, + }, + { + "name": "clip001_quality.json", + "type": "json", + "path": "/tmp/non-existent-clip001_quality.json", + "data": {"clarity": 0.9, "passed": True}, + }, + { + "name": "clip001_config.json", + "type": "json", + "path": "/tmp/non-existent-clip001_config.json", + "data": {"collectionId": "collection-1"}, + }, + ] + fake_client = _FakeBosClient() + + def fake_write_video_segment(_source_path, destination, _start_second, _end_second): + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(b"clip-bytes") + return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"} + + with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")), mock.patch.object( + MODULE, "source_path_from_item", return_value=source_path + ), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment): + artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs) + + uploaded_keys = {item["key"] for item in fake_client.uploads} + self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys) + self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys) + self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys) + self.assertIn("factory/output/demo/packages/clip001_package/clip001_quality.json", uploaded_keys) + self.assertIn("factory/output/demo/packages/clip001_package/clip001_config.json", uploaded_keys) + self.assertFalse( + (output_dir / "uploaded" / "_materialized_upload_inputs" / "packages" / "clip001_package" / "clip001.mp4").exists() + ) + uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json") + uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary) + self.assertEqual(uploaded_summary_payload["uploadedCount"], 5) + + def test_upload_artifacts_are_json_serializable_when_preview_url_is_bytes(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) + package_root = root / "package" + package_dir = package_root / "packages" / "clip001_package" + package_dir.mkdir(parents=True) + + manifest_path = package_root / "packages-manifest.json" + manifest_path.write_text('{"packageCount":1}', encoding="utf-8") + clip_path = package_dir / "clip001.mp4" + clip_path.write_bytes(b"video-bytes") + + output_dir = root / "upload" + node_data = { + "label": "结果上传", + "bucket": "datastorage", + "prefix": "factory/output/demo", + "hyperparams": [ + {"key": "DATA_SOURCE", "value": "BOS"}, + {"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"}, + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"}, + ], + } + inputs = [ + MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}), + MODULE.build_artifact("run-1", "package", clip_path, "video"), + ] + + with mock.patch.object(MODULE, "build_bos_client", return_value=(_FakeBosClientBytesURL(), "https")): + artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs) + + json.dumps({"artifacts": artifacts}, ensure_ascii=False) + + +class XPlatformClipHandoffTests(unittest.TestCase): + def test_video_slicing_manifest_preserves_source_reference(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + source_path = output_dir / "source.mp4" + source_path.write_bytes(b"source-video") + inputs = [ + { + "name": "filtered-videos.json", + "type": "json", + "data": { + "count": 1, + "items": [ + { + "filename": "DJI_20260318112834_0001_D.MP4", + "duration": 41.8, + "collectionId": "collection-1", + "meta": {"type": "家庭"}, + "quality": {"passed": True}, + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4", + "previewURL": "https://example.com/source.mp4", + }, + } + ], + }, + } + ] + node_data = { + "label": "视频切片", + "hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}], + } + + def fake_write_video_segment(_source_path, destination, _start_second, _end_second): + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(b"clip-bytes") + return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"} + + with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object( + MODULE, "source_path_from_item", return_value=source_path + ), mock.patch.object( + MODULE, + "inspect_video_file", + return_value={"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}, + ), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment), mock.patch.object( + MODULE, + "extract_preview_data", + return_value={"preview": {"kind": "video", "format": "mp4"}}, + ): + artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs) + + manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json") + manifest_payload = MODULE.load_artifact_payload(manifest) + clip = manifest_payload["items"][0] + self.assertEqual(clip["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4") + self.assertEqual(clip["clipName"], "DJI_20260318112834_0001_D_clip001.mp4") + + def test_video_slicing_uses_single_pass_writer_for_multi_clip_video(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + source_path = output_dir / "source.mp4" + source_path.write_bytes(b"source-video") + inputs = [ + { + "name": "filtered-videos.json", + "type": "json", + "data": { + "count": 1, + "items": [ + { + "filename": "DJI_20260318112834_0001_D.MP4", + "duration": 365.0, + "collectionId": "collection-1", + "meta": {"type": "家庭"}, + "quality": {"passed": True}, + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4", + "previewURL": "https://example.com/source.mp4", + }, + } + ], + }, + } + ] + node_data = { + "label": "视频切片", + "hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}], + } + + def fake_write_video_segments(_source_path, segments): + outputs = [] + for segment in segments: + destination = pathlib.Path(segment["destination"]) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(f"clip-{segment['clipIndex']}".encode("utf-8")) + outputs.append( + { + **segment, + "duration": round(float(segment["endSecond"]) - float(segment["startSecond"]), 3), + "frameRate": 29.97, + "resolution": "3840x2160", + } + ) + return outputs + + with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object( + MODULE, "source_path_from_item", return_value=source_path + ), mock.patch.object( + MODULE, + "inspect_video_file", + return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"}, + ), mock.patch.object( + MODULE, + "write_video_segments", + side_effect=fake_write_video_segments, + ) as multi_writer, mock.patch.object( + MODULE, + "write_video_segment", + ) as single_writer, mock.patch.object( + MODULE, + "extract_preview_data", + return_value={"preview": {"kind": "video", "format": "mp4"}}, + ): + artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs) + + manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json") + manifest_payload = MODULE.load_artifact_payload(manifest) + self.assertEqual(manifest_payload["clipCount"], 2) + self.assertEqual([item["clipName"] for item in manifest_payload["items"]], [ + "DJI_20260318112834_0001_D_clip001.mp4", + "DJI_20260318112834_0001_D_clip002.mp4", + ]) + multi_writer.assert_called_once() + single_writer.assert_not_called() + + def test_write_video_segment_with_ffmpeg_prefers_stream_copy_and_uses_inspected_metrics(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) + source_path = root / "source.mp4" + source_path.write_bytes(b"source-video") + destination = root / "clips" / "clip001.mp4" + + def fake_run(args, stdout=None, stderr=None, text=None): + self.assertEqual(args[0], "ffmpeg") + self.assertIn(str(source_path), args) + self.assertEqual(args[-1], str(destination)) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(b"clip-video") + return mock.Mock(returncode=0, stdout="", stderr="") + + with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object( + MODULE.subprocess, + "run", + side_effect=fake_run, + ) as run_mock, mock.patch.object( + MODULE, + "inspect_video_file", + return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"}, + ): + metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0) + + self.assertEqual(metrics["duration"], 21.5) + self.assertEqual(metrics["frameRate"], 29.97) + self.assertEqual(metrics["resolution"], "3840x2160") + run_args = run_mock.call_args.args[0] + self.assertIn("-ss", run_args) + self.assertIn("-t", run_args) + self.assertIn("-c", run_args) + self.assertIn("copy", run_args) + self.assertNotIn("-movflags", run_args) + self.assertNotIn("+faststart", run_args) + + def test_write_video_segment_with_ffmpeg_falls_back_to_reencode_when_copy_fails(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = pathlib.Path(temp_dir) + source_path = root / "source.mp4" + source_path.write_bytes(b"source-video") + destination = root / "clips" / "clip001.mp4" + + calls = [] + + def fake_run(args, stdout=None, stderr=None, text=None): + calls.append(list(args)) + if len(calls) == 1: + return mock.Mock(returncode=1, stdout="", stderr="copy failed") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(b"clip-video") + return mock.Mock(returncode=0, stdout="", stderr="") + + with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object( + MODULE.subprocess, + "run", + side_effect=fake_run, + ), mock.patch.object( + MODULE, + "inspect_video_file", + return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"}, + ): + metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0) + + self.assertEqual(metrics["duration"], 21.5) + self.assertEqual(len(calls), 2) + self.assertIn("copy", calls[0]) + self.assertIn("libx264", calls[1]) + self.assertIn("aac", calls[1]) + + def test_video_slicing_prefers_ffmpeg_when_available(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + source_path = output_dir / "source.mp4" + source_path.write_bytes(b"source-video") + inputs = [ + { + "name": "filtered-videos.json", + "type": "json", + "data": { + "count": 1, + "items": [ + { + "filename": "DJI_20260318112834_0001_D.MP4", + "duration": 365.0, + "collectionId": "collection-1", + "meta": {"type": "家庭"}, + "quality": {"passed": True}, + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4", + "previewURL": "https://example.com/source.mp4", + }, + } + ], + }, + } + ] + node_data = { + "label": "视频切片", + "hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}], + } + + def fake_ffmpeg_writer(_source_path, destination, start_second, end_second): + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(f"{start_second}-{end_second}".encode("utf-8")) + return { + "duration": round(float(end_second) - float(start_second), 3), + "frameRate": 29.97, + "resolution": "3840x2160", + } + + with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object( + MODULE, "source_path_from_item", return_value=source_path + ), mock.patch.object( + MODULE, + "inspect_video_file", + return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"}, + ), mock.patch.object( + MODULE, + "write_video_segment_with_ffmpeg", + side_effect=fake_ffmpeg_writer, + ) as ffmpeg_writer, mock.patch.object( + MODULE, + "write_video_segments", + ) as multi_writer, mock.patch.object( + MODULE, + "write_video_segment", + ) as single_writer, mock.patch.object( + MODULE, + "extract_preview_data", + return_value={"preview": {"kind": "video", "format": "mp4"}}, + ): + artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs) + + manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json") + manifest_payload = MODULE.load_artifact_payload(manifest) + self.assertEqual(manifest_payload["clipCount"], 2) + self.assertEqual(ffmpeg_writer.call_count, 2) + multi_writer.assert_not_called() + single_writer.assert_not_called() + + def test_video_annotation_reuses_shared_clip_path_without_copying_clip_artifact(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + source_path = output_dir / "source.mp4" + source_path.write_bytes(b"source-video") + shared_clip_path = output_dir / "shared" / "clip001.mp4" + shared_clip_path.parent.mkdir(parents=True, exist_ok=True) + shared_clip_path.write_bytes(b"clip-bytes") + inputs = [ + { + "name": "clips-manifest.json", + "type": "json", + "data": { + "clipCount": 1, + "items": [ + { + "clipId": "clip001", + "clipName": "clip001.mp4", + "clipPath": str(shared_clip_path), + "sourceVideo": "DJI_20260318112834_0001_D.MP4", + "startSecond": 0, + "endSecond": 41.8, + "duration": 41.8, + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4", + "previewURL": "https://example.com/source.mp4", + }, + } + ], + }, + } + ] + node_data = { + "label": "视频标注", + "hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "1.0"}], + } + + with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object( + MODULE, + "annotate_video_clip", + return_value={ + "duration": 41.8, + "actions": [{"label": "walking", "confidence": 0.92, "start": 0, "end": 41.8}], + "objects": ["人"], + "minConfidence": 0.92, + }, + ), mock.patch.object( + MODULE, + "extract_preview_data", + return_value={"preview": {"kind": "video", "format": "mp4"}}, + ): + artifacts = MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs) + + manifest = next(item for item in artifacts if item["name"] == "annotations-manifest.json") + manifest_payload = MODULE.load_artifact_payload(manifest) + self.assertEqual(manifest_payload["annotationCount"], 1) + self.assertEqual(manifest_payload["items"][0]["clipPath"], str(shared_clip_path)) + self.assertFalse(any(item["name"] == "clip001.mp4" for item in artifacts)) + + def test_materialize_clip_from_source_prefers_remote_segment_generation(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + item = { + "clipName": "clip001.mp4", + "sourceVideo": "DJI_20260318112834_0001_D.MP4", + "startSecond": 0, + "endSecond": 41.8, + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4", + }, + } + + def fake_remote_segment(_source_url, destination, _start_second, _end_second): + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(b"clip-bytes") + return {"duration": 41.8} + + with mock.patch.object(MODULE, "remote_video_url_for_source", return_value="https://signed.example.com/video.mp4"), mock.patch.object( + MODULE, + "write_video_segment_from_remote_url", + side_effect=fake_remote_segment, + ) as remote_writer, mock.patch.object(MODULE, "source_path_from_item") as local_materializer: + resolved = MODULE.materialize_clip_from_source(item, output_dir) + + self.assertIsNotNone(resolved) + self.assertTrue(resolved.exists()) + self.assertEqual(resolved.read_bytes(), b"clip-bytes") + remote_writer.assert_called_once() + local_materializer.assert_not_called() + + def test_result_package_rebuilds_clip_from_source_when_clip_artifact_is_missing(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + source_path = output_dir / "source.mp4" + source_path.write_bytes(b"source-video") + inputs = [ + { + "name": "annotations-manifest.json", + "type": "json", + "data": { + "annotationCount": 1, + "items": [ + { + "clipId": "clip001", + "clipName": "clip001.mp4", + "sourceVideo": "DJI_20260318112834_0001_D.MP4", + "startSecond": 0, + "endSecond": 41.8, + "duration": 41.8, + "collectionId": "collection-1", + "meta": {"type": "家庭"}, + "quality": {"clarity": 0.9, "passed": True}, + "annotation": {"actions": [], "objects": ["人"], "minConfidence": 0.92}, + "source": { + "path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4", + "previewURL": "https://example.com/source.mp4", + }, + } + ], + }, + } + ] + node_data = { + "label": "成品打包", + "hyperparams": [], + } + + def fake_write_video_segment(_source_path, destination, _start_second, _end_second): + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(b"clip-bytes") + return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"} + + with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object( + MODULE, + "write_video_segment", + side_effect=fake_write_video_segment, + ), mock.patch.object( + MODULE, + "extract_preview_data", + return_value={"preview": {"kind": "video", "format": "mp4"}}, + ): + artifacts = MODULE.builtin_result_package("run-1", "package", output_dir, node_data, inputs) + + manifest = next(item for item in artifacts if item["name"] == "packages-manifest.json") + manifest_payload = MODULE.load_artifact_payload(manifest) + package_clip = next(item for item in artifacts if item["name"] == "clip001.mp4") + self.assertEqual(manifest_payload["packageCount"], 1) + manifest_item = manifest_payload["items"][0] + self.assertEqual(manifest_item["clipName"], "clip001.mp4") + self.assertEqual(manifest_item["collectionId"], "collection-1") + self.assertEqual(manifest_item["startSecond"], 0) + self.assertEqual(manifest_item["endSecond"], 41.8) + self.assertEqual(manifest_item["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4") + self.assertEqual(package_clip["name"], "clip001.mp4") + + +if __name__ == "__main__": + unittest.main() diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 6579d5dc..2a461c9b 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -44,11 +44,14 @@ func main() { }() dataStore := store.NewMongoStore(client.Database(cfg.MongoDB)) - mode := "local" - if cfg.K8sEnabled { - mode = "k8s" - } - runtime, err := worker.NewRuntime(dataStore, mode, cfg.K8sNamespace, cfg.K8sImagePullSecretName, cfg.OperatorRunnerImage, cfg.LocalOperatorRunnerCmd) + runtime, err := worker.NewRuntime( + dataStore, + cfg.K8sNamespace, + cfg.K8sImagePullSecretName, + cfg.OperatorRunnerImage, + cfg.WorkspacePVCName, + cfg.WorkspaceMountPath, + ) if err != nil { fmt.Printf("[worker] failed to initialize runtime: %v\n", err) os.Exit(1) diff --git a/docs.zip b/docs.zip new file mode 100644 index 00000000..0dc71d6c Binary files /dev/null and b/docs.zip differ diff --git a/docs/api.md b/docs/api.md index 6b847a1c..6be2a8ed 100644 --- a/docs/api.md +++ b/docs/api.md @@ -20,6 +20,34 @@ Base path: `/api` } ``` +典型响应: + +```json +{ + "name": "关键帧抽取", + "category": "AI", + "description": "接收视频输入,按固定时间间隔抽取关键帧并输出图片列表与摘要。", + "inputs": [ + { "name": "input", "type": "video", "transmission": "bus", "description": "视频输入或视频 manifest" } + ], + "outputs": [ + { "name": "frames", "type": "image", "transmission": "bus", "description": "关键帧图片列表" }, + { "name": "summary", "type": "json", "transmission": "bus", "description": "抽帧摘要" } + ], + "hyperparams": [ + { "key": "FRAME_INTERVAL_SECONDS", "value": "5", "description": "按秒抽帧间隔", "type": "string" }, + { "key": "MAX_FRAMES_PER_VIDEO", "value": "50", "description": "单视频最多保留关键帧数", "type": "string" } + ], + "skillDefinition": "接收视频文件路径或包含视频路径的 JSON manifest。若路径为 bos://...,先物化到本地工作目录,再按 FRAME_INTERVAL_SECONDS 抽取关键帧,输出图片列表与摘要。" +} +``` + +如果你想把这份响应直接转成自定义算子 Markdown,可以把响应保存为本地 JSON 后运行: + +```bash +python3 scripts/generate_operator_docs.py --input-json /path/to/generated-operator.json +``` + `POST /api/ai/workflows/generate` ```json @@ -79,6 +107,43 @@ Base path: `/api` } ``` +Scan workflows are created through the same workflow APIs. Their payload can include `kind: "scan"` and a workflow-level `scanConfig` object. For BOS scans, `scanConfig` now supports either inline credentials or a reusable `connectionId`: + +```json +{ + "name": "X-platform BOS 扫描", + "type": "streaming", + "kind": "scan", + "schedule": { + "mode": "every_days_at_time", + "dayInterval": 2, + "timeOfDay": "18:30", + "timezone": "Asia/Shanghai" + }, + "scanConfig": { + "dataSource": "BOS", + "connectionId": "storage-abc123", + "bucket": "datastorage", + "prefix": "algo/X-platform-test", + "scanMode": "INCREMENTAL", + "enabled": true + }, + "nodes": [], + "edges": [] +} +``` + +When `connectionId` is present, the workflow stores the reference instead of duplicating AK/SK. The worker and BOS preview API resolve the referenced storage connection at execution time. + +Workflow-level `schedule` now supports three modes: + +- `interval_seconds` + - uses `interval` +- `every_hours` + - uses `intervalHours` +- `every_days_at_time` + - uses `dayInterval`, `timeOfDay`, and optional `timezone` + ### Templates - `GET /api/templates` @@ -97,8 +162,17 @@ Templates now persist the same workflow-level defaults used by project workflows - `PUT /api/operators/:id` - `DELETE /api/operators/:id` -The reconciled built-in catalog includes the original workflow primitives plus: +The built-in catalog includes the original workflow primitives plus: +- `BOS 增量扫描` +- `X 平台数据集索引` +- `采集目录校验` +- `视频质检` +- `数据过滤` +- `视频切片` +- `视频标注` +- `成品打包` +- `抽检采样` - `目录扫描` - `CSV 转 JSON` - `JSON 转 CSV` @@ -129,6 +203,8 @@ The reconciled built-in catalog includes the original workflow primitives plus: - `GET /api/runs` - `GET /api/runs/:id` +- `GET /api/workflows/:id/runs` +- `POST /api/runs/:id/rerun-subtree` - `GET /api/runs/:id/logs` - `GET /api/runs/:id/resources` - `GET /api/runs/:id/artifacts` @@ -142,6 +218,62 @@ Each artifact item can now include: - `data`: the real JSON payload for JSON artifacts - `data.preview`: a compact preview object for non-JSON artifacts such as text, CSV/TSV, JSONL, image thumbnails, audio/video metadata, PDF hints, SQLite/rosbag summaries, and LeRobot dataset summaries +`POST /api/runs/:id/rerun-subtree` + +- body: `{ "nodeId": "" }` +- only allowed on failed runs +- creates a queued child run with: + - `trigger = rerun_subtree` + - `parentRunId = source failed run id` + - `restartFromNodeId = selected node id` +- the child run reuses completed upstream baseline executions and re-executes the selected node plus its downstream subtree + +### Data Platform + +- `GET /api/scan-project` +- `GET /api/scan-configs` +- `POST /api/scan-configs` +- `GET /api/scan-configs/:id` +- `PUT /api/scan-configs/:id` +- `DELETE /api/scan-configs/:id` +- `POST /api/scan-configs/:id/run` +- `GET /api/source-data/collections` +- `GET /api/source-data/collections/:runId/:collectionId` +- `GET /api/source-data/collections/:runId/:collectionId/entries?prefix=` +- `GET /api/packages` +- `GET /api/packages/:runId/:packageName` +- `GET /api/sampling-tasks` +- `GET /api/sampling-tasks/:id` +- `POST /api/sampling-items/:id/review` + +`POST /api/scan-configs` + +```json +{ + "name": "X-platform-test 自采数据扫描", + "description": "扫描并索引真实采集目录", + "sourcePath": "bos://datastorage/algo/X-platform-test", + "bucket": "datastorage", + "prefix": "algo/X-platform-test", + "pattern": "", + "interval": 86400, + "enabled": true, + "dataSource": "BOS" +} +``` + +Scan configs are implemented as real workflows under the system project `数据源扫描`, not as a second execution engine. For BOS-backed pipelines the first node now uses the built-in `BOS 增量扫描` source operator, which persists checkpoint state per `workflowId + nodeId + sourceFingerprint`. The worker only commits that checkpoint after the whole run succeeds, so downstream failures do not silently skip data on the next retry. `GET /api/scan-project` resolves the system project for UI redirects, `POST /api/scan-configs/:id/run` enqueues a normal run, and `GET /api/source-data/collections` returns data from any completed workflow that produced a real `dataset-index.json` artifact. + +For BOS-backed X-platform scans, source collections are inferred from real `meta.json` roots inside the scanned prefix. `collectionId` is a URL-safe stable slug for routing, `collectionName` is the human-readable leaf folder name such as `BJ_001_0001_OsmoNano_2026-03-19_14-51-43`, and `relativePath` preserves the full BOS-relative directory such as `raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43`. The frontend groups these source collections together with completed package outputs inside a single `数据资产` module, and drives project/workflow scoping via query filters instead of separate first-level menus. + +`GET /api/packages` + +Returns finished data packages aggregated from real `packages-manifest.json` run artifacts. Each item includes the source `projectId`, `workflowId`, `runId`, package metadata, embedded `quality` / `annotation` summaries, and a `previewArtifact` when a previewable file such as the clip video exists. + +`GET /api/sampling-tasks` + +Sampling tasks are synced from real `audit-samples.json` run artifacts. The API persists review state separately from the originating run so reviewers can submit decisions later without mutating the original workflow outputs. + ### Audit - `GET /api/audit-tasks` @@ -152,9 +284,36 @@ Each artifact item can now include: - `GET /api/users` - `PUT /api/users/:id` +- `GET /api/storage/connections` +- `POST /api/storage/connections` +- `GET /api/storage/connections/:id` +- `PUT /api/storage/connections/:id` +- `DELETE /api/storage/connections/:id` +- `GET /api/external/business-db/tokens` +- `POST /api/external/business-db/tokens` +- `PUT /api/external/business-db/tokens/:id` +- `POST /api/external/business-db/tokens/:id/rotate` +- `DELETE /api/external/business-db/tokens/:id` - `GET /api/settings` - `PUT /api/settings` +`POST /api/storage/connections` + +```json +{ + "name": "BOS 主连接", + "description": "X-platform 扫描使用", + "provider": "BOS", + "endpoint": "https://bj.bcebos.com", + "accessKeyId": "ak-...", + "secretAccessKey": "sk-..." +} +``` + +`GET /api/storage/connections` + +The response never returns plaintext AK/SK. It only exposes `hasAccessKeyId`, `hasSecretAccessKey`, and masked previews so the UI can confirm a real secret is already stored. + `GET /api/settings` ```json @@ -197,6 +356,121 @@ Each artifact item can now include: } ``` +### External Business DB Write API + +For a copy-paste-ready partner handoff guide, see [docs/external-business-db-write-api-integration.md](/Users/longtaowu/workspace/droboticflow/docs/external-business-db-write-api-integration.md). + +Use this surface when a third party needs to write into a preconfigured business Mongo connection without seeing the raw Mongo URI. The external caller must use a server-generated API key in header `X-API-Key`. + +Internal token-management routes: + +- `GET /api/external/business-db/tokens` +- `POST /api/external/business-db/tokens` +- `PUT /api/external/business-db/tokens/:id` +- `POST /api/external/business-db/tokens/:id/rotate` +- `DELETE /api/external/business-db/tokens/:id` + +`POST /api/external/business-db/tokens` + +```json +{ + "name": "partner-a", + "description": "订单同步写入", + "enabled": true, + "scopes": [ + { + "connectionId": "mongo_abc123", + "collections": ["orders", "customers"] + } + ] +} +``` + +Successful create and rotate responses return the plaintext `apiKey` once: + +```json +{ + "token": { + "id": "exttoken_abc123", + "name": "partner-a", + "enabled": true, + "tokenPreview": "dbw_****7f3a", + "scopes": [ + { + "connectionId": "mongo_abc123", + "collections": ["customers", "orders"] + } + ] + }, + "apiKey": "dbw_9f..." +} +``` + +Third-party write routes: + +- `POST /api/external/business-db/documents` +- `PATCH /api/external/business-db/documents/:documentId` +- `POST /api/external/business-db/upserts` + +All of these routes require header: + +```http +X-API-Key: dbw_... +``` + +`POST /api/external/business-db/documents` + +```json +{ + "connectionId": "mongo_abc123", + "collection": "orders", + "document": { + "externalId": "EXT-1001", + "status": "new", + "amount": 20 + } +} +``` + +`PATCH /api/external/business-db/documents/:documentId` + +```json +{ + "connectionId": "mongo_abc123", + "collection": "orders", + "patch": { + "status": "done", + "amount": 25 + } +} +``` + +`POST /api/external/business-db/upserts` + +```json +{ + "connectionId": "mongo_abc123", + "collection": "orders", + "match": { + "externalId": "EXT-1001" + }, + "patch": { + "status": "done", + "amount": 25 + }, + "setOnInsert": { + "source": "partner-a" + } +} +``` + +External write responses use: + +- `operation: "created"` for inserts and create-path upserts +- `operation: "updated"` for patch and update-path upserts + +External write validation intentionally does not expose raw Mongo update operators. `document`, `patch`, `match`, and `setOnInsert` must all be plain JSON objects. + Workflow create/update payloads also accept workflow-level execution settings: - `0`: inherit the global worker default @@ -242,6 +516,8 @@ Base path: `/worker-api` - list endpoints return `{ "items": [...] }` - delete endpoints return `204 No Content` - validation failures return `400` +- missing or invalid external API key returns `401` +- authenticated external token without the requested connection or collection permission returns `403` - missing resources return `404` - AI service unavailable returns `503` - upstream AI gateway failures return `502` @@ -255,7 +531,7 @@ Base path: `/worker-api` - AI generation is implemented by [`internal/ai`](/Users/longtaowu/workspace/droboticflow/internal/ai) and is called only from the Go API, never directly from the browser. - `GET /api/settings` never returns the saved AI key in plaintext. It only exposes `ai.hasAPIKey` so the browser can show whether a server-side key is already configured. - `timezone` uses IANA timezone identifiers such as `Asia/Shanghai` or `America/Los_Angeles`. The frontend formats timestamps with second precision in that timezone. -- Monitoring previews are rendered from the persisted artifact payload itself, so text/table/robotics previews keep working after the Kubernetes Job has exited and its local filesystem is gone. +- Monitoring previews are rendered from the persisted artifact payload itself, so text/table/robotics previews keep working after the Kubernetes Job has exited and its temporary filesystem is gone. - OSS 相关配置不通过 `/api/settings` 管理。系统设置页只维护 K8s、worker、AI 和默认资源;OSS bucket/prefix 等参数由具体工作流节点维护。 - `/api/storage/*` 也不再读取系统级存储根目录。浏览请求必须显式传入当前 OSS 节点超参里的 `SOURCE_PATH`,API 只在这个节点上下文内列 bucket 和对象。 - Kubernetes Jobs launched for operator nodes use the Python entrypoint at [`backend/operator-runner/main.py`](/Users/longtaowu/workspace/droboticflow/backend/operator-runner/main.py). diff --git a/docs/assets/manuals/platform-data-assets.png b/docs/assets/manuals/platform-data-assets.png new file mode 100644 index 00000000..afdde82f Binary files /dev/null and b/docs/assets/manuals/platform-data-assets.png differ diff --git a/docs/assets/manuals/platform-operators.png b/docs/assets/manuals/platform-operators.png new file mode 100644 index 00000000..9178b1ef Binary files /dev/null and b/docs/assets/manuals/platform-operators.png differ diff --git a/docs/assets/manuals/platform-projects.png b/docs/assets/manuals/platform-projects.png new file mode 100644 index 00000000..871455e5 Binary files /dev/null and b/docs/assets/manuals/platform-projects.png differ diff --git a/docs/assets/manuals/platform-templates.png b/docs/assets/manuals/platform-templates.png new file mode 100644 index 00000000..c7be6c7a Binary files /dev/null and b/docs/assets/manuals/platform-templates.png differ diff --git a/docs/assets/manuals/platform-workflow-editor.png b/docs/assets/manuals/platform-workflow-editor.png new file mode 100644 index 00000000..72274f15 Binary files /dev/null and b/docs/assets/manuals/platform-workflow-editor.png differ diff --git a/docs/builtin-operators-manual.zh-CN.md b/docs/builtin-operators-manual.zh-CN.md new file mode 100644 index 00000000..3cd4c194 --- /dev/null +++ b/docs/builtin-operators-manual.zh-CN.md @@ -0,0 +1,278 @@ +# Drobodata 内建算子手册 + +本文档不是“把每个算子再抄一遍”的长列表,而是当前内建算子体系的总览、使用方法和验收规则。单算子细节请进入分册目录 [`docs/operators/README.md`](/Users/longtaowu/workspace/droboticflow/docs/operators/README.md)。 + +截至 `2026-04-21`,当前线上算子目录中可见 `25` 个 `builtin` 算子、`0` 个 `custom` 实例。 + +![算子列表](./assets/manuals/platform-operators.png) + +## 1. 文档结构 + +- 总手册:说明平台级规则、编排方式、通用坑点和验收口径 +- 分册目录:[`docs/operators/README.md`](/Users/longtaowu/workspace/droboticflow/docs/operators/README.md) +- 自定义算子:[`docs/custom-operator-manual.zh-CN.md`](/Users/longtaowu/workspace/droboticflow/docs/custom-operator-manual.zh-CN.md) +- 算子开发参考:[`docs/operator-authoring.md`](/Users/longtaowu/workspace/droboticflow/docs/operator-authoring.md) + +建议阅读顺序: + +1. 先读本手册,理解平台约束 +2. 再按类别进入单算子分册 +3. 最后回到工作流设计和回归验证 + +## 2. 平台级规则 + +内建算子的价值不在于“名字固定”,而在于它们遵循统一运行时契约: + +- worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH` +- 节点配置通过 `NODE_DATA_JSON` 注入 runner +- 算子在 `OUTPUT_DIR` 中产出业务文件 +- worker 再收集 `RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` +- 大文件和大 JSON 默认不会走 API 内联,而是走 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用 + +你在设计 DAG 时,应该优先想清楚这五件事: + +1. 这个节点的输入到底是结构化 JSON,还是远程对象引用,还是本地文件路径 +2. 这个节点的输出是否需要被下游继续消费 +3. 这个节点是否有真实外部副作用,例如上传或写库 +4. 这个节点失败后如何重试,是否幂等 +5. 这个节点验收时应该看平台运行记录,还是必须去外部系统核对 + +## 3. 运行时模型 + +```mermaid +sequenceDiagram + participant UI as Workflow UI + participant Worker as Go Worker + participant Job as Kubernetes Job + participant Runner as operator-runner + participant Storage as PVC / BOS / DB + + UI->>Worker: 保存工作流 DAG + Worker->>Worker: 汇总上游 artifact 与节点超参数 + Worker->>Job: 创建节点 Job + Job->>Runner: 注入 NODE_DATA_JSON / INPUT_ARTIFACTS_PATH / OUTPUT_DIR + Runner->>Storage: 读取源文件、处理业务、写结果 + Runner-->>Worker: result.json / stage-manifest.json + Worker->>Worker: 持久化 execution / run +``` + +你在理解内建算子时,不要把它想成“函数调用”,而应该把它想成“受平台约束的一次节点执行”。这会直接影响你对下列问题的判断: + +- 是否允许把大视频、大包体结果直接塞进日志或 Mongo +- 是否应该用路径引用代替内联内容 +- 为什么一个输入端口不能接多条上游边 +- 为什么上传和写库必须把外部副作用单独验收 + +## 4. 什么时候优先用内建算子 + +优先选择内建算子的情形: + +- 逻辑已经稳定,且会被多条工作流重复使用 +- 输入输出契约已经清晰,可被 UI 和 AI 工作流生成器复用 +- 需要受控地处理 BOS、PVC、Mongo、审核流、视频链路等平台强约束逻辑 +- 需要长期维护、回归、上线和扩展 + +不应该勉强塞进内建算子的情形: + +- 只是临时调试脚本 +- 逻辑尚未稳定 +- 接口和产物结构还在频繁变化 +- 需要快速验证单次项目需求 + +这类场景更适合先走 [`docs/custom-operator-manual.zh-CN.md`](/Users/longtaowu/workspace/droboticflow/docs/custom-operator-manual.zh-CN.md) 里的自定义路径。 + +## 5. 当前算子分类地图 + +### 5.1 IO / 源与汇 + +- [BOS 增量扫描](/Users/longtaowu/workspace/droboticflow/docs/operators/BOS-增量扫描.md) +- [真实数据源](/Users/longtaowu/workspace/droboticflow/docs/operators/真实数据源.md) +- [目录扫描](/Users/longtaowu/workspace/droboticflow/docs/operators/目录扫描.md) +- [结果上传](/Users/longtaowu/workspace/droboticflow/docs/operators/结果上传.md) + +适合解决: + +- 外部数据引入 +- 增量发现 +- 目录级概览 +- 最终结果落盘和上传 + +### 5.2 DATASET / 数据集整理 + +- [X 平台数据集索引](/Users/longtaowu/workspace/droboticflow/docs/operators/X-平台数据集索引.md) +- [采集目录校验](/Users/longtaowu/workspace/droboticflow/docs/operators/采集目录校验.md) + +适合解决: + +- 原始数据集结构解包 +- 采集目录合法性验证 +- 为视频链路提供标准化 manifest + +### 5.3 DATA / 结构化处理中间层 + +- [CSV 转 JSON](/Users/longtaowu/workspace/droboticflow/docs/operators/CSV-转-JSON.md) +- [JSON 转 CSV](/Users/longtaowu/workspace/droboticflow/docs/operators/JSON-转-CSV.md) +- [JSON 字段提取](/Users/longtaowu/workspace/droboticflow/docs/operators/JSON-字段提取.md) +- [JSON 过滤](/Users/longtaowu/workspace/droboticflow/docs/operators/JSON-过滤.md) +- [JSON 映射重命名](/Users/longtaowu/workspace/droboticflow/docs/operators/JSON-映射重命名.md) +- [MongoDB 写入](/Users/longtaowu/workspace/droboticflow/docs/operators/MongoDB-写入.md) + +适合解决: + +- 数据转换 +- 条件筛选 +- 字段清洗 +- 报表/外部库落地 + +### 5.4 VIDEO / 视频主链路 + +- [视频质检](/Users/longtaowu/workspace/droboticflow/docs/operators/视频质检.md) +- [数据过滤](/Users/longtaowu/workspace/droboticflow/docs/operators/数据过滤.md) +- [视频切片](/Users/longtaowu/workspace/droboticflow/docs/operators/视频切片.md) +- [视频标注](/Users/longtaowu/workspace/droboticflow/docs/operators/视频标注.md) +- [成品打包](/Users/longtaowu/workspace/droboticflow/docs/operators/成品打包.md) + +适合解决: + +- 质检 +- 筛选 +- clip 切分 +- 标注 +- 成品组装 + +### 5.5 QA / HUMAN / LOGIC / UTIL / AI / ROBOTICS + +- [抽检采样](/Users/longtaowu/workspace/droboticflow/docs/operators/抽检采样.md) +- [人工审核](/Users/longtaowu/workspace/droboticflow/docs/operators/人工审核.md) +- [条件分流](/Users/longtaowu/workspace/droboticflow/docs/operators/条件分流.md) +- [文本模板渲染](/Users/longtaowu/workspace/droboticflow/docs/operators/文本模板渲染.md) +- [Shell 命令执行](/Users/longtaowu/workspace/droboticflow/docs/operators/Shell-命令执行.md) +- [模型推理](/Users/longtaowu/workspace/droboticflow/docs/operators/模型推理.md) +- [LeRobot 数据集概览](/Users/longtaowu/workspace/droboticflow/docs/operators/LeRobot-数据集概览.md) +- [ROSBag 数据概览](/Users/longtaowu/workspace/droboticflow/docs/operators/ROSBag-数据概览.md) + +这几类算子承担的是: + +- 人工决策 +- 路由控制 +- 模板渲染 +- 受控命令执行 +- 模型/脚本推理 +- 机器人数据集概览 + +## 6. 常见编排模式 + +### 6.1 增量扫描链路 + +```mermaid +flowchart LR + A["BOS 增量扫描"] --> B["X 平台数据集索引"] + B --> C["采集目录校验"] + C --> D["视频质检 / 数据过滤"] +``` + +适合持续发现新增目录或新增对象,并只处理本轮增量。 + +### 6.2 视频生产链路 + +```mermaid +flowchart LR + A["视频质检"] --> B["数据过滤"] + B --> C["视频切片"] + C --> D["视频标注"] + D --> E["成品打包"] + E --> F["结果上传"] +``` + +适合把原始视频逐步转成可交付 clip、annotation 和 package。 + +### 6.3 审核链路 + +```mermaid +flowchart LR + A["候选数据"] --> B["人工审核"] + B -->|"approved"| C["结果上传 / 写库"] + B -->|"rejected"| D["拒绝支路归档"] +``` + +审核节点后一定要显式拆分分支,不要让通过和拒绝结果流入同一后继节点。 + +## 7. 最容易踩的坑 + +### 7.1 把内建算子当成“内存函数” + +错误理解: + +- 认为上游输出会直接以内存对象传给下游 +- 认为大 JSON 一定能在运行记录里完整看到 + +正确理解: + +- 节点之间靠 artifact、manifest、PVC、BOS 引用通信 +- 大结果很可能已经外置 + +### 7.2 一个上传节点同时接多条线 + +当前平台明确限制: + +- 一个输入端口只能有一条上游边 +- 想上传多个不同结果,就拆多个上传节点 + +### 7.3 只看平台状态,不看真实外部副作用 + +这在以下算子上最危险: + +- 结果上传 +- MongoDB 写入 +- 人工审核 + +验收时必须去目标 BOS、目标库或审核流里核对真实结果。 + +### 7.4 在视频链路里传大对象而不是传引用 + +视频类算子的正确姿势是: + +- 传 manifest +- 传本地路径 +- 传 `bos://...` 引用 +- 让 runner 在必要时物化文件 + +而不是把大量媒体信息硬塞进内联 payload。 + +## 8. 验收标准 + +每个内建算子至少应该满足下面四层验收: + +1. 节点执行层:execution 状态为 `completed` +2. 产物层:`stage-manifest.json` 和 `OUTPUT_DIR` 中存在期望结果 +3. 链路层:下游节点能真实消费结果 +4. 外部副作用层:上传、写库、审核等副作用在外部系统中真实可见 + +推荐证据: + +- 一次成功 run 的 execution 日志 +- 一份 stage manifest +- 一组代表性输出文件路径 +- 外部系统侧的核验截图或查询结果 + +## 9. 文档使用方式 + +当你要修改工作流或给 AI 生成工作流时,建议这样使用这些文档: + +1. 先在本手册中确认平台级约束 +2. 到 [`docs/operators/README.md`](/Users/longtaowu/workspace/droboticflow/docs/operators/README.md) 选中目标算子 +3. 阅读该算子的单册,确认端口、超参数、排障和验收 +4. 再回到工作流设计和回归 + +这样做的好处是: + +- 不会因为只看名字误判算子职责 +- 不会遗漏端口语义和外部副作用 +- 能更快定位失败点 + +## 10. 相关文档 + +- [平台使用手册](/Users/longtaowu/workspace/droboticflow/docs/platform-usage-manual.zh-CN.md) +- [内建算子分册索引](/Users/longtaowu/workspace/droboticflow/docs/operators/README.md) +- [自定义算子手册](/Users/longtaowu/workspace/droboticflow/docs/custom-operator-manual.zh-CN.md) +- [算子开发文档](/Users/longtaowu/workspace/droboticflow/docs/operator-authoring.md) diff --git a/docs/custom-operator-manual.zh-CN.md b/docs/custom-operator-manual.zh-CN.md new file mode 100644 index 00000000..e20cd3b7 --- /dev/null +++ b/docs/custom-operator-manual.zh-CN.md @@ -0,0 +1,516 @@ +# Drobodata 自定义算子手册 + +本文档面向平台研发、算法工程师和工作流设计者,说明在当前架构下如何设计、实现、调试、文档化和验收自定义算子。 + +配套文档: + +- [内建算子手册](/Users/longtaowu/workspace/droboticflow/docs/builtin-operators-manual.zh-CN.md) +- [内建算子分册索引](/Users/longtaowu/workspace/droboticflow/docs/operators/README.md) +- [自定义算子分册体系](/Users/longtaowu/workspace/droboticflow/docs/custom-operators/README.md) +- [算子开发文档](/Users/longtaowu/workspace/droboticflow/docs/operator-authoring.md) + +相关实现入口: + +- `backend/operator-runner/main.py` +- `internal/worker/executor.go` +- `internal/worker/runtime.go` +- `internal/k8s/launcher.go` +- `internal/api/server.go` +- `internal/domain/types.go` +- `src/pages/Operators.tsx` + +## 1. 自定义算子的定位 + +自定义算子不是“代码随便跑一下”的兜底概念,而是平台用来承接非标准化逻辑的受控扩展点。 + +它适合: + +- 需求刚出现,逻辑尚未稳定 +- 需要快速验证一个新处理链路 +- 需要接一个内建算子暂时没有覆盖的业务逻辑 +- 需要独立镜像、独立依赖或项目级脚本 + +它不适合: + +- 已经被多个工作流复用的稳定公共能力 +- 明显属于平台核心语义的能力,例如增量扫描、统一上传、审核流、标准视频处理 +- 需要长期维护但没有人负责的临时脚本 + +如果一个自定义算子会长期存在,请从一开始就把它当成“将来可能要内建化的标准件”来设计,而不是当作一次性脚本。 + +## 2. 三种落地方式 + +### 2.1 命令型验证 + +最常见的起点是: + +- [Shell 命令执行](/Users/longtaowu/workspace/droboticflow/docs/operators/Shell-命令执行.md) +- [模型推理](/Users/longtaowu/workspace/droboticflow/docs/operators/模型推理.md) + +适合: + +- 快速验证 +- 小范围项目定制 +- 临时脚本或受控命令执行 + +优点: + +- 上手快 +- 无需改平台主逻辑 + +问题: + +- 约束弱 +- 复用差 +- 难统一回归 + +### 2.2 注册型自定义算子 + +通过平台算子管理页或 API 创建 `type=custom` 的算子元数据,补齐: + +- 名称 +- 类别 +- 描述 +- 输入端口 +- 输出端口 +- 超参数 +- 镜像 +- `skillDefinition` + +适合: + +- 已经形成稳定的输入输出契约 +- 需要给 UI、工作流生成器、调用方提供统一入口 +- 需要把逻辑包装为更像“产品能力”的算子 + +### 2.3 内建化 + +当一个自定义算子满足下面任意条件时,应该考虑内建化: + +- 复用频率高 +- 失败模式明确且可以平台化治理 +- 需要和 BOS/PVC/审核流/回归体系深度集成 +- 需要严格的文档、测试和验收 + +## 3. 元数据字段该怎么设计 + +当前 `OperatorRecord` 关键字段包括: + +- `id` +- `name` +- `type` +- `category` +- `description` +- `inputs` +- `outputs` +- `hyperparams` +- `image` +- `skillDefinition` + +设计建议: + +### 3.1 名称 + +- 用业务动作命名,不要只写技术实现 +- 名称应该让工作流设计者一眼看出用途 +- 避免“脚本1”“临时处理”“新版聚合”这类无语义名字 + +### 3.2 分类 + +- 选最贴近职责的类别 +- 不要为了“好找”滥用一个分类 +- 如果算子主要做数据清洗,就归 `DATA` +- 如果主要做文件引入或导出,就归 `IO` + +### 3.3 description + +`description` 负责回答一个问题: + +“这个算子到底做什么,不做什么。” + +应该写清: + +- 主职责 +- 主要输入 +- 主要输出 + +### 3.4 image + +`image` 字段表示“运行这个自定义算子时实际拉起的容器镜像”。 + +当前平台的责任边界很明确: + +- 平台不会帮你构建自定义镜像 +- 平台不会帮你推送镜像到仓库 +- 平台只会保存你填写的镜像地址,并在运行时直接用它启动 Kubernetes Job + +因此,自定义镜像的标准流程是: + +1. 用户自己在外部容器仓库构建并推送镜像 +2. 用户把完整镜像地址填写到算子的 `image` 字段 +3. 工作流运行时,平台直接拉取这个地址执行 + +填写要求: + +- 必须填写完整镜像引用,格式建议为 `//:` +- 不要依赖 `latest`,应使用明确版本标签,方便回滚和排障 +- 镜像仓库必须能被当前集群拉取 + +当前部署环境里,推荐沿用与平台一致的百度云 CCR 命名方式,例如: + +```text +ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/custom-: +``` + +如果你当前接的是百度云 CCR `xspark` 命名空间,并希望让用户自己推镜像再在算子里填写 `image`,请直接参考: + +- [百度云 CCR `xspark` 自定义算子接入指南](/Users/longtaowu/workspace/droboticflow/docs/custom-operators/百度云-CCR-xspark-自定义算子接入指南.md) + +这份分册包含: + +- `xspark` 私有镜像的真实验收结论 +- 登录、推送、填写 `image` 的标准流程 +- 自定义镜像最小运行协议 +- 一个可直接复制的最小 Dockerfile / `run.sh` 示例 +- 常见失败模式与排障方式 + +特别注意: + +- 不要把 `droboticflow-operator-runner:*` 当成自定义镜像名填写 +- 这类平台内建 runner 镜像在运行时可能被系统镜像覆盖,不适合作为自定义算子的独立镜像地址 + +### 3.5 skillDefinition + +`skillDefinition` 不应该只写一句简介。 + +它应该是给未来工作流生成器、平台维护者和文档脚本看的高密度说明,至少覆盖: + +- 输入需要什么结构 +- 输出会产生什么结果 +- 超参数分别控制什么 +- 哪些场景适合用 +- 哪些场景不该用 +- 失败时怎么理解 + +如果你愿意,`skillDefinition` 可以直接按 Markdown 写,这也是本仓库现在推荐的方向。 + +## 4. 输入输出契约 + +### 4.1 输入不是“只有 path” + +自定义算子最常见的错误,是假设输入永远长这样: + +```python +path = artifact["path"] +``` + +这是不够的。你必须同时考虑以下几类输入: + +- `artifact.data`:结构化 JSON 已经在内存可用 +- `artifact.path`:本地文件路径 +- `artifact.path = bos://bucket/key`:远程对象引用 +- `dataRef.payloadPath`:外置 payload 文件 +- `metadata`:上游附带的摘要信息 + +推荐读取顺序: + +1. 先看 `artifact.data` +2. 再看本地 `artifact.path` +3. 再判断是否需要物化 `bos://...` +4. 最后处理外置 payload + +### 4.2 输出必须满足 artifact 契约 + +一个可靠的自定义算子应该明确产出: + +- `name` +- `type` +- `path` +- `data` +- `metadata` +- 多分支时补齐 `portId` / `portName` + +推荐原则: + +- 小 JSON:可以直接放 `data` +- 大 JSON:外置为文件或 payload +- 大文件:写到 `OUTPUT_DIR` +- 长期结果:上传到对象存储后再返回远程引用 + +### 4.3 多输出必须显式绑定端口 + +如果你的算子有多个输出分支: + +- 每个输出 artifact 都要明确绑定 `portId` +- 下游边必须接到对应输出端口 +- 不要把不同语义的结果混在一个输出口里 + +## 5. 运行时环境 + +平台当前是 `Kubernetes Job-only + 共享 PVC + BOS` 的执行模型。 + +常见环境变量和文件: + +- `RUN_ID` +- `WORKFLOW_ID` +- `NODE_ID` +- `NODE_LABEL` +- `NODE_DATA_JSON` +- `INPUT_ARTIFACTS_PATH` +- `OUTPUT_DIR` +- `RESULT_OUTPUT_PATH` +- `STAGE_MANIFEST_PATH` + +如果节点涉及存储,还可能注入: + +- `SOURCE_PATH` +- `BOS_ENDPOINT` +- `BOS_ACCESS_KEY_ID` +- `BOS_SECRET_ACCESS_KEY` +- `STORAGE_BUCKET` +- `STORAGE_PREFIX` + +这意味着自定义算子要从一开始就接受一个事实: + +- 你不是在写“本地脚本” +- 你是在写“平台节点执行逻辑” + +## 6. 数据路径设计 + +### 6.1 什么时候只传引用 + +优先只传引用的场景: + +- 视频 +- 大图片集 +- 大 JSON 清单 +- 压缩包 +- 成品目录 + +### 6.2 什么时候要物化到本地 + +必须物化的场景: + +- ffmpeg / ffprobe 需要真实文件 +- OpenCV 需要直接读本地路径 +- 业务脚本只能消费本地文件 + +### 6.3 什么时候要上传 BOS + +需要跨 Job 生命周期保留时,就不要只留在 `OUTPUT_DIR`: + +- clip +- annotation +- package +- 报表 +- 大型结果集 + +## 7. 自定义算子的文档体系 + +这次仓库把自定义算子文档拆成三层: + +1. 本手册:平台级规则 +2. 模板与示例:[`docs/custom-operators/README.md`](/Users/longtaowu/workspace/droboticflow/docs/custom-operators/README.md) +3. 实例级分册:[`docs/custom-operators/instances/README.md`](/Users/longtaowu/workspace/droboticflow/docs/custom-operators/instances/README.md) + +你应该为每个长期存在的自定义算子单独维护一篇 Markdown,至少包括: + +- 作用与边界 +- 输入输出契约 +- 端口语义 +- 超参数 +- 运行时依赖 +- 配置示例 +- 失败模式 +- 验收口径 + +## 8. 推荐开发流程 + +### 8.1 第一步:先写文档,再写实现 + +在动手之前,先用 [`自定义算子设计模板`](/Users/longtaowu/workspace/droboticflow/docs/custom-operators/templates/自定义算子设计模板.md) 写清楚: + +- 它解决什么问题 +- 它的输入是什么 +- 它的输出是什么 +- 为什么现有内建算子不够 +- 它未来是否可能内建化 + +### 8.2 第二步:先走命令型验证 + +如果逻辑还在探索期,优先使用: + +- `Shell 命令执行` +- `模型推理` + +目标不是“马上做成正式算子”,而是先确认: + +- 输入输出契约跑得通 +- 依赖准备齐全 +- 产物结构合理 + +### 8.3 第三步:收敛成注册型自定义算子 + +当输入输出已经稳定,就进入平台注册并补齐: + +- 端口 +- 超参数 +- 镜像 +- 详细 `skillDefinition` +- 单算子 Markdown 文档 + +### 8.4 第四步:补测试和回归 + +至少覆盖: + +- 输入解析 +- 关键超参数 +- 输出 artifact 结构 +- 失败分支 +- 一条真实工作流回归 + +### 8.5 第五步:决定是否内建化 + +如果该算子开始被多个项目或多个流程复用,就不应长期停留在“项目级脚本”状态。 + +## 9. 代码模板建议 + +### 9.1 结构化 JSON 型 + +适合: + +- 字段提取 +- 聚合统计 +- 路由判断 +- 清洗和映射 + +基本流程: + +1. 读取 `INPUT_ARTIFACTS_PATH` +2. 优先从 `artifact.data` 取 payload +3. 做纯结构化处理 +4. 输出 JSON artifact + +### 9.2 文件处理型 + +适合: + +- 视频 +- 压缩包 +- 大型报表 +- 目录级产物 + +基本流程: + +1. 解析 artifact 引用 +2. 必要时物化到本地 +3. 在 `OUTPUT_DIR` 写结果 +4. 生成 artifact 和 stage manifest +5. 必要时上传对象存储 + +### 9.3 双分支或多分支型 + +适合: + +- 审核 +- 条件路由 +- 成功/失败分流 + +要求: + +- 每个分支都有明确输出端口 +- 每个 artifact 都绑定端口 +- 每个分支都能单独验收 + +## 10. 安全、幂等和资源约束 + +### 10.1 安全 + +- 不要把密钥硬编码进脚本或文档 +- 优先使用环境变量和平台连接配置 +- 对外部命令和路径输入做校验 +- 不要让脚本任意扫描工作目录之外的敏感路径 + +### 10.2 幂等 + +自定义算子要尽量回答清楚: + +- 重跑同一个 run 会不会重复写入 +- 上传重试会不会覆盖 +- 写库失败重试会不会插入重复数据 + +### 10.3 资源 + +要明确: + +- 这个算子是 CPU 型还是 IO 型 +- 是否依赖 GPU +- 是否会占大量磁盘 +- 是否会生成大量中间文件 + +## 11. 常见问题 + +### 11.1 只在 `OUTPUT_DIR` 写结果,不做长期保存 + +问题: + +- Job 结束后临时文件不等于长期结果 + +做法: + +- 需要保留的结果显式上传 BOS 或写入目标系统 + +### 11.2 误把 preview URL 当业务输入 + +问题: + +- preview 链接不等于生产消费路径 + +做法: + +- 真正处理时优先使用 `bos://...` 或本地物化路径 + +### 11.3 文档只写“干什么”,不写“怎么验收” + +问题: + +- 后续维护者无法判断节点是否真的工作 + +做法: + +- 每篇自定义算子 Markdown 都必须包含验收章节 + +### 11.4 skillDefinition 太短 + +问题: + +- AI 和维护者都无法从元数据中理解正确使用方式 + +做法: + +- 至少写清输入、输出、超参数、使用边界和失败语义 + +## 12. 验收要求 + +每个正式自定义算子上线前,至少满足: + +1. 有一篇独立 Markdown 文档 +2. 有明确输入输出契约 +3. 有最小测试或脚本级验证 +4. 有一次真实工作流回归 +5. 有外部副作用核验证据 + +建议证据: + +- 文档链接 +- 节点配置示例 +- 一次成功 run +- 一份 stage manifest +- 一组真实产物 + +## 13. 相关资料 + +- [内建算子手册](/Users/longtaowu/workspace/droboticflow/docs/builtin-operators-manual.zh-CN.md) +- [自定义算子分册体系](/Users/longtaowu/workspace/droboticflow/docs/custom-operators/README.md) +- [算子开发文档](/Users/longtaowu/workspace/droboticflow/docs/operator-authoring.md) +- [平台使用手册](/Users/longtaowu/workspace/droboticflow/docs/platform-usage-manual.zh-CN.md) diff --git a/docs/custom-operators/AI-生成到正式算子文档流程.md b/docs/custom-operators/AI-生成到正式算子文档流程.md new file mode 100644 index 00000000..a80e23a0 --- /dev/null +++ b/docs/custom-operators/AI-生成到正式算子文档流程.md @@ -0,0 +1,161 @@ +# AI 生成到正式算子文档流程 + +本文档回答一个非常具体的问题: + +平台已经支持 `POST /api/ai/operators/generate` 生成算子定义,那么怎样把这份 AI 输出变成一篇可维护、可验收、可长期留存的正式 Markdown 文档? + +结论是:不要直接把 AI 返回结果当最终文档,而是把它当第一版元数据草稿。 + +## 1. AI 输出能解决什么 + +AI 生成接口最适合做三件事: + +1. 给出算子名称、类别、输入输出端口和超参数的初稿 +2. 帮你形成第一版 `skillDefinition` +3. 帮你把“自然语言需求”收敛成结构化算子定义 + +它不能替你完成的部分: + +- 真实运行时依赖确认 +- 大文件/大 JSON 的数据路径设计 +- 幂等和外部副作用设计 +- 真实工作流验收 + +## 2. 推荐流程 + +### 第一步:先拿 AI 生成结构化定义 + +请求示例: + +```json +{ + "description": "生成一个算子,用于接收视频输入,抽取关键帧并输出图片列表。" +} +``` + +返回结果里通常会包含: + +- `name` +- `category` +- `description` +- `inputs` +- `outputs` +- `hyperparams` +- `skillDefinition` + +### 第二步:人工补齐缺失信息 + +AI 输出之后,必须人工核对至少五件事: + +1. 输入是否真的能从当前平台 artifact 契约中拿到 +2. 输出是内联 JSON、文件路径还是对象存储引用 +3. 超参数是否足够支撑实际运行 +4. 镜像、依赖和资源要求是否明确 +5. 失败模式和验收方式是否清楚 + +### 第三步:生成第一版 Markdown + +如果你已经拿到了单个 operator JSON,可以直接运行: + +```bash +python3 scripts/generate_operator_docs.py --input-json /path/to/generated-operator.json +``` + +脚本现在支持三种输入形式: + +- `/api/operators` 风格:`{ "items": [...] }` +- 纯数组:`[ {...}, {...} ]` +- 单个 operator 对象:`{ ... }` + +对于单个 `custom` operator,生成结果会出现在: + +- `docs/custom-operators/instances/` + +### 第四步:把生成稿改成“正式文档” + +生成稿只是基于元数据自动铺开的第一版,你还需要补: + +- 具体上下游编排方式 +- 真实输入输出示例 +- 运行时依赖 +- 失败排查步骤 +- 验收证据要求 + +推荐直接参考: + +- [自定义算子设计模板](./templates/自定义算子设计模板.md) +- [自定义算子实现模板](./templates/自定义算子实现模板.md) +- [自定义算子验收模板](./templates/自定义算子验收模板.md) + +### 第五步:注册并回归 + +当文档和实现都达到可维护标准,再去: + +- 算子管理页注册 +- 或者通过 `/api/operators` 创建 + +然后至少完成一次真实工作流回归。 + +## 3. 生成稿最常见的问题 + +### 3.1 AI 过度乐观 + +常见现象: + +- 把输出都写成 `json` +- 默认所有输入都能直接读取 +- 忽略镜像、依赖和路径物化问题 + +解决方式: + +- 人工把真正的大文件结果改成路径引用或外置 payload +- 明确哪些输入必须先物化 + +### 3.2 skillDefinition 太像宣传语 + +常见现象: + +- 只写“该算子用于高效处理数据” +- 没有输入输出和失败语义 + +解决方式: + +- 按 [skillDefinition 编写指南](./skillDefinition-编写指南.md) 重写 + +### 3.3 文档像字段列表,不像可执行说明 + +常见现象: + +- 有字段,但没有使用方式 +- 有端口,但没有编排建议 +- 有超参数,但没有边界说明 + +解决方式: + +- 补具体配置示例 +- 补上下游组合 +- 补失败与验收章节 + +## 4. 一份正式文档至少应回答的问题 + +1. 这个算子解决什么问题 +2. 为什么不能直接用现有内建算子 +3. 输入到底长什么样 +4. 输出到底给下游什么 +5. 哪些超参数必须填 +6. 哪些依赖必须提前准备 +7. 失败时先看哪里 +8. 成功时应该去哪里核验 + +如果一篇文档答不完这八个问题,它通常还不能算正式可维护文档。 + +## 5. 推荐配套物 + +每个正式自定义算子,建议同时保留: + +- 一篇实例 Markdown +- 一份最小配置示例 +- 一次真实 run 记录 +- 一份验收记录 + +这会让后续维护成本显著下降。 diff --git a/docs/custom-operators/README.md b/docs/custom-operators/README.md new file mode 100644 index 00000000..72b88d87 --- /dev/null +++ b/docs/custom-operators/README.md @@ -0,0 +1,54 @@ +# 自定义算子 Markdown 体系 + +当前目录用于承载“自定义算子的详细 Markdown 文档”,目标不是只放一篇总手册,而是把设计、实现、验收和实例说明拆开。 + +目录结构: + +- 模板: + - [自定义算子设计模板](./templates/自定义算子设计模板.md) + - [自定义算子实现模板](./templates/自定义算子实现模板.md) + - [自定义算子验收模板](./templates/自定义算子验收模板.md) +- 方法文档: + - [AI 生成到正式算子文档流程](./AI-生成到正式算子文档流程.md) + - [skillDefinition 编写指南](./skillDefinition-编写指南.md) + - [百度云 CCR `xspark` 自定义算子接入指南](./百度云-CCR-xspark-自定义算子接入指南.md) +- 示例: + - [示例:自定义 JSON 聚合算子](./examples/示例-自定义-JSON-聚合算子.md) + - [示例:自定义视频窗口采样算子](./examples/示例-自定义-视频窗口采样算子.md) + - [示例:自定义关键帧抽取算子](./examples/示例-自定义-关键帧抽取算子.md) + - [示例:自定义图像分割算子](./examples/示例-自定义-图像分割算子.md) +- 实例级导出: + - [实例索引](./instances/README.md) + +使用方式: + +1. 先阅读 [`../custom-operator-manual.zh-CN.md`](../custom-operator-manual.zh-CN.md) +2. 用模板写出自己的算子说明 +3. 参考示例补齐输入输出契约、失败模式和验收章节 +4. 当环境中存在真实 `custom` 算子实例时,运行文档生成脚本把实例级 Markdown 导出到 `instances/` + +自定义镜像使用约束: + +- 用户自己构建并推送镜像到可被集群拉取的容器仓库 +- 用户自己把完整镜像地址填写到算子的 `image` 字段 +- 平台不会帮你构建或推送自定义镜像 +- 建议填写明确 tag 的完整地址,例如 `ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/custom-:` +- 当前 `wjzh` 环境已经真实验证过 `ccr-29eug8s3-pub.cnc.bj.baidubce.com/xspark/:` 这类私有镜像可正常拉取和执行,详见 [百度云 CCR `xspark` 自定义算子接入指南](./百度云-CCR-xspark-自定义算子接入指南.md) +- 不要把 `droboticflow-operator-runner:*` 当成自定义镜像填写 + +生成命令示例: + +```bash +python3 scripts/generate_operator_docs.py --api-base-url http://114.111.24.223:30080 +``` + +如果你手上只有一个 AI 生成的单算子 JSON,也可以直接生成文档,不必先手工包装成 `{ "items": [...] }`: + +```bash +python3 scripts/generate_operator_docs.py --input-json /path/to/generated-operator.json +``` + +说明: + +- 当前线上算子目录里还没有稳定存在的 `custom` 实例,因此 `instances/` 暂时只有索引页 +- 这并不影响先把自定义算子的文档模板和规范建立好 diff --git a/docs/custom-operators/examples/示例-自定义-JSON-聚合算子.md b/docs/custom-operators/examples/示例-自定义-JSON-聚合算子.md new file mode 100644 index 00000000..c3acbfab --- /dev/null +++ b/docs/custom-operators/examples/示例-自定义-JSON-聚合算子.md @@ -0,0 +1,94 @@ +# 示例:自定义 JSON 聚合算子 + +本文档是一个“注册型自定义算子”的示例,不代表当前线上已经存在该实例。它的目的,是展示一篇合格的自定义算子 Markdown 应该写到什么粒度。 + +## 1. 作用 + +该算子接收上游多个结构化 JSON artifact,把它们按指定字段做聚合统计,输出一份聚合结果和一份摘要报表。 + +## 2. 适用场景 + +- 多个清单合并后做指标汇总 +- 统计每个目录、日期、地点、设备维度的样本数 +- 为下游写库、报表或审核提供聚合结果 + +## 3. 不适用场景 + +- 需要真正做关系型 join +- 需要处理超大体量列式数据 +- 需要复杂 SQL 语义 + +## 4. 输入契约 + +### 输入端口 + +- `in-1 / input / json / bus`:多个上游 artifact 的结构化 JSON + +### 允许输入形态 + +- `artifact.data.items` +- `artifact.data.entries` +- 单对象 `artifact.data` +- 外置 `dataRef.payloadPath` + +### 最小输入示例 + +```json +{ + "items": [ + {"city": "Beijing", "scene": "park", "count": 3}, + {"city": "Shanghai", "scene": "indoor", "count": 2} + ] +} +``` + +## 5. 输出契约 + +### 输出端口 + +- `out-1 / aggregated / json / bus` + +### 输出内容 + +- `groups`:按字段聚合后的结果 +- `summary`:输入记录数、聚合组数、过滤掉的脏记录数 + +## 6. 超参数 + +- `GROUP_BY_FIELDS`:逗号分隔字段列表,例如 `city,scene` +- `COUNT_FIELD`:如果存在,按该字段求和;如果为空,则按记录条数统计 +- `DROP_EMPTY_GROUP`:是否丢弃分组字段为空的记录 + +## 7. 运行时注意事项 + +- 优先从 `artifact.data` 读取,不要强制要求本地文件 +- 如果输入经由 `dataRef.payloadPath` 外置,应先加载 payload +- 输出 JSON 如果体积较大,应外置为 payload,而不是全部塞回内联字段 + +## 8. 配置示例 + +```json +{ + "label": "按城市和场景聚合", + "operatorId": "custom-json-aggregate", + "hyperparams": { + "GROUP_BY_FIELDS": "city,scene", + "COUNT_FIELD": "count", + "DROP_EMPTY_GROUP": "true" + } +} +``` + +## 9. 失败模式 + +- `GROUP_BY_FIELDS` 为空:直接失败 +- 输入不是对象也不是列表:直接失败 +- `COUNT_FIELD` 不是数值:记录错误并跳过该条 + +## 10. 验收 + +至少检查: + +- 输出聚合组数是否正确 +- 总数是否与原始样本一致 +- 下游写库或报表节点能否直接消费结果 diff --git a/docs/custom-operators/examples/示例-自定义-关键帧抽取算子.md b/docs/custom-operators/examples/示例-自定义-关键帧抽取算子.md new file mode 100644 index 00000000..18c05c16 --- /dev/null +++ b/docs/custom-operators/examples/示例-自定义-关键帧抽取算子.md @@ -0,0 +1,110 @@ +# 示例:自定义关键帧抽取算子 + +这个示例直接对应平台 API 文档里 `POST /api/ai/operators/generate` 的典型需求:“接收视频输入,抽取关键帧并输出图片列表”。 + +它不是当前线上已注册实例,而是一份“足够接近真实注册稿”的实例级 Markdown。 + +## 1. 作用 + +该算子接收上游视频引用或视频 manifest,按固定时间间隔抽取关键帧,输出关键帧图片列表与摘要清单。 + +## 2. 为什么做成自定义算子 + +当前内建算子里有: + +- [视频切片](/Users/longtaowu/workspace/droboticflow/docs/operators/视频切片.md) +- [视频标注](/Users/longtaowu/workspace/droboticflow/docs/operators/视频标注.md) + +但没有一个“只做关键帧抽取”的轻量算子。它更适合作为: + +- 预览链路 +- 轻量质检链路 +- 下游图像模型前置步骤 + +## 3. 输入契约 + +### 输入端口 + +- `in-1 / input / video / bus` + +### 允许输入形态 + +- 上游直接给本地 mp4 路径 +- `artifact.data` 中带视频路径字段 +- `bos://...` 远程视频引用 +- 外置 manifest 中列出多个视频条目 + +### 必要字段 + +如果输入不是直接文件路径,而是结构化对象,则至少需要一个可解析的视频路径字段,例如: + +- `path` +- `videoPath` +- `clipPath` + +## 4. 输出契约 + +### 输出端口 + +- `out-1 / frames / image / bus` +- `out-2 / summary / json / bus` + +### 输出内容 + +`frames` 端口应提供: + +- 每张关键帧图片的本地路径或对象存储路径 +- 对应源视频 +- 帧号或时间戳 + +`summary` 端口应提供: + +- 视频总数 +- 成功抽帧视频数 +- 失败视频数 +- 每个视频的关键帧数量 + +## 5. 超参数 + +- `FRAME_INTERVAL_SECONDS`:按秒抽帧的间隔 +- `MAX_FRAMES_PER_VIDEO`:每个视频最多保留多少帧 +- `UPLOAD_TO_BOS`:是否把关键帧图片上传到对象存储 +- `OUTPUT_PREFIX`:上传时使用的目标前缀 + +## 6. 运行时依赖 + +- ffmpeg +- ffprobe +- 足够的磁盘空间 +- 若启用上传,还需要对象存储配置 + +## 7. 典型编排 + +```mermaid +flowchart LR + A["视频过滤 / 结果筛选"] --> B["自定义关键帧抽取"] + B --> C["图像类模型推理"] + B --> D["结果上传"] +``` + +## 8. skillDefinition 示例 + +```md +接收视频文件路径或包含视频路径的 JSON manifest。若路径为 bos://...,先物化到本地工作目录,再用 ffmpeg 按 FRAME_INTERVAL_SECONDS 抽取关键帧。输出 frames 端口,返回关键帧图片文件路径列表;输出 summary 端口,返回抽帧摘要与失败清单。适合视频预览、轻量图像模型前置处理和审核抽样,不适合替代完整视频切片链路。若输入缺少可解析视频路径则直接失败;若单个视频抽帧失败则记录错误并继续其他视频。 +``` + +## 9. 常见失败 + +- 输入对象没有视频路径字段 +- 远程视频没有先物化到本地 +- ffmpeg 不存在或无法读取视频 +- 抽出的关键帧过多导致磁盘占用超预期 + +## 10. 验收 + +至少检查: + +- 关键帧数量与参数配置一致 +- 每张关键帧图片可打开 +- `summary` 中的成功/失败统计正确 +- 下游图像类算子能直接消费 `frames` 输出 diff --git a/docs/custom-operators/examples/示例-自定义-图像分割算子.md b/docs/custom-operators/examples/示例-自定义-图像分割算子.md new file mode 100644 index 00000000..2c78a1d2 --- /dev/null +++ b/docs/custom-operators/examples/示例-自定义-图像分割算子.md @@ -0,0 +1,102 @@ +# 示例:自定义图像分割算子 + +这个示例对应 `POST /api/operators` 文档里的典型注册对象:“自定义分割算子”。它更接近“已经准备注册到平台”的版本。 + +## 1. 作用 + +该算子接收单张或多张图像,调用分割模型生成 mask、类别信息和结构化摘要,并把分割结果返回给下游。 + +## 2. 适用场景 + +- 图像预处理后的语义分割 +- 机器人或视觉数据标注预辅助 +- 需要把原图进一步裁剪、过滤或统计的链路 + +## 3. 不适用场景 + +- 需要完整训练流程的模型开发 +- 只是做通用命令验证的探索期需求 +- 没有稳定模型镜像和依赖的临时实验 + +## 4. 输入契约 + +### 输入端口 + +- `in-1 / 图像 / image / bus` + +### 输入说明 + +允许: + +- 单张图片路径 +- 多张图片 manifest +- `artifact.data` 中携带图片路径数组 + +如果输入是远程对象: + +- 需要先下载或物化到本地再送给模型 + +## 5. 输出契约 + +### 输出端口 + +- `out-1 / 结果 / json / bus` +- 可选补充:`out-2 / masks / file / bus` + +### 输出说明 + +`结果` 端口建议至少包含: + +- 每张图的 mask 路径 +- 类别列表 +- 置信度 +- 处理耗时 + +若 mask 文件较大: + +- 不要全部内联 JSON +- 应通过路径或对象存储引用返回 + +## 6. 超参数 + +- `MODEL_NAME`:使用哪个分割模型 +- `CONFIDENCE_THRESHOLD`:过滤低置信度结果 +- `RETURN_MASK_IMAGE`:是否输出 mask 图片 +- `UPLOAD_TO_BOS`:是否把 mask 和可视化结果上传 + +## 7. 运行时依赖 + +- 分割模型镜像 +- 模型权重或远程模型服务 +- 可能需要 GPU +- 可能需要较大的显存或内存 + +## 8. 编排建议 + +```mermaid +flowchart LR + A["图像输入 / 关键帧抽取"] --> B["自定义图像分割"] + B --> C["JSON 过滤 / 字段提取"] + B --> D["结果上传"] +``` + +## 9. skillDefinition 示例 + +```md +接收单张图像路径、多图 manifest 或 artifact.data 中的图像路径数组。调用指定分割模型生成 mask、类别和置信度摘要。结果通过 result 端口输出结构化 JSON;当 RETURN_MASK_IMAGE=true 时,同时输出 mask 文件路径或对象存储引用。适合图像类分割、标注预辅助和下游统计链路,不适合训练场景或大规模离线批处理探索。若模型依赖不可用则直接失败;若单张图推理失败则记录错误并继续处理其他图片。 +``` + +## 10. 风险点 + +- 模型依赖重,镜像体积大 +- GPU 资源不足时性能和稳定性都可能出问题 +- 多图批处理容易让输出 JSON 过大 + +## 11. 验收 + +至少检查: + +- 每张图都能得到预期结果或明确错误 +- mask 文件路径真实存在 +- 下游 JSON 类节点可以消费结构化结果 +- 上传开启时,目标对象存储中能看到对应 mask 和可视化产物 diff --git a/docs/custom-operators/examples/示例-自定义-视频窗口采样算子.md b/docs/custom-operators/examples/示例-自定义-视频窗口采样算子.md new file mode 100644 index 00000000..2c74e0f4 --- /dev/null +++ b/docs/custom-operators/examples/示例-自定义-视频窗口采样算子.md @@ -0,0 +1,84 @@ +# 示例:自定义视频窗口采样算子 + +本文档演示一个更偏“命令型”的自定义算子说明方式。它适合从视频或 clip 中按固定窗口抽样若干片段或帧序列。 + +## 1. 作用 + +该算子从上游视频引用中读取文件,按固定窗口长度和间隔做采样,输出新的窗口 clip 清单。 + +## 2. 为什么不用内建视频切片 + +内建 [视频切片](/Users/longtaowu/workspace/droboticflow/docs/operators/视频切片.md) 解决的是标准固定切片问题;本算子则强调“窗口采样”: + +- 可能不是全量切 +- 可能按滑窗或步长抽样 +- 可能只对满足某些规则的视频做部分采样 + +因此更适合作为项目级自定义算子验证。 + +## 3. 输入契约 + +### 输入端口 + +- `in-1 / input / json / bus` + +### 输入要求 + +- 上游 manifest 中必须能解析出视频路径 +- 路径可以是本地文件,也可以是 `bos://...` +- 如果是远程对象,算子必须先物化到本地再交给 ffmpeg + +## 4. 输出契约 + +- `out-1 / sampled / json / bus` + +输出应包含: + +- 每个窗口 clip 的本地路径或远程路径 +- 对应源视频 +- 起止时间 +- 采样策略 + +## 5. 超参数 + +- `WINDOW_SECONDS`:窗口长度 +- `STEP_SECONDS`:窗口步长 +- `MAX_WINDOWS_PER_VIDEO`:每个视频最多保留多少个窗口 +- `UPLOAD_TO_BOS`:是否把采样结果上传 BOS + +## 6. 运行时依赖 + +- ffmpeg +- ffprobe +- 足够的磁盘空间 +- 若启用上传,还需要对象存储配置 + +## 7. 编排建议 + +推荐放在: + +```mermaid +flowchart LR + A["视频过滤"] --> B["自定义视频窗口采样"] + B --> C["结果上传"] +``` + +不建议放在: + +- 没有先做视频筛选的超大规模全量链路 +- 已经可以直接用标准切片满足需求的场景 + +## 8. 风险点 + +- 会产生大量中间 clip,占磁盘 +- 输入是远程对象时,物化成本高 +- 如果幂等设计不好,重复运行会重复生成窗口结果 + +## 9. 验收 + +至少验证: + +- 抽出的窗口数量与规则一致 +- 每个 clip 都可播放 +- 起止时间正确 +- 上传结果与 manifest 一致 diff --git a/docs/custom-operators/instances/README.md b/docs/custom-operators/instances/README.md new file mode 100644 index 00000000..5d765a7a --- /dev/null +++ b/docs/custom-operators/instances/README.md @@ -0,0 +1,16 @@ +# 自定义算子实例分册 + +当前目录专门存放从算子注册表导出的“实例级自定义算子 Markdown”。 + +- 当前实例数量:`0` +- 配套总览:[`../README.md`](../README.md) +- 配套手册:[`../../custom-operator-manual.zh-CN.md`](../../custom-operator-manual.zh-CN.md) + +## 当前状态 + +当前导出的算子目录中没有 `custom` 类型实例,因此本目录暂时只有索引页。 + +建议做法: + +- 先参考上一级目录中的模板与示例撰写自定义算子说明。 +- 当环境中出现真实自定义算子后,再重新运行生成脚本为每个实例导出单独 Markdown。 diff --git a/docs/custom-operators/skillDefinition-编写指南.md b/docs/custom-operators/skillDefinition-编写指南.md new file mode 100644 index 00000000..2a194129 --- /dev/null +++ b/docs/custom-operators/skillDefinition-编写指南.md @@ -0,0 +1,152 @@ +# skillDefinition 编写指南 + +`skillDefinition` 是自定义算子最容易被低估的字段,但它实际上承担三层职责: + +1. 给 AI 工作流生成器理解算子 +2. 给平台维护者理解算子 +3. 给文档脚本提供高密度语义来源 + +如果这个字段写得太短,后面 UI、AI、文档和维护都会变差。 + +## 1. 不要怎么写 + +下面这些写法都不够: + +- “用于处理输入数据。” +- “高效完成推理任务。” +- “支持多种输入格式。” +- “用于业务场景中的批处理。” + +问题在于: + +- 没说输入是什么 +- 没说输出是什么 +- 没说边界是什么 +- 没说失败语义是什么 + +## 2. 推荐结构 + +一份高质量 `skillDefinition`,建议至少包含下面六块信息: + +### 2.1 作用 + +一句话讲清主职责。 + +示例: + +“接收视频 manifest,按固定时间间隔抽取关键帧,并输出关键帧图片列表与摘要 JSON。” + +### 2.2 输入要求 + +要写明: + +- 接受哪个端口 +- 期待的是 `artifact.data`、本地路径还是 `bos://...` +- 必须字段有哪些 + +### 2.3 输出说明 + +要写明: + +- 输出到哪个端口 +- 输出是 JSON、文件路径还是远程引用 +- 下游应该如何消费 + +### 2.4 超参数语义 + +不要只列名字,要说明每个超参数改变什么行为。 + +### 2.5 适用与不适用场景 + +这是避免算子被误用的关键。 + +### 2.6 失败语义 + +至少说明: + +- 参数缺失会怎样 +- 输入为空会怎样 +- 外部依赖失败会怎样 + +## 3. 推荐模板 + +你可以直接按下面结构写: + +```md +该算子用于…… + +输入要求: +- 接收…… +- 优先读取…… +- 若输入为 bos://...,需要先物化…… + +输出说明: +- 输出到…… +- 产出…… +- 下游应按……消费 + +超参数: +- PARAM_A:控制…… +- PARAM_B:控制…… + +适用场景: +- …… + +不适用场景: +- …… + +失败语义: +- 当……时直接失败 +- 当……时跳过并记录错误 +``` + +## 4. 关键原则 + +### 4.1 写“约束”,不是写“愿景” + +好文案不重要,约束最重要。 + +### 4.2 写“平台内怎么跑”,不是只写“业务上想做什么” + +要反映平台真实执行方式: + +- artifact +- stage manifest +- `OUTPUT_DIR` +- PVC / BOS + +### 4.3 写“下游怎么接” + +如果不写下游消费方式,工作流设计者会误连端口或误解输出。 + +## 5. 示例 + +### 差的写法 + +“该算子用于智能抽帧,支持高效处理视频内容。” + +### 好的写法 + +“接收包含视频路径的 JSON manifest。若路径为 `bos://...`,先物化到本地工作目录,再用 ffmpeg 按 `FRAME_INTERVAL_SECONDS` 抽取关键帧。输出 `frames` 端口,返回关键帧图片文件路径列表和 `frames-manifest.json` 摘要。适合在视频筛选后做轻量预览,不适合替代完整视频切片。若输入 manifest 中缺少视频路径则直接失败;若单个视频抽帧失败则记录错误并继续后续视频。” + +## 6. 与 Markdown 文档的关系 + +`skillDefinition` 不是完整文档,但应该能作为完整文档的压缩版语义源。 + +理想关系是: + +- Markdown 文档是完整版 +- `skillDefinition` 是高密度摘要版 + +二者不应互相矛盾。 + +## 7. 验收标准 + +你可以用下面的问题检查一份 `skillDefinition` 是否合格: + +1. 我能不能仅凭它知道输入从哪里来 +2. 我能不能仅凭它知道输出给谁用 +3. 我能不能仅凭它知道什么时候不该用 +4. 我能不能仅凭它知道失败时会发生什么 + +如果四个问题里有两个答不出来,这份 `skillDefinition` 通常还不够好。 diff --git a/docs/custom-operators/templates/自定义算子实现模板.md b/docs/custom-operators/templates/自定义算子实现模板.md new file mode 100644 index 00000000..8ad7f116 --- /dev/null +++ b/docs/custom-operators/templates/自定义算子实现模板.md @@ -0,0 +1,88 @@ +# 自定义算子实现模板 + +## 1. 元数据 + +- 名称: +- `type=custom` +- 类别: +- 镜像: +- 描述: + +## 2. 端口定义 + +### 输入端口 + +- `in-1`: +- `in-2`: + +### 输出端口 + +- `out-1`: +- `out-2`: + +## 3. 超参数 + +| key | 类型 | 默认值 | 说明 | +| --- | --- | --- | --- | +| `EXAMPLE_KEY` | `string` | `""` | 说明 | + +## 4. skillDefinition 草稿 + +这里直接写给平台和 AI 使用的详细 Markdown,至少包括: + +- 作用 +- 输入要求 +- 输出说明 +- 超参数说明 +- 适用场景 +- 不适用场景 +- 失败语义 + +## 5. 实现方式 + +- 命令型还是镜像型: +- 入口脚本: +- 主要依赖: +- 与 runner 的集成方式: + +## 6. 输入解析逻辑 + +写清: + +- 优先读取 `artifact.data` 还是 `artifact.path` +- 是否需要处理 `bos://...` +- 是否需要读取 `dataRef.payloadPath` + +## 7. 输出生成逻辑 + +写清: + +- 结果写入 `OUTPUT_DIR` 的哪些文件 +- 是否会外置 payload +- 是否需要上传 BOS +- 是否需要写库 + +## 8. 错误处理 + +至少说明: + +- 参数缺失时如何报错 +- 输入为空时如何处理 +- 外部依赖失败时如何重试 +- 部分成功时如何暴露结果 + +## 9. 测试 + +至少列出: + +- 单元测试 +- 集成测试 +- 真实工作流回归 + +## 10. 文档 + +上线前应补齐: + +- 一篇独立实例 Markdown +- 一份配置示例 +- 一份验收记录 diff --git a/docs/custom-operators/templates/自定义算子设计模板.md b/docs/custom-operators/templates/自定义算子设计模板.md new file mode 100644 index 00000000..3838ed20 --- /dev/null +++ b/docs/custom-operators/templates/自定义算子设计模板.md @@ -0,0 +1,100 @@ +# 自定义算子设计模板 + +## 1. 基本信息 + +- 算子名称: +- 算子类别: +- 负责人: +- 当前状态:`设计中 | 验证中 | 已上线 | 待内建化` + +## 2. 业务目标 + +回答三个问题: + +1. 这个算子解决什么问题 +2. 为什么现有内建算子不够 +3. 它在整条工作流中的位置是什么 + +## 3. 作用与边界 + +必须写清: + +- 它负责什么 +- 它不负责什么 +- 上游应该在它之前完成什么 +- 下游应该在它之后完成什么 + +## 4. 输入契约 + +逐项写清: + +- 输入端口列表 +- 每个端口的类型和传输方式 +- 输入是 `artifact.data`、本地路径还是远程对象 +- 必要字段 +- 可选字段 +- 空输入或脏输入如何处理 + +建议补一个最小输入示例。 + +## 5. 输出契约 + +逐项写清: + +- 输出端口列表 +- 每个端口的类型和传输方式 +- 输出写在 `data` 还是 `path` +- 是否会产生外部副作用 +- 下游如何消费 + +建议补一个最小输出示例。 + +## 6. 超参数设计 + +对每个超参数逐项说明: + +- key +- 类型 +- 默认值 +- 是否必须填写 +- 对行为的影响 +- 错填后的失败模式 + +## 7. 运行时依赖 + +写清: + +- 需要的镜像 +- 依赖工具或系统命令 +- 是否需要 GPU +- 是否需要外部存储或数据库 +- 是否有磁盘、内存或时长要求 + +## 8. 风险点 + +至少说明: + +- 幂等风险 +- 数据量风险 +- 外部依赖风险 +- 安全风险 +- 可维护性风险 + +## 9. 验收口径 + +至少包含: + +- 平台内要看什么 +- 外部系统要看什么 +- 什么叫成功 +- 什么叫部分成功 +- 什么叫失败 + +## 10. 是否需要内建化 + +判断标准: + +- 是否会被多个项目复用 +- 是否属于平台共性能力 +- 是否需要统一治理 +- 是否已经稳定 diff --git a/docs/custom-operators/templates/自定义算子验收模板.md b/docs/custom-operators/templates/自定义算子验收模板.md new file mode 100644 index 00000000..7bb8606d --- /dev/null +++ b/docs/custom-operators/templates/自定义算子验收模板.md @@ -0,0 +1,75 @@ +# 自定义算子验收模板 + +## 1. 基本信息 + +- 算子名称: +- 版本或镜像: +- 验收日期: +- 验收人: + +## 2. 验收环境 + +- API 地址: +- Worker 镜像: +- Operator 镜像: +- 存储配置: +- 外部依赖: + +## 3. 验收工作流 + +- 工作流名称: +- 工作流 ID: +- 该算子所在节点: +- 上游节点: +- 下游节点: + +## 4. 输入准备 + +说明: + +- 使用了什么输入数据 +- 输入规模多大 +- 是否包含边界情况 +- 是否包含脏数据或空数据 + +## 5. 预期结果 + +写清: + +- 节点应产生哪些 artifact +- 外部系统应产生哪些副作用 +- 下游节点应该收到什么 + +## 6. 实际结果 + +记录: + +- execution 状态 +- 核心日志 +- stage manifest 路径 +- 关键输出文件路径 +- 外部系统核验结果 + +## 7. 异常与排查 + +若有异常,说明: + +- 现象 +- 根因 +- 修复方式 +- 是否复验通过 + +## 8. 结论 + +- `通过` +- `有条件通过` +- `不通过` + +## 9. 附件 + +建议附上: + +- run 链接或 ID +- 产物路径 +- 查询截图 +- diff 或修复记录 diff --git a/docs/custom-operators/百度云-CCR-xspark-自定义算子接入指南.md b/docs/custom-operators/百度云-CCR-xspark-自定义算子接入指南.md new file mode 100644 index 00000000..402402d2 --- /dev/null +++ b/docs/custom-operators/百度云-CCR-xspark-自定义算子接入指南.md @@ -0,0 +1,284 @@ +# 百度云 CCR `xspark` 自定义算子接入指南 + +本文档面向需要把自定义算子镜像推送到百度云 CCR `xspark` 命名空间,并在 Drobodata 工作流平台中注册使用的用户。 + +## 1. 结论 + +当前 `wjzh` 线上环境已经做过真实验证: + +- `xspark` 命名空间私有镜像可以被平台当前的 Kubernetes 拉镜像 secret 正常拉取 +- 自定义算子的 `image` 字段可以直接填写 `ccr-29eug8s3-pub.cnc.bj.baidubce.com/xspark/:` +- 只要镜像内部满足平台运行协议,就可以被工作流真实执行 + +2026-04-23 的实测链路包括: + +- 推送测试镜像:`ccr-29eug8s3-pub.cnc.bj.baidubce.com/xspark/droboticflow-custom-echo:20260423-amd64-v2` +- 创建临时 `custom` 算子 +- 创建单节点 workflow +- 手动触发 run +- run 成功完成,并产出 `json` artifact + +## 2. 镜像地址怎么填 + +在自定义算子的 `image` 字段里填写完整镜像地址: + +```text +ccr-29eug8s3-pub.cnc.bj.baidubce.com/xspark/: +``` + +示例: + +```text +ccr-29eug8s3-pub.cnc.bj.baidubce.com/xspark/droboticflow-custom-echo:20260423-amd64-v2 +``` + +要求: + +- 必须带明确 `tag` +- 不要使用 `latest` +- 不要把镜像名写成包含 `droboticflow-operator-runner` 的字符串 + +最后这一条非常重要。当前运行时会对包含 `droboticflow-operator-runner` 的镜像名做系统 runner 覆盖,因此自定义镜像仓库名、镜像名、tag 都不要带这个子串。 + +## 3. 仓库登录方式 + +当前使用的公网 CCR host: + +```text +ccr-29eug8s3-pub.cnc.bj.baidubce.com +``` + +当前自定义算子约定使用的命名空间: + +```text +xspark +``` + +用户名: + +```text +ccr$xspark +``` + +出于安全要求,仓库文档不明文保存密码。请从管理员或安全渠道获取密码后,在本地环境变量中使用。 + +推荐写法: + +```bash +export CCR_HOST=ccr-29eug8s3-pub.cnc.bj.baidubce.com +export CCR_NAMESPACE=xspark +export CCR_USERNAME='ccr$xspark' +export CCR_PASSWORD='<从安全渠道获取>' + +printf '%s' "$CCR_PASSWORD" | docker login "$CCR_HOST" --username "$CCR_USERNAME" --password-stdin +``` + +## 4. 推送镜像的标准流程 + +### 4.1 最小命令 + +```bash +export CCR_HOST=ccr-29eug8s3-pub.cnc.bj.baidubce.com +export CCR_NAMESPACE=xspark +export CCR_USERNAME='ccr$xspark' +export CCR_PASSWORD='<从安全渠道获取>' +export IMAGE_NAME=my-custom-operator +export IMAGE_TAG=20260423-amd64 + +printf '%s' "$CCR_PASSWORD" | docker login "$CCR_HOST" --username "$CCR_USERNAME" --password-stdin + +docker buildx build \ + --platform linux/amd64 \ + --push \ + -t "$CCR_HOST/$CCR_NAMESPACE/$IMAGE_NAME:$IMAGE_TAG" \ + . +``` + +### 4.2 为什么建议 `linux/amd64` + +当前线上集群与平台镜像口径都是 `linux/amd64`,因此自定义算子也建议统一构建成 `linux/amd64`,避免出现镜像能推上去、但节点调度时拉不到对应架构层的问题。 + +## 5. 平台里的使用流程 + +1. 本地构建并推送镜像到 `xspark` +2. 在平台创建一个 `type=custom` 的算子 +3. 在算子配置中把 `image` 字段填写为完整 CCR 地址 +4. 在工作流里使用该算子 +5. 运行 workflow,平台会直接拉取这个镜像启动 Kubernetes Job + +平台不会做的事情: + +- 不会帮你构建镜像 +- 不会帮你推送镜像 +- 不会自动补全 tag +- 不会把代码打包成镜像再替你上传 + +## 6. 自定义镜像必须满足的运行协议 + +“能拉到镜像”不等于“能执行成功”。自定义镜像至少要满足下面三个条件: + +1. 写出 `RESULT_OUTPUT_PATH` +2. 写出 `STAGE_MANIFEST_PATH` +3. 向 stdout 打印一行: + +```text +DROBOTICFLOW_RESULT_PATH= +``` + +其中: + +- `RESULT_OUTPUT_PATH` 是 worker 读取结构化执行结果的主文件 +- `STAGE_MANIFEST_PATH` 是阶段摘要文件。即使没有 `dataRefs`,也要至少写成 `{"dataRefs":[]}` +- 如果缺少 `STAGE_MANIFEST_PATH`,run 最后会被判定为失败 + +当前最小结果 JSON 结构如下: + +```json +{ + "logs": [ + { + "timestamp": "2026-04-23T07:14:52Z", + "level": "INFO", + "message": "xspark custom operator completed successfully", + "operator": "xspark 自定义测试" + } + ], + "artifacts": [ + { + "name": "xspark-result.json", + "type": "json", + "path": "/workspace/runs//nodes//outputs/xspark-result.json", + "portId": "out-1", + "portName": "result", + "data": { + "message": "xspark custom operator ok" + } + } + ], + "resourceSeries": [] +} +``` + +## 7. 最小可用示例 + +### 7.1 Dockerfile + +```dockerfile +FROM busybox:1.36 +WORKDIR /app +COPY run.sh /app/run.sh +RUN chmod +x /app/run.sh +CMD ["/app/run.sh"] +``` + +### 7.2 `run.sh` + +```sh +#!/bin/sh +set -eu + +now="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +mkdir -p "${OUTPUT_DIR:?}" + +artifact_path="${OUTPUT_DIR}/xspark-result.json" + +cat > "${artifact_path}" < "${STAGE_MANIFEST_PATH:?}" < "${RESULT_OUTPUT_PATH:?}" </nodes//stage-manifest.json: no such file or directory +``` + +### 9.2 镜像地址填了但实际被系统 runner 覆盖 + +常见根因: + +- 镜像名里包含 `droboticflow-operator-runner` + +解决方式: + +- 换一个不包含该子串的仓库名或镜像名 + +### 9.3 镜像推上去了,但集群拉不到 + +检查项: + +- 镜像是否真的在 `xspark` 命名空间 +- tag 是否拼写正确 +- 是否构建成了集群可运行的架构,建议 `linux/amd64` +- 当前环境的 image pull secret 是否能访问该仓库 + +## 10. 推荐的用户操作规范 + +- 每次改镜像都发新 tag,不覆盖旧 tag +- 镜像名按算子职责命名,不要用 `test-final-v3` +- 文档里明确输入、输出、超参数和失败模式 +- 每个正式自定义算子至少保留一份单独 Markdown + +## 11. 相关文档 + +- [自定义算子总手册](/Users/longtaowu/workspace/droboticflow/docs/custom-operator-manual.zh-CN.md) +- [自定义算子 Markdown 体系索引](/Users/longtaowu/workspace/droboticflow/docs/custom-operators/README.md) +- [算子开发文档](/Users/longtaowu/workspace/droboticflow/docs/operator-authoring.md) diff --git a/docs/deployment.md b/docs/deployment.md index 2d85f742..1aaf2fb5 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -42,16 +42,10 @@ WORKER_IMAGE=... OPERATOR_RUNNER_IMAGE=... ``` -If the target cluster cannot pull `mongo:7` directly from Docker Hub, mirror it to the same Baidu registry: +You can also load a named deployment preset: ```bash -docker buildx build \ - --platform linux/amd64 \ - -t ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-mongo:7-amd64 \ - --push \ - - <<'EOF' -FROM mongo:7 -EOF +DEPLOY_ENV=wjzh IMAGE_TAG=20260421 ./scripts/build-and-push-images.sh ``` ## 3. Prepare Kubernetes Access @@ -65,7 +59,7 @@ export K8S_NAMESPACE=droboticflow export API_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-api:20260408 export WORKER_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-worker:20260408 export OPERATOR_RUNNER_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:20260408 -export MONGO_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-mongo:7-amd64 +export MONGO_URI=mongodb://:@:27017/ export REGISTRY_SERVER=ccr-29eug8s3-pub.cnc.bj.baidubce.com export REGISTRY_USERNAME= export REGISTRY_PASSWORD= @@ -75,25 +69,40 @@ export AI_MODEL=glm-5 export AI_API_KEY= ``` +For the `wjzh` environment, the repo now carries non-secret defaults in [`scripts/deploy-envs/wjzh.env`](/Users/longtaowu/workspace/droboticflow/scripts/deploy-envs/wjzh.env). Sensitive values stay out of git and can be stored in either: + +- `$HOME/.config/droboticflow/deploy/wjzh.secrets.env` +- `.env.wjzh.local` + +The loader also looks for `$HOME/.config/droboticflow/kubeconfigs/wjzh.config` and exports `KUBECONFIG` automatically when `DEPLOY_ENV=wjzh` is set. + ## 4. Deploy ```bash ./scripts/deploy-k8s.sh ``` +Or, with the reusable preset: + +```bash +DEPLOY_ENV=wjzh ./scripts/deploy-k8s.sh +``` + The deploy script performs these actions: 1. creates the namespace if it does not exist 2. creates or updates the image pull secret -3. creates or updates MongoDB credentials, reusing the existing Secret values when present -4. creates or updates the `droboticflow-ai-secret` Secret for server-side AI generation -5. renders `k8s/droboticflow.template.yaml` -6. applies the manifest to the cluster -7. waits for MongoDB, API, and worker rollouts +3. loads `MONGO_URI` from the current cluster secret when omitted, then validates that it points to an external MongoDB instance +4. creates or updates the `droboticflow-mongo-secret` Secret with `MONGO_URI` +5. creates or updates the `droboticflow-ai-secret` Secret for server-side AI generation +6. renders `k8s/droboticflow.template.yaml` +7. applies the manifest to the cluster +8. waits for API and worker rollouts + +The main deployment path no longer creates a platform MongoDB Pod or Service. You must provision the backing MongoDB outside this manifest and provide its URI explicitly. ## 5. Services Created -- `mongodb`: ClusterIP service for MongoDB - `droboticflow-api`: NodePort service that exposes the frontend and API - `droboticflow-worker`: ClusterIP service for worker health/debugging @@ -125,13 +134,11 @@ http://:30080 - `API_IMAGE` - `WORKER_IMAGE` - `OPERATOR_RUNNER_IMAGE` -- `MONGO_IMAGE` ## 7. Notes -- The provided manifest uses a single-replica MongoDB Deployment with `emptyDir` storage so it can run on clusters that do not expose a default `StorageClass`. -- MongoDB should be mirrored to the Baidu registry as an `amd64` image when the cluster cannot pull from Docker Hub. -- For production, replace the `emptyDir` volume with a PVC-backed deployment or StatefulSet. +- The main manifest only deploys the application components. It does not create MongoDB resources in-cluster. +- `MONGO_URI` must reference an external MongoDB instance; localhost, `mongodb:27017`, and in-cluster `.svc` addresses are rejected by `scripts/deploy-k8s.sh`. - API and worker entrypoints are Go binaries with Mongo startup retry, so they tolerate the database becoming ready after the Deployments start. - The worker runs with Kubernetes RBAC that allows creating and watching Jobs in the namespace. - The operator-runner Job template inherits the private registry pull secret name from `K8S_IMAGE_PULL_SECRET_NAME`. diff --git a/docs/external-business-db-write-api-integration.md b/docs/external-business-db-write-api-integration.md new file mode 100644 index 00000000..5558fc25 --- /dev/null +++ b/docs/external-business-db-write-api-integration.md @@ -0,0 +1,193 @@ +# External Business DB Write API Integration Guide + +This document is the handoff guide for internal teams and third-party integrators who need to write data into a preconfigured business MongoDB connection through Drobodata. + +## 1. Integration Target + +- Base URL: `http://114.111.24.223:30080` +- Auth header: `X-API-Key` +- Allowed connectionId: `mongo-6fc71b9f604165b38279f068c4294423` +- Allowed collection whitelist: + - `businessWriteValidation` + +Current API key: + +```text +dbw_ +``` + +Do not keep the live plaintext key in git, wiki pages, or chat history. Distribute it out of band to the third party. + +If this key is rotated later, the old key immediately becomes invalid. + +## 2. Supported Operations + +Third parties can call these routes: + +- `POST /api/external/business-db/documents` +- `PATCH /api/external/business-db/documents/:documentId` +- `POST /api/external/business-db/upserts` + +All requests must include: + +```http +Content-Type: application/json +X-API-Key: dbw_... +``` + +## 3. Request Rules + +- `connectionId` must be the authorized one +- `collection` must be in the token whitelist +- `document`, `patch`, `match`, and `setOnInsert` must be plain JSON objects +- Mongo update operators such as `$set`, `$inc`, `$push` are not accepted + +## 4. Create Document + +Request: + +```bash +curl -X POST 'http://114.111.24.223:30080/api/external/business-db/documents' \ + -H 'Content-Type: application/json' \ + -H 'X-API-Key: dbw_' \ + --data '{ + "connectionId": "mongo-6fc71b9f604165b38279f068c4294423", + "collection": "businessWriteValidation", + "document": { + "runLabel": "partner-demo-001", + "source": "partner-a", + "value": 123, + "createdAt": "2026-04-14T17:20:00+08:00" + } + }' +``` + +Typical success response: + +```json +{ + "connectionId": "mongo-6fc71b9f604165b38279f068c4294423", + "collection": "businessWriteValidation", + "operation": "created", + "document": { + "_id": { + "$oid": "69de3ab85edc4783ca5ce18b" + }, + "runLabel": "partner-demo-001", + "source": "partner-a", + "value": 123, + "createdAt": "2026-04-14T17:20:00+08:00" + } +} +``` + +## 5. Patch Existing Document By _id + +Request: + +```bash +curl -X PATCH 'http://114.111.24.223:30080/api/external/business-db/documents/' \ + -H 'Content-Type: application/json' \ + -H 'X-API-Key: dbw_' \ + --data '{ + "connectionId": "mongo-6fc71b9f604165b38279f068c4294423", + "collection": "businessWriteValidation", + "patch": { + "value": 789, + "status": "done" + } + }' +``` + +Typical success response: + +```json +{ + "connectionId": "mongo-6fc71b9f604165b38279f068c4294423", + "collection": "businessWriteValidation", + "operation": "updated", + "document": { + "_id": { + "$oid": "69de3ab85edc4783ca5ce18b" + }, + "runLabel": "partner-demo-001", + "source": "partner-a", + "value": 789, + "status": "done" + } +} +``` + +## 6. Upsert By Business Key + +Use this when the third party wants "insert if not exists, otherwise update". + +Request: + +```bash +curl -X POST 'http://114.111.24.223:30080/api/external/business-db/upserts' \ + -H 'Content-Type: application/json' \ + -H 'X-API-Key: dbw_' \ + --data '{ + "connectionId": "mongo-6fc71b9f604165b38279f068c4294423", + "collection": "businessWriteValidation", + "match": { + "runLabel": "partner-demo-001" + }, + "patch": { + "source": "partner-a", + "value": 456, + "updatedAt": "2026-04-14T17:21:00+08:00" + }, + "setOnInsert": { + "createdBy": "partner-a" + } + }' +``` + +Typical success response: + +- Insert path: `operation` is `created` +- Update path: `operation` is `updated` + +## 7. Error Cases + +Common failures: + +- `401 Unauthorized` + - Missing `X-API-Key` + - Invalid API key + - Rotated or disabled API key +- `403 Forbidden` + - `connectionId` not allowed for this token + - `collection` not allowed for this token +- `400 Bad Request` + - Invalid JSON + - Missing required fields + - `document` / `patch` / `match` not plain JSON objects + +## 8. Verified Live Test On April 14, 2026 + +This configuration was tested against the live remote Kubernetes deployment on April 14, 2026. + +Verified successfully: + +- create document +- upsert document +- patch document by `_id` + +Test collection: + +- `businessWriteValidation` + +Sample live test markers: + +- `external-api-test-202604141712` +- `external-api-upsert-202604141712` + +## 9. Operational Notes + +- This API does not expose raw MongoDB credentials to third parties +- A token should be scoped to the smallest possible set of collections +- If a third party no longer needs access, disable or delete the token +- If a key is leaked, rotate it immediately diff --git a/docs/feature-overview-playwright-validation-2026-04-15.html b/docs/feature-overview-playwright-validation-2026-04-15.html new file mode 100644 index 00000000..e48e91ab --- /dev/null +++ b/docs/feature-overview-playwright-validation-2026-04-15.html @@ -0,0 +1,662 @@ + + + + + + Drobodata 功能介绍与线上验收报告 + + + +
+
+
+
Drobodata Platform Validation
+

Drobodata 功能介绍与线上验收报告

+

+ 本页基于 2026-04-15 对 Kubernetes 部署环境 + http://114.111.24.223:30080 + 的真实验收结果生成。内容同时包含功能说明、Playwright 回归范围、页面截图和一份可回放的录屏。 +

+ +
+
+ 活跃项目 + 11 +
+
+ 已注册算子 + 26 +
+
+ 当日 Run + 33 +
+
+ 通过结论 + 核心链路通过 +
+
+
+
+ +
+
+
+
+

报告摘要

+

+ 当前线上环境已经具备作为演示和持续开发基础的完整主链路:项目创建、工作流编排、BOS 扫描、Kubernetes Job + 执行、人工审核、执行监控、模板沉淀与复用都已被真实跑通。 +

+ +
+
+
环境快照
+

+ BOS 连接:2
+ 业务 Mongo 连接:4
+ 模板:6
+ 源数据集合:0
+ 成品包:0
+ 抽检任务:0 +

+

+ 线上环境仍保留 k8s.enabled 与“启用 K8s 调度”开关,尚未完全同步到当前仓库的 Kubernetes-only 设计。 +

+
+
+
+ +
+

一级功能模块

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
模块入口核心能力验证结果
仪表盘/聚合项目、算子、作业和审核概览通过
项目管理/projects创建项目并承载工作流资产通过
工作流编辑器/workflow/:idDAG 编排、保存、运行、边产物查看通过
算子管理/operators内置算子浏览与自定义算子 CRUD通过
工作流模板/templates模板导出、搜索、复用通过
执行监控/monitoringRun 列表、日志、资源、产出数据通过
任务审核/audit人工审核节点的 approve / reject通过
业务数据库/databaseMongo 连接、集合、索引和查询通过
对象存储/object-storageBOS 连接、目录浏览、对象操作通过
数据资产 / 抽检/data-assets / /sampling结果查看与质量抽检入口页面可用,当前环境无样例数据
用户与系统配置/users / /settings角色管理、K8s/Worker/时区/AI 配置通过
+
+ +
+

工作流主链路

+
+
+
场景 A
+

BOS 增量扫描工作流

+

+ 实测添加 BOS 增量扫描 算子,配置连接、Bucket 和 Prefix,收藏常用路径,触发运行并验证增量状态写入与重置。 +

+

+ 结论:对象存储接入、收藏路径和 checkpoint 状态管理已经形成闭环。 +

+
+
+
场景 B
+

普通处理工作流

+

+ 实测 真实数据源 -> 人工审核 -> 结果上传 + 链路,从编辑器直接启动 Run,经审核页处理后完成下游推进。 +

+

+ 结论:编辑器、Run、审核、监控之间的数据流和控制流是连通的。 +

+
+
+
+ +
+

Playwright 覆盖范围

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
验证项动作结果
系统配置保存修改轮询间隔并保存,再改回原值通过
算子 CRUD创建、编辑、搜索、删除自定义算子通过
BOS 扫描工作流创建、配置、收藏路径、运行、重置状态通过
普通工作流执行编辑器内发起运行并停留当前页面通过
人工审核分别验证 approve 与 reject通过
执行监控搜索 Run,打开日志、资源和产出数据通过
边输出面板验证 running / completed / failed 三种状态通过
模板复用导出模板、搜索模板、回填到项目通过
用户管理创建、编辑、删除用户通过
+

+ 执行命令: + BASE_URL=http://114.111.24.223:30080 node scripts/playwright-functional-check.mjs +

+
+ +
+

关键截图

+ +
+ +
+

演示视频

+
+ +
+

+ 如果当前环境无法直接内嵌播放,请改用 + 独立打开视频文件。 +

+
+ +
+

当前建议

+
+
+
部署一致性
+

同步 Kubernetes-only 配置语义

+

+ 建议下一次发布时移除线上设置页中的 k8s.enabled 旧语义,避免用户误解为仍支持本地模式切换。 +

+
+
+
演示完整度
+

补齐数据资产与抽检样例

+

+ 当前环境的源数据、成品数据和抽检任务为空。补一套最小演示数据后,数据资产模块的说明会更完整。 +

+
+
+
+ + +
+
+ + diff --git a/docs/feature-overview-playwright-validation-2026-04-15.md b/docs/feature-overview-playwright-validation-2026-04-15.md new file mode 100644 index 00000000..90d08ad3 --- /dev/null +++ b/docs/feature-overview-playwright-validation-2026-04-15.md @@ -0,0 +1,411 @@ +# Drobodata 功能介绍与线上验收报告 + +测试时间:2026-04-15 +测试环境:`http://114.111.24.223:30080` +测试方式:真实线上环境 API 采样 + Playwright 端到端自动化 +测试结论:核心功能链路已完成两轮完整回归,主流程全部通过 + +## 1. 报告范围 + +这份报告覆盖两层信息: + +1. 当前线上环境实际提供了哪些功能,以及页面上呈现成什么状态。 +2. 我基于 Playwright 对真实环境做了哪些端到端验证,哪些能力已经被实测打通。 + +需要单独说明的一点是:当前线上部署版本和当前仓库主工作区代码并不完全一致。最明显的差异是: + +- 当前仓库已经朝“仅 Kubernetes Job 模式”收敛。 +- 当前线上环境的系统配置页和 `/api/settings` 仍然保留了 `k8s.enabled` 语义,页面上也还能看到“启用 K8s 调度”开关。 + +因此,下面的结论都以“线上环境当前真实表现”为准,同时会在必要处标出与当前仓库代码语义的差异。 + +## 2. 产品能力总览 + +从前端路由、侧边导航、页面实现和线上行为来看,当前系统可以拆成 13 个一级功能模块。 + +| 模块 | 入口 | 核心作用 | 本次验证结果 | +| --- | --- | --- | --- | +| 仪表盘 | `/` | 汇总项目数、算子数、当日 Job 执行数、成功率、审核待办 | 通过 | +| 项目管理 | `/projects` | 创建项目、搜索项目、进入项目详情、承载工作流列表 | 通过 | +| 项目详情 | `/project/:id` | 新建工作流、从项目保存模板、查看项目下工作流 | 通过 | +| 工作流编辑器 | `/workflow/:id` | 设计 DAG、配置算子、保存、运行、导出模板、查看边产物 | 通过 | +| 算子管理 | `/operators` | 浏览内置算子、注册自定义算子、编辑、删除 | 通过 | +| 工作流模板 | `/templates` | 浏览模板、搜索模板、从模板创建工作流 | 通过 | +| 执行监控 | `/monitoring` | 查看运行状态、过滤搜索、查看日志/资源/产出数据 | 通过 | +| 任务审核 | `/audit` | 处理人工审核节点产生的待审核任务,支持通过/拒绝 | 通过 | +| 数据资产 | `/data-assets` | 汇总源数据和成品数据,按目录与文件检索 | 页面可用,当前环境暂无数据 | +| 业务数据库 | `/database` | 管理业务 MongoDB 连接、集合、索引与查询 | 通过 | +| 对象存储 | `/object-storage` | 管理 BOS 连接、浏览 bucket/prefix/object、写入对象 | 通过 | +| 抽检任务 | `/sampling` | 管理抽样质检任务与状态 | 页面可用,当前环境暂无任务 | +| 用户管理 | `/users` | 创建、编辑、删除系统用户并分配角色 | 通过 | +| 系统配置 | `/settings` | 配置 K8s 集群、Worker 默认资源、时区和 AI 服务 | 通过 | + +## 3. 功能点详细分析 + +### 3.1 平台总览与权限入口 + +平台采用左侧导航 + 右侧业务视图的标准控制台布局。不同角色看到的导航项不同: + +- `admin` 可见全部模块。 +- `operator` 主要覆盖项目、工作流、模板、执行监控、抽检。 +- `auditor` 主要覆盖仪表盘、数据资产、审核、执行监控。 + +这意味着产品的职责边界已经比较清晰: + +- 管理侧:系统配置、用户、对象存储、业务数据库。 +- 生产侧:项目、工作流、算子、模板、执行监控。 +- 审核侧:人工审核、抽检、数据资产查看。 + +![仪表盘截图](../output/playwright/01-dashboard.png) + +图 1:线上仪表盘,展示活跃项目、已注册算子和当日 Job 执行数。 + +### 3.2 项目与工作流管理 + +项目是工作流的业务容器。围绕项目,系统提供了完整的工作流生命周期入口: + +- 创建项目。 +- 进入项目详情页。 +- 在项目下创建普通工作流。 +- 从工作流导出模板。 +- 在项目详情页直接“保存为模板”。 +- 使用模板回填到项目,生成新的工作流。 + +本次自动化验证实际打通了两条路径: + +1. 在项目详情页直接新建工作流并进入编辑器。 +2. 先从工作流导出模板,再从模板页复用回项目。 + +![项目详情截图](../output/playwright/09-project-detail.png) + +图 2:项目详情页是工作流创建、模板沉淀和项目级查看的核心入口。 + +### 3.3 工作流编辑器与算子编排 + +工作流编辑器是整个平台最核心的能力,当前已经具备以下关键功能: + +- 节点式 DAG 编排。 +- 左侧算子库搜索与拖入画布。 +- 节点双击打开配置面板。 +- 工作流级配置切换手动执行 / 周期执行。 +- 保存工作流。 +- 直接从编辑器发起运行。 +- 不离开编辑器观察当前执行态。 +- 查看连线输出面板。 +- 导出为模板。 + +本次验证覆盖了两个关键工作流场景: + +#### 场景 A:BOS 增量扫描工作流 + +验证内容: + +- 在编辑器中添加 `BOS 增量扫描` 算子。 +- 配置 BOS 连接、Bucket、Prefix。 +- 收藏常用路径。 +- 刷新后验证“常用路径”是否持久存在。 +- 手动触发运行。 +- 验证源状态是否写入。 +- 验证“重置当前状态”是否生效。 + +这个场景证明当前系统的“对象存储扫描 + 增量状态管理 + 收藏路径”是打通的。 + +![BOS 扫描工作流截图](../output/playwright/10-workflow-editor-scan.png) + +图 3:基于 BOS 的扫描工作流编辑界面。 + +#### 场景 B:普通处理工作流 + +验证内容: + +- 用 API 写入三节点链路:`真实数据源 -> 人工审核 -> 结果上传`。 +- 在前端编辑器修改源节点超参数。 +- 在编辑器直接点击“运行工作流”。 +- 保持停留在编辑器,不强制跳转。 +- 运行后进入执行监控。 +- 自动发现并处理人工审核任务。 +- 验证 Run 最终完成。 + +这个场景证明了“编辑器配置 -> 发起运行 -> 审核阻塞 -> 审核放行 -> 下游继续执行”的完整主链路是可用的。 + +![普通工作流编辑器截图](../output/playwright/11-workflow-editor.png) + +图 4:普通工作流编辑页面,支持运行、保存和工作流配置切换。 + +### 3.4 算子体系 + +算子管理页当前既承担“算子资产浏览”,也承担“平台注册新算子”的职责。页面可见的内置算子覆盖了典型数据生产链: + +- `MongoDB 写入` +- `BOS 增量扫描` +- `抽检采样` +- `成品打包` +- `视频标注` +- `视频切片` + +Playwright 实测覆盖了: + +- 新建一个自定义算子。 +- 编辑描述信息。 +- 搜索该算子。 +- 删除该算子。 + +因此这部分不只是静态展示,后台 CRUD 能力已被实际验证。 + +![算子管理截图](../output/playwright/03-operators.png) + +图 5:算子管理页,内置算子和自定义算子共用同一管理入口。 + +### 3.5 执行监控与运行可观测性 + +执行监控页负责聚合所有 Run 的执行历史。当前已验证的能力包括: + +- 以工作流名称检索 Run。 +- 查看状态、开始时间、耗时。 +- 打开“执行日志”弹窗。 +- 打开“资源监控”弹窗。 +- 打开“产出数据”弹窗。 + +此外,我还专门验证了工作流编辑器里的“边输出面板”三种状态: + +- 上游已完成、下游运行中。 +- 整体执行成功。 +- 下游执行失败。 + +这说明当前系统不只是有 Run 列表,还能把连线级产物与节点级状态串起来。 + +![执行监控截图](../output/playwright/12-monitoring.png) + +图 6:执行监控页已经可以按工作流检索,并展示运行状态与执行历史。 + +### 3.6 人工审核 + +人工审核是当前工作流里最重要的“人工介入点”。我验证了两种结论分支: + +- 审核通过。 +- 审核拒绝。 + +并确认: + +- 两类任务都会在审核页出现。 +- 点击后可完成处理。 +- 处理后对应 Run 会继续推进并落到完成态。 + +这意味着当前的审核设计不是“纯展示”,而是真正挂在工作流推进链路中的控制点。 + +![任务审核截图](../output/playwright/15-audit.png) + +图 7:任务审核页能同时展示待审核卡片和右侧审核工作区。 + +### 3.7 模板复用 + +模板能力已经形成闭环: + +- 从工作流编辑器导出模板。 +- 在模板列表页搜索模板。 +- 从模板选择目标项目并生成新工作流。 +- 在项目详情页直接把已有工作流保存为模板。 + +这类能力对标准化生产非常关键,因为它决定了流程是否能沉淀成可复用资产,而不是每次都重新搭。 + +![模板页截图](../output/playwright/13-templates.png) + +图 8:模板列表页支持搜索、进入复用流程和继续编辑。 + +### 3.8 业务数据库 + +业务数据库页面向的是“用户自定义的业务 MongoDB 连接”。当前页面已经能做以下事情: + +- 切换 Mongo 连接。 +- 测试连接。 +- 查看数据库名。 +- 管理集合。 +- 运行 JSON 查询。 +- 查看与创建索引。 +- 管理第三方写入 API Token。 + +这块能力说明平台并不只关心文件和对象存储,也具备把结构化结果回写到业务数据库的承接面。 + +![业务数据库截图](../output/playwright/04-database-browser.png) + +图 9:业务数据库页支持连接管理、查询、集合操作和索引维护。 + +### 3.9 对象存储 + +对象存储页是 BOS 连接的运维入口,当前页面功能包括: + +- 管理 BOS 连接。 +- 绑定 bucket。 +- 浏览目录和对象列表。 +- 预览对象。 +- 创建新的文本对象。 + +本次测试没有只停留在页面展示,而是把对象存储能力和工作流里的 `BOS 增量扫描` 算子连起来验证了: + +- 对象存储连接能在工作流节点中选择。 +- Bucket/Prefix 能被算子消费。 +- 收藏路径能被重新回填。 + +因此它是“工作流输入源”的一部分,不是孤立的资源浏览器。 + +![对象存储截图](../output/playwright/07-object-storage.png) + +图 10:对象存储页已经具备连接、目录浏览和对象操作能力。 + +### 3.10 数据资产与抽检 + +数据资产和抽检属于“偏结果查看”的功能层。 + +当前线上环境里: + +- `/api/source-data/collections` 返回空。 +- `/api/packages` 返回空。 +- `/api/sampling-tasks` 返回空。 + +因此本次能确认的是: + +- 页面本身可以正常进入。 +- 标签切换和搜索框渲染正常。 +- 当前测试环境还没有挂载出可展示的源数据、成品数据和抽检任务。 + +也就是说,这两部分目前更像“功能入口已就位,但线上样例数据为空”。 + +![数据资产截图](../output/playwright/05-data-assets-source.png) + +图 11:数据资产页可正常加载,但当前环境下源数据和成品数据都为空。 + +### 3.11 用户与系统配置 + +用户管理页本次实测通过了: + +- 创建用户。 +- 修改角色。 +- 删除用户。 + +系统配置页本次实测通过了: + +- 读取配置。 +- 修改 Worker 轮询间隔。 +- 提交保存。 +- 再次改回原值并保存。 + +从当前线上截图可见,配置页仍包含: + +- K8s 集群 `endpoint / namespace / kubeconfig` +- Worker 轮询和默认资源 +- 时区 +- 以及“启用 K8s 调度”开关 + +这里再次提醒:这个开关属于线上部署和当前代码分支之间的差异点。 + +![系统配置截图](../output/playwright/02-settings.png) + +图 12:线上配置页仍然保留 `启用 K8s 调度` 开关,说明部署版本尚未完全同步到最新的 Kubernetes-only 设计。 + +## 4. Playwright 端到端验证范围 + +本次不是只点了一遍页面,而是做了真实的跨模块自动化回归。以下链路都已经被脚本验证通过,而且我连续执行了两轮,结果一致: + +| 验证项 | 具体动作 | 结果 | +| --- | --- | --- | +| 首次进入系统 | 检查是否出现 bootstrap 用户创建页 | 通过 | +| 系统配置保存 | 修改轮询间隔并保存,再改回原值 | 通过 | +| 算子 CRUD | 创建、编辑、搜索、删除自定义算子 | 通过 | +| 业务数据库 | 进入页面、切换连接、验证集合与索引区域 | 通过 | +| 数据资产 | 进入页面,源数据/成品数据标签加载 | 通过 | +| 对象存储 | 进入页面,连接与目录渲染 | 通过 | +| 抽检任务 | 进入页面并完成基础渲染校验 | 通过 | +| 旧路由兼容 | `/scan-configs` 自动回跳 `/projects` | 通过 | +| 项目创建 | 创建项目并进入详情页 | 通过 | +| BOS 扫描工作流 | 新建、配置、收藏路径、保存 | 通过 | +| BOS 增量状态 | 运行后写入状态,再重置状态 | 通过 | +| 普通工作流执行 | 在编辑器中运行,不跳离页面 | 通过 | +| Run 监控 | 搜索 Run,查看日志/资源/产出数据弹窗 | 通过 | +| 人工审核 | 自动发现任务并完成 approve | 通过 | +| 连线输出面板 | 覆盖 running / completed / failed 三种状态 | 通过 | +| 模板导出 | 从工作流导出模板 | 通过 | +| 模板复用 | 从模板选择项目并生成工作流 | 通过 | +| 项目保存模板 | 在项目详情页保存为模板 | 通过 | +| 用户管理 | 创建、编辑、删除用户 | 通过 | +| 审核通过/拒绝 | 分别触发 approve / reject 两种流转 | 通过 | + +执行命令如下: + +```bash +BASE_URL=http://114.111.24.223:30080 node scripts/playwright-functional-check.mjs +``` + +脚本文件位置: + +- [scripts/playwright-functional-check.mjs](/Users/longtaowu/workspace/droboticflow/scripts/playwright-functional-check.mjs) + +产物目录: + +- [output/playwright](/Users/longtaowu/workspace/droboticflow/output/playwright) + +## 5. 线上环境快照 + +测试结束后对线上 API 的抽样结果如下: + +- 活跃项目:`11` +- 已注册算子:`26` +- 当日执行 Run:`33` +- 审核待办:`0` +- BOS 连接:`2` +- 业务 Mongo 连接:`4` +- 模板:`6` +- 源数据集合:`0` +- 成品包:`0` +- 抽检任务:`0` + +这说明当前线上环境更像是“流程验证和能力展示环境”: + +- 主业务主链路是通的。 +- 管理侧连接资源是存在的。 +- 数据资产和抽检结果数据还没有铺满。 + +## 6. 演示视频 + +Playwright 录制的视频已经生成: + +- [playwright-functional-check.webm](/Users/longtaowu/workspace/droboticflow/output/playwright/playwright-functional-check.webm) + +如果当前查看器支持内嵌视频,可直接播放: + + + +## 7. 本次发现的差异与建议 + +### 7.1 线上部署版本仍残留旧配置语义 + +当前线上环境里: + +- 设置页还显示“启用 K8s 调度”。 +- `/api/settings` 仍有 `k8s.enabled` 字段。 + +而当前仓库工作区已经朝“仅 Kubernetes Job 模式”收敛。建议下一次部署时同步这部分前后端版本,避免用户误以为还支持可切换的本地模式。 + +### 7.2 数据资产、成品数据、抽检页面缺少展示数据 + +页面本身可进入,但线上环境当前没有对应业务样本。建议补一套最小演示数据: + +- 一个源数据目录。 +- 一个成品包。 +- 一个抽检任务。 + +这样文档、演示和验收会更完整。 + +- + +## 8. 结论 + +如果从“今天这个线上环境能不能作为一个可演示、可验证、可继续开发的 Kubernetes Job 工作流平台”来判断,答案是可以。 + +当前已经实测通过的核心闭环是: + +1. 配置 K8s / 对象存储 / 业务数据库等基础资源。 +2. 创建项目与工作流。 +3. 使用 BOS 扫描类算子和普通处理类算子构建流程。 +4. 在编辑器内发起运行。 +5. 通过执行监控与审核页推进流程。 +6. 将流程沉淀为模板复用。 diff --git a/docs/operations.md b/docs/operations.md index 788d9006..b26f8880 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -12,18 +12,23 @@ ### Worker - polls MongoDB for queued runs -- auto-enqueues streaming workflows based on `schedule.interval` +- auto-enqueues streaming workflows based on workflow-level schedule modes: + - `schedule.interval` + - `schedule.intervalHours` + - `schedule.dayInterval + schedule.timeOfDay` - executes workflow nodes in topological order - pauses runs on human-review nodes - resumes approved or rejected audit branches -- creates Kubernetes Jobs when `K8S_ENABLED=true` +- creates Kubernetes Jobs for every operator execution +- writes node input manifests into the shared workspace PVC before launching Jobs - runs as a single Go binary ### Operator Runner - receives `RUN_ID`, `WORKFLOW_ID`, `NODE_ID`, and `NODE_LABEL` - executes a generic Python operator contract -- emits logs for the worker to associate with the run +- reads input manifests from the shared workspace PVC +- writes `result.json` and `stage-manifest.json` back to the shared workspace PVC ## MongoDB Collections @@ -67,6 +72,16 @@ Smoke test the API and worker: ./scripts/smoke-test.sh ``` +Run the production-style WJZH real-data regression: + +```bash +API_BASE_URL=http://:30080 \ +STORAGE_CONNECTION_ID= \ +python3 scripts/wjzh_operator_regression.py --preflight-only +``` + +See the full runbook in [`docs/wjzh-operator-regression.md`](/Users/longtaowu/workspace/droboticflow/docs/wjzh-operator-regression.md). + Reconcile persistent system data and remove legacy demo records: ```bash @@ -91,6 +106,12 @@ kubectl -n droboticflow logs deploy/droboticflow-worker kubectl -n droboticflow logs job/ ``` +Check the shared workspace: + +```bash +kubectl -n droboticflow exec deploy/droboticflow-worker -- ls -R /workspace/runs | head -200 +``` + ## Common Failure Modes ### API starts but returns Mongo errors @@ -105,6 +126,14 @@ kubectl -n droboticflow logs job/ - confirm Role and RoleBinding exist in the namespace - inspect worker logs for Kubernetes API errors - confirm the operator-runner image pull secret name matches `K8S_IMAGE_PULL_SECRET_NAME` +- confirm the shared PVC exists and is bound +- confirm worker and API pods both mount the same workspace path + +### Job starts but downstream nodes cannot see outputs + +- confirm the worker pod and operator-runner Job mount the same PVC and mount path +- confirm the node directory contains `input-artifacts.json`, `result.json`, `stage-manifest.json`, and `outputs/` +- inspect worker logs for `DROBOTICFLOW_RESULT_PATH` parse failures ### Image pull failure diff --git a/docs/operator-authoring.md b/docs/operator-authoring.md new file mode 100644 index 00000000..9656efcb --- /dev/null +++ b/docs/operator-authoring.md @@ -0,0 +1,529 @@ +# 算子开发文档 + +本文档基于当前仓库的真实实现,说明在 Drobodata 中如何创建新算子、算子运行时能拿到什么、如何接入 BOS、如何正确产出 artifact,以及在 Kubernetes 模式下需要注意哪些持久化问题。 + +相关实现入口: + +- `internal/worker/executor.go` +- `internal/worker/runtime.go` +- `internal/k8s/launcher.go` +- `backend/operator-runner/main.py` +- `internal/domain/types.go` + +## 1. 总体执行模型 + +Drobodata 的算子执行分为三层: + +1. 前端保存工作流 DAG,也就是 `nodes + edges` +2. Go worker 负责调度 DAG、收集上游 artifact、启动节点执行 +3. Python operator-runner 真正执行算子逻辑,并把结果回传给 worker + +注意:算子之间传输的不是共享内存,也不是直接传文件句柄,而是 `artifact`。 + +执行链路如下: + +1. 上游节点执行完成 +2. 上游节点产出 `ExecutionArtifact[]` +3. worker 把这些 artifact 持久化到当前 run 的执行记录中 +4. 下游节点启动前,worker 按 DAG 边关系筛选对应的 artifact +5. worker 把这些 artifact 写入共享工作区的 `input-artifacts.json` +6. worker 通过 `INPUT_ARTIFACTS_PATH` 把输入文件路径传给下游算子 + +也就是说,跨算子的数据传输核心仍然是 artifact 契约,但主链路已经改成“共享 PVC 上的文件化 manifest + 轻量环境变量指针”,不再依赖大体积 env 直传。 + +## 2. 算子运行时会收到什么 + +worker 在执行节点前,会构造一组环境变量注入到算子进程或算子容器中。 + +基础环境变量包括: + +- `RUN_ID` +- `WORKFLOW_ID` +- `NODE_ID` +- `NODE_LABEL` +- `NODE_TYPE` +- `NODE_CATEGORY` +- `NODE_DESCRIPTION` +- `NODE_DATA_JSON` +- `INPUT_ARTIFACTS_PATH` +- `RESULT_OUTPUT_PATH` +- `STAGE_MANIFEST_PATH` +- `OUTPUT_DIR` +- `WORKFLOW_PREVIOUS_RUN_TIMESTAMP` +- `WORKFLOW_CURRENT_RUN_TIMESTAMP` + +还可能注入额外变量: + +- `STORAGE_BUCKET` +- `STORAGE_PREFIX` +- `STORAGE_PATTERN` +- `CPU_LIMIT` +- `MEMORY_LIMIT` +- `NODE_IMAGE` +- `PLATFORM_MONGO_URI` +- `PLATFORM_MONGO_DB` +- 由 hyperparams 展开的变量,例如 `BOS_ENDPOINT`、`BOS_ACCESS_KEY_ID`、`COMMAND`、`MONGO_URI` + +其中两个工作流级时间变量需要特别注意: + +- `WORKFLOW_PREVIOUS_RUN_TIMESTAMP` + - 仅对 `scheduled` run 注入 + - 上一轮 workflow trigger 的真实时间字符串 + - 格式固定为 `Asia/Shanghai` 时区下的 RFC3339,例如 `2026-04-22T14:37:53+08:00` + - 首次周期运行时固定为空字符串 +- `WORKFLOW_CURRENT_RUN_TIMESTAMP` + - 本轮 workflow trigger 的真实时间字符串 + - 格式固定为 `Asia/Shanghai` 时区下的 RFC3339,例如 `2026-04-22T14:37:53+08:00` + - 在 run 创建时固化,对该 run 内所有节点保持一致 + +Python 侧的 operator-runner 会: + +1. 读取 `NODE_DATA_JSON` +2. 优先读取 `INPUT_ARTIFACTS_PATH` +3. 根据节点 label 选择内置 handler,或者执行 `COMMAND/SCRIPT` +4. 在 `OUTPUT_DIR` 下产出文件 +5. 把完整执行结果写到 `RESULT_OUTPUT_PATH` +6. 把阶段摘要写到 `STAGE_MANIFEST_PATH` +7. 最后向 stdout 打印一行 `DROBOTICFLOW_RESULT_PATH=` + +worker 再通过结果文件读取日志、artifact、资源采样,并回写到 run 记录。 + +## 3. Artifact 契约 + +artifact 是算子之间的统一数据交换格式。一个 artifact 可能包含以下字段: + +- `id` +- `runId` +- `nodeId` +- `portId` +- `portName` +- `name` +- `type` +- `size` +- `path` +- `metadata` +- `data` + +### 3.1 常见 `type` + +当前常用类型有: + +- `json` +- `image` +- `video` +- `file` +- `tensor` + +建议: + +- 结构化输出优先用 `json` +- 图像、视频尽量标准确切类型 +- 其他通用文件用 `file` + +### 3.2 `metadata` 的用途 + +`metadata` 主要用于保存文件属性与预览相关信息,常见字段有: + +- `mimeType` +- `sha256` +- `duration` +- `resolution` +- `codec` +- `thumbnail` + +如果输出文件可预览,尽量补齐 `mimeType`。 + +### 3.3 `data` 的用途 + +`data` 在当前系统里非常重要: + +- JSON artifact 通常把完整结构化内容直接放在 `data` +- 非 JSON artifact 往往把预览信息放在 `data.preview` +- 远程对象 artifact 往往把来源信息放在 `data.source` +- 上传到 BOS 后的 artifact 常带 `data.relativePath` + +很多前端和 API 页面会优先读 `data` 来展示结果,而不是总去读取 `path` 对应文件。 + +## 4. DAG 边如何路由数据 + +worker 不会把一个节点的所有输出无脑传给所有下游节点,而是按边来筛选。 + +筛选规则如下: + +1. 找到当前节点所有入边 +2. 对每条入边,找到对应的 source 节点执行结果 +3. 查看 source 节点的 artifact 列表 +4. 如果这条边带 `sourceHandle`,并且 source artifact 带 `portId`,则只传 `artifact.portId == edge.sourceHandle` 的 artifact +5. 如果没有端口级信息,则退化成传 source 节点的全部 artifact + +因此,多输出算子必须把输出 artifact 绑定到正确的 output port,否则下游会拿到混合数据。 + +## 5. 两种输出模式 + +新算子输出一般有两种模式。 + +### 5.1 临时文件 artifact + +适用于: + +- 在 `OUTPUT_DIR` 下真实生成了文件 +- 输出内容较小,或者后续不要求强持久化 + +常用 helper: + +- `write_json_artifact(...)` +- `write_text_artifact(...)` +- `write_csv_artifact(...)` +- `collect_output_artifacts(...)` +- `build_artifact(...)` + +特点: + +- `path` 指向共享 PVC 上当前节点的输出文件 +- `metadata.sha256` 可计算 +- `data` 可放完整 JSON 或预览摘要 + +### 5.2 虚拟远程 artifact + +适用于: + +- 结果主要存放在 BOS +- 输出文件较大 +- 结果需要跨 Kubernetes Job 生命周期持久存在 + +常用 helper: + +- `build_virtual_artifact(...)` + +特点: + +- `path` 可以是 `bos://bucket/key` +- `data.source.path` 也可以指向远程对象 +- `data.preview.url` 或 `data.source.previewURL` 可放预签名 URL +- 文件不一定真的存在于 `OUTPUT_DIR` + +在 K8s 模式下,大文件结果更推荐这种方式。 + +## 6. BOS 的接入方式 + +当前系统里,BOS 不是以卷挂载的方式给算子使用,而是通过 SDK 直接访问。 + +### 6.1 BOS 凭据如何传到算子里 + +工作流节点上通常不会直接保存裸的 endpoint/AK/SK,而是可能保存一个 `connectionId`。 + +worker 在执行前会把它解析成实际连接参数,并写进节点 hyperparams: + +- `DATA_SOURCE=BOS` +- `SOURCE_PATH` +- `BOS_ENDPOINT` +- `BOS_ACCESS_KEY_ID` +- `BOS_SECRET_ACCESS_KEY` + +然后这些值会随着环境变量进入 operator-runner。 + +### 6.2 内置 source 算子如何使用 BOS + +`builtin_source(...)` 的逻辑是: + +1. 判断数据源是不是 BOS +2. 如果是,则构造 `BosClient` +3. 列 BOS 对象,形成 entries +4. 产出一个 `source-manifest.json` +5. 为每个远程对象产出一个“虚拟 artifact” + +这些 artifact 往往不会把对象本体下载到 Job 容器临时目录,而是保留: + +- `path = bos://bucket/key` +- `data.source.path = bos://...` +- `data.source.previewURL = 预签名 URL` + +因此下游算子拿到的可能是“远程对象描述符”,不是 Job 临时文件。 + +### 6.3 内置 upload 算子如何使用 BOS + +`builtin_upload(...)` 的逻辑是: + +1. 确定上传目标 bucket/prefix +2. 必要时先把输入 artifact 物化到临时 staging 目录 +3. 调用 `put_object_from_file(...)` 上传到 BOS +4. 为每个上传结果生成新的“远程 artifact” +5. 对视频等可预览媒体,生成预签名 URL + +这意味着 upload 类算子除了做上传,还负责把远程落点重新包装成工作流中可继续流转的 artifact。 + +## 7. 新算子应该怎么设计 + +新增算子前,先明确它属于哪一类。 + +### 7.1 结构化处理算子 + +适用于: + +- JSON 提取 +- JSON 过滤 +- manifest 聚合 +- 路由判断 +- 规则映射 + +推荐方式: + +1. 优先从 `artifact.data` 读取结构化输入 +2. 尽量不依赖 Job 临时文件 +3. 输出使用 `write_json_artifact(...)` + +这是最稳妥的算子类型。 + +### 7.2 文件处理算子 + +适用于: + +- 视频切片 +- 图像转换 +- OCR +- 媒体推理 +- 解压缩 + +推荐方式: + +1. 先把输入标准化成可处理的临时文件 +2. 在 Job 容器的临时工作目录处理 +3. 输出写入 `OUTPUT_DIR` +4. 用 `build_artifact(...)` 或 `collect_output_artifacts(...)` 产出 artifact +5. 如果结果需要长期保留,再上传 BOS 并产出远程 artifact + +### 7.3 Source 算子 + +适用于: + +- BOS 前缀扫描 +- BOS 前缀扫描 +- 外部数据集发现 + +推荐方式: + +1. 收集外部数据条目 +2. 产出一个 manifest JSON +3. 按需为每个条目生成 artifact +4. 如果支持增量语义,再输出 checkpoint artifact + +### 7.4 Sink 算子 + +适用于: + +- 上传到 BOS +- 写入 MongoDB +- 推送到外部业务系统 + +推荐方式: + +1. 接受上游 artifact +2. 解析出结构化内容或临时文件 +3. 写入目标系统 +4. 输出一份 summary JSON +5. 如果目标系统里有远程对象,再顺便产出虚拟远程 artifact + +## 8. 新算子的输入处理规范 + +不要直接假设 `artifact["path"]` 一定可读。 + +建议按下面顺序处理输入: + +1. 如果算子可以直接消费结构化内容,优先读 `artifact.data` +2. 如果 `artifact.path` 是 Job 容器内的临时路径且文件存在,直接读取该文件 +3. 如果 `artifact.path` 是 `bos://...`,要么下载到临时目录,要么按远程模式处理 +4. 如果 `artifact.data.source.path` 是 `bos://...`,可把它当真实远程源 +5. 如果只有 `data.preview.url` 或 `data.source.previewURL`,只能把它当预览信息,不能默认等价于原始文件 +6. 上述都不满足时,明确报错 + +如果你的算子需要“真实文件”,建议统一在 `OUTPUT_DIR/work` 或类似 staging 目录里做输入物化。这里的 `OUTPUT_DIR` 已经位于共享 PVC 的节点工作目录下。 + +## 9. 新算子的输出处理规范 + +每次实现输出时,先问自己几个问题: + +1. 这是结构化 JSON 吗? +2. 这是 Job 临时文件吗? +3. 这是应该长期保存的远程对象吗? +4. 这个算子有多个输出口吗? +5. 前端是否需要预览? + +推荐映射: + +- 结构化 JSON:`write_json_artifact(...)` +- 临时文件:`build_artifact(...)` +- 整个输出目录:`collect_output_artifacts(...)` +- 远程 BOS 输出:上传后 `build_virtual_artifact(...)` + +如果算子有多个输出口,必须把 artifact 绑定到对应的 output。 + +## 10. 多输出算子必须做的事 + +如果一个算子定义了多个 output port,生成 artifact 时必须传入对应的 output binding。 + +否则: + +- `artifact.portId` 不会被设置 +- worker 就无法按 `edge.sourceHandle` 路由 +- 下游节点会错误地接收到全部输出 + +结论很直接:多输出算子如果不写 `portId/portName`,基本等于没实现好多输出。 + +## 11. Kubernetes 模式下最重要的注意事项 + +这是当前实现中最容易踩坑的地方。 + +在 Kubernetes 模式下: + +- worker 会创建一个 Job +- worker、API 和 Job 会共享同一个工作 PVC +- 节点目录下至少会出现: + - `input-artifacts.json` + - `result.json` + - `stage-manifest.json` + - `outputs/` + +因此: + +- `OUTPUT_DIR` 下的文件可以在后续 Job 中继续读取 +- 但它们仍然属于运行期热数据,不应替代长期归档存储 + +所以开发新算子时必须遵守一个原则: + +- 如果输出是关键结果、体积较大、后续还要访问,就不要只写临时文件 +- 应优先上传到 BOS 或其他持久化后端,并返回远程 artifact + +当前系统只支持 Kubernetes Job 执行,因此不要把这类临时路径当成跨 Job 生命周期可复用的稳定存储。 +当前系统只支持 Kubernetes Job 执行,因此共享 PVC 适合作为热数据与中间结果层,长期结果仍应优先上传到 BOS。 + +## 12. 创建新算子时除了代码还要做什么 + +只写 handler 还不够,还需要把算子注册进系统目录。 + +一个完整的算子定义至少应包含: + +- `name` +- `type` +- `category` +- `description` +- `inputs` +- `outputs` +- 可选 `image` +- 可选 `hyperparams` + +特别要注意: + +- Python 内置 handler 的选择目前与节点 label 强相关 +- 所以算子目录中的 `name/label` 要和 handler 路由保持一致 +- 算子定义里的输入输出端口,要和实际 artifact 的输出绑定一致 + +如果这是一个依赖自定义容器镜像的算子,还要额外遵守下面的镜像登记规则: + +- 镜像由用户自己构建并推送到外部容器仓库 +- 平台不会代替用户构建、推送或托管自定义镜像 +- 用户需要把完整镜像地址填写到 `image` 字段,格式建议为 `//:` +- 不要使用 `latest` 作为长期运行版本标签 +- 不要把 `droboticflow-operator-runner:*` 这类平台内建 runner 镜像当成自定义镜像填写 + +在当前部署环境中,推荐使用与平台一致的百度云 CCR,例如: + +```text +ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/custom-: +``` + +## 13. 新算子开发检查清单 + +在提交前,至少检查下面这些点: + +- 算子在空输入、坏路径、非法 BOS 参数下能明确报错 +- 算子不假设所有输入都有可持续存在的临时文件 +- 算子支持 BOS artifact 或能清晰拒绝不支持的输入 +- 临时文件全部写在 `OUTPUT_DIR` 下 +- 最终一定返回合法 artifact,而不是只打印日志 +- 结构化结果优先用 `write_json_artifact(...)` +- 需要长期保存的大文件输出会上传 BOS 或其他持久化后端 +- 多输出 artifact 正确带 `portId/portName` +- 预览友好的输出带 `metadata.mimeType` +- UI 需要预览时,尽量补 `data.preview` +- 即使临时文件未来消失,artifact 的 `data` 依然尽可能保留有用摘要 + +## 14. 新内置算子的推荐代码模板 + +在 `backend/operator-runner/main.py` 中新增 handler 时,建议按这个结构写: + +```python +def builtin_my_operator( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + + work_dir = output_dir / "work" + work_dir.mkdir(parents=True, exist_ok=True) + + # 1. 解析 inputs + # 2. 必要时把 bos:// 或远程对象下载到 work_dir + # 3. 执行业务逻辑 + # 4. 写临时输出或上传远程输出 + # 5. 返回 artifact 列表 + + payload = { + "ok": True, + "inputCount": len(inputs), + } + + return [ + write_json_artifact( + run_id, + node_id, + output_dir, + "result.json", + payload, + output=output_binding(node_data), + ) + ] +``` + +然后再把它接入 `main()` 的 handler 分发逻辑。 + +## 15. 常见错误 + +以下是新增算子时最常见的实现错误: + +- 直接读取 `artifact.path`,却没有判断它是不是 `bos://...` +- 只打印日志,没有输出 `DROBOTICFLOW_RESULT_PATH=...` +- 生成了文件,但没有包装成 artifact +- 多输出算子没写 `portId` +- 把 `previewURL` 当成原文件长期来源 +- 在 K8s 模式下把共享 PVC 当成长期归档存储 +- 漏掉 `mimeType`,导致预览和 API 内容服务退化 +- 对大文件直接把完整内容塞进 `data.preview` + +## 16. 什么时候应该在算子内部直接上传 BOS + +以下情况建议在算子内部直接上传 BOS,而不是只写临时文件: + +- 输出文件很大 +- 输出必须跨 Job 生命周期保留 +- API 或前端后续需要稳定的远程路径 +- 需要为媒体生成可访问的预签名预览 URL + +以下情况临时输出也可以接受: + +- 输出很小 +- 主要价值已经体现在 `data` 里 +- 只作为单个 Job 内部的中间产物使用 +- 后续节点只关心结构化摘要,不关心原始文件 + +## 17. 相关文件 + +- [backend/operator-runner/main.py](/Users/longtaowu/workspace/droboticflow/backend/operator-runner/main.py) +- [internal/worker/executor.go](/Users/longtaowu/workspace/droboticflow/internal/worker/executor.go) +- [internal/worker/runtime.go](/Users/longtaowu/workspace/droboticflow/internal/worker/runtime.go) +- [internal/k8s/launcher.go](/Users/longtaowu/workspace/droboticflow/internal/k8s/launcher.go) +- [internal/domain/types.go](/Users/longtaowu/workspace/droboticflow/internal/domain/types.go) +- [docs/api.md](/Users/longtaowu/workspace/droboticflow/docs/api.md) +- [docs/operations.md](/Users/longtaowu/workspace/droboticflow/docs/operations.md) diff --git a/docs/operators/BOS-增量扫描.md b/docs/operators/BOS-增量扫描.md new file mode 100644 index 00000000..a9213c0d --- /dev/null +++ b/docs/operators/BOS-增量扫描.md @@ -0,0 +1,124 @@ +# BOS 增量扫描 + +- 算子 ID:`operator-builtin-bos-incremental-scan` +- 类别:`IO` +- 类型:`builtin` + +## 作用 + +按 bucket/prefix 增量扫描 BOS,仅输出本次新增对象并提交 checkpoint 给 runtime。 + +## 定位与职责边界 + +- 该算子属于 `IO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向工作流入口,职责是把外部世界的数据引用整理成平台统一 artifact。 + +## 平台语义 + +统一工作流里的有状态源算子。每次 run 只产出 BOS 中新增的对象,状态由 runtime 在 run 成功后提交。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_source(...)`。 +- 该类源算子没有显式上游输入,主要依赖节点超参数和运行时注入的存储连接信息。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[BOS 增量扫描] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- 无显式输入端口。通常作为源算子使用,主要依赖节点超参数和运行时环境。 + +## 输出端口详解 + +- `out-1` / **manifest** / `json` / `bus`:增量对象清单 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `DATA_SOURCE`:类型 `string`,默认值 `BOS`。固定为 BOS 可直接使用默认值,也可在节点级覆盖。 +- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。完整 bos://bucket/prefix 路径 通常建议在工作流中显式配置。 +- `BOS_ENDPOINT`:类型 `string`,默认值 `https://bj.bcebos.com`。BOS Endpoint 可直接使用默认值,也可在节点级覆盖。 +- `BOS_ACCESS_KEY_ID`:类型 `string`,默认值 空字符串或由工作流显式填写。BOS Access Key ID 通常建议在工作流中显式配置。 +- `BOS_SECRET_ACCESS_KEY`:类型 `string`,默认值 空字符串或由工作流显式填写。BOS Secret Access Key 通常建议在工作流中显式配置。 +- `SCAN_MODE`:类型 `string`,默认值 `INCREMENTAL`。固定为 INCREMENTAL 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要预先准备连接配置、凭据和目标路径,优先通过平台连接配置而不是把密钥散落到节点超参数里。 + +## 数据通信与存储路径 + +- 源算子通常先输出轻量级 manifest,记录 `entries`、`catalogEntries`、`bos://` 路径或摘要信息,而不是直接下载整个数据集。 +- 下游第一次真正读取文件内容时,才会通过 runner 的物化逻辑把远程对象缓存到共享 PVC 或本地工作目录。 + +## 典型上下游编排 + +- 常接 `X 平台数据集索引`、`目录扫描`、`模型推理` 或任意自定义 JSON 处理节点,作为工作流入口。 +- 当需要持续监听新增目录或新增对象时,应把它放在整条 DAG 的最上游,并让后续节点只消费本轮增量。 + +## 节点配置示例 + +```json +{ + "label": "BOS 增量扫描", + "operatorId": "operator-builtin-bos-incremental-scan", + "type": "builtin", + "category": "IO", + "hyperparams": { + "DATA_SOURCE": "BOS", + "SOURCE_PATH": "<按节点填写>", + "BOS_ENDPOINT": "https://bj.bcebos.com", + "BOS_ACCESS_KEY_ID": "<按节点填写>", + "BOS_SECRET_ACCESS_KEY": "<按节点填写>", + "SCAN_MODE": "INCREMENTAL" + } +} +``` + +## 输入输出示例 + +- 示例配置:`SOURCE_PATH=bos://wjzh-data-migration/wjzh-0330-Data-migration`,`SCAN_MODE=INCREMENTAL`,`INCREMENTAL_UNIT=DIRECTORY`。 +- 典型输出:`source-manifest.json`,其中 `entries` 表示本轮增量对象,`catalogEntries` 表示本轮完整列举结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 +- 输出 manifest 与真实源路径内容一致,没有漏扫、重扫或错误分类。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 对象存储异常时,先核对 `SOURCE_PATH`、bucket/prefix、endpoint、AK/SK 和目标对象是否真实存在。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 +- 对照真实源目录核验 `entries`、目录数、对象数和增量语义是否正确。 + +## 相关文档 + +- [IO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/CSV-转-JSON.md b/docs/operators/CSV-转-JSON.md new file mode 100644 index 00000000..e2f26eda --- /dev/null +++ b/docs/operators/CSV-转-JSON.md @@ -0,0 +1,108 @@ +# CSV 转 JSON + +- 算子 ID:`operator-builtin-csv-to-json` +- 类别:`DATA` +- 类型:`builtin` + +## 作用 + +将真实 CSV/TSV 文件转换成结构化 JSON 行列表。 + +## 定位与职责边界 + +- 该算子属于 `DATA` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。 + +## 平台语义 + +读取 SOURCE_PATH 或上游 source manifest 指向的 csv/tsv 文件,输出完整 JSON 行数据。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_csv_to_json(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[CSV 转 JSON] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:CSV/TSV 输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **table** / `json` / `bus`:JSON 行列表 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。csv/tsv 文件路径 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "CSV 转 JSON", + "operatorId": "operator-builtin-csv-to-json", + "type": "builtin", + "category": "DATA", + "hyperparams": { + "SOURCE_PATH": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [DATA 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/JSON-字段提取.md b/docs/operators/JSON-字段提取.md new file mode 100644 index 00000000..978b464f --- /dev/null +++ b/docs/operators/JSON-字段提取.md @@ -0,0 +1,108 @@ +# JSON 字段提取 + +- 算子 ID:`operator-builtin-json-field-extract` +- 类别:`DATA` +- 类型:`builtin` + +## 作用 + +按字段路径从 JSON 输入中提取指定值。 + +## 定位与职责边界 + +- 该算子属于 `DATA` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。 + +## 平台语义 + +根据 FIELD_PATH 从 JSON 输入中提取字段。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_json_extract(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[JSON 字段提取] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:JSON 输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **extracted** / `json` / `bus`:提取结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `FIELD_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。点分字段路径,例如 metrics.score 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "JSON 字段提取", + "operatorId": "operator-builtin-json-field-extract", + "type": "builtin", + "category": "DATA", + "hyperparams": { + "FIELD_PATH": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [DATA 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/JSON-映射重命名.md b/docs/operators/JSON-映射重命名.md new file mode 100644 index 00000000..a1927a65 --- /dev/null +++ b/docs/operators/JSON-映射重命名.md @@ -0,0 +1,108 @@ +# JSON 映射重命名 + +- 算子 ID:`operator-builtin-json-map-rename` +- 类别:`DATA` +- 类型:`builtin` + +## 作用 + +按映射表批量重命名 JSON 字段。 + +## 定位与职责边界 + +- 该算子属于 `DATA` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。 + +## 平台语义 + +根据 FIELD_MAP_JSON 真实执行字段名重映射。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_json_map(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[JSON 映射重命名] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:JSON 输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **mapped** / `json` / `bus`:映射结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `FIELD_MAP_JSON`:类型 `string`,默认值 `{}`。JSON 字段映射,例如 {"old":"new"} 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "JSON 映射重命名", + "operatorId": "operator-builtin-json-map-rename", + "type": "builtin", + "category": "DATA", + "hyperparams": { + "FIELD_MAP_JSON": "{}" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [DATA 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/JSON-转-CSV.md b/docs/operators/JSON-转-CSV.md new file mode 100644 index 00000000..d7e13631 --- /dev/null +++ b/docs/operators/JSON-转-CSV.md @@ -0,0 +1,106 @@ +# JSON 转 CSV + +- 算子 ID:`operator-builtin-json-to-csv` +- 类别:`DATA` +- 类型:`builtin` + +## 作用 + +将 JSON 行数据导出为真实 CSV 文件。 + +## 定位与职责边界 + +- 该算子属于 `DATA` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。 + +## 平台语义 + +把上游 JSON items/entries 或单条对象导出为真实 CSV 文件,并附带摘要。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_json_to_csv(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[JSON 转 CSV] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:JSON 行列表 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **csv** / `file` / `bus`:CSV 文件 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "JSON 转 CSV", + "operatorId": "operator-builtin-json-to-csv", + "type": "builtin", + "category": "DATA", + "hyperparams": {} +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [DATA 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/JSON-过滤.md b/docs/operators/JSON-过滤.md new file mode 100644 index 00000000..f2014baf --- /dev/null +++ b/docs/operators/JSON-过滤.md @@ -0,0 +1,114 @@ +# JSON 过滤 + +- 算子 ID:`operator-builtin-json-filter` +- 类别:`DATA` +- 类型:`builtin` + +## 作用 + +按字段条件过滤 JSON 列表。 + +## 定位与职责边界 + +- 该算子属于 `DATA` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。 + +## 平台语义 + +按 FIELD_PATH + OPERATOR + EXPECTED 条件真实过滤 JSON items。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_json_filter(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[JSON 过滤] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:JSON 输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **filtered** / `json` / `bus`:过滤结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `FIELD_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。待判断字段路径 通常建议在工作流中显式配置。 +- `OPERATOR`:类型 `string`,默认值 `eq`。eq/ne/contains/gt/gte/lt/lte/exists/missing 可直接使用默认值,也可在节点级覆盖。 +- `EXPECTED`:类型 `string`,默认值 空字符串或由工作流显式填写。字符串比较值 通常建议在工作流中显式配置。 +- `EXPECTED_JSON`:类型 `string`,默认值 空字符串或由工作流显式填写。JSON 格式比较值,优先于 EXPECTED 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "JSON 过滤", + "operatorId": "operator-builtin-json-filter", + "type": "builtin", + "category": "DATA", + "hyperparams": { + "FIELD_PATH": "<按节点填写>", + "OPERATOR": "eq", + "EXPECTED": "<按节点填写>", + "EXPECTED_JSON": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [DATA 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/LeRobot-数据集概览.md b/docs/operators/LeRobot-数据集概览.md new file mode 100644 index 00000000..8c18b579 --- /dev/null +++ b/docs/operators/LeRobot-数据集概览.md @@ -0,0 +1,107 @@ +# LeRobot 数据集概览 + +- 算子 ID:`operator-builtin-lerobot-summary` +- 类别:`ROBOTICS` +- 类型:`builtin` + +## 作用 + +读取真实 LeRobot 数据集目录,输出 meta/tasks/parquet/videos 概览。 + +## 定位与职责边界 + +- 该算子属于 `ROBOTICS` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 + +## 平台语义 + +面向 LeRobot 数据集目录的真实概览算子,适合机器人示教数据检查。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_lerobot_summary(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[LeRobot 数据集概览] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:目录输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **summary** / `json` / `bus`:概览结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。LeRobot 数据集根目录 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "LeRobot 数据集概览", + "operatorId": "operator-builtin-lerobot-summary", + "type": "builtin", + "category": "ROBOTICS", + "hyperparams": { + "SOURCE_PATH": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [ROBOTICS 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/MongoDB-写入.md b/docs/operators/MongoDB-写入.md new file mode 100644 index 00000000..a37b27b8 --- /dev/null +++ b/docs/operators/MongoDB-写入.md @@ -0,0 +1,126 @@ +# MongoDB 写入 + +- 算子 ID:`operator-builtin-mongodb-write` +- 类别:`DATA` +- 类型:`builtin` + +## 作用 + +将上游 JSON 数据写入用户配置的业务 MongoDB,支持批量插入和按键 upsert,并允许显式覆盖连接。 + +## 定位与职责边界 + +- 该算子属于 `DATA` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向工作流出口,职责是把平台内部产物同步到外部对象存储或业务数据库。 + +## 平台语义 + +将 JSON 文档写入业务 MongoDB 集合。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_mongodb_write(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[MongoDB 写入] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `input` / **input** / `json` / `bus`:待写入的 JSON 文档或 items 列表 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `result` / **result** / `json` / `bus`:写入结果摘要 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `MONGO_COLLECTION`:类型 `string`,默认值 空字符串或由工作流显式填写。目标集合名称(禁止写系统保留集合) 通常建议在工作流中显式配置。 +- `MONGO_MODE`:类型 `string`,默认值 `insert_many`。写入模式:insert_many 或 upsert 可直接使用默认值,也可在节点级覆盖。 +- `MONGO_CONNECTION_ID`:类型 `string`,默认值 空字符串或由工作流显式填写。业务 MongoDB 连接 ID;推荐优先配置 通常建议在工作流中显式配置。 +- `MONGO_TARGET`:类型 `string`,默认值 `business`。兼容字段;默认 business。若显式传入 URI/DB,会优先使用显式连接 可直接使用默认值,也可在节点级覆盖。 +- `UPSERT_KEY`:类型 `string`,默认值 `_id`。upsert 模式使用的唯一键 可直接使用默认值,也可在节点级覆盖。 +- `DOCUMENT_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。从输入 payload 中提取待写入文档的字段路径 通常建议在工作流中显式配置。 +- `MONGO_DB`:类型 `string`,默认值 空字符串或由工作流显式填写。可选数据库名;显式填写后会覆盖 business/platform 默认连接 通常建议在工作流中显式配置。 +- `MONGO_URI`:类型 `string`,默认值 空字符串或由工作流显式填写。可选 Mongo URI;显式填写后会覆盖 business/platform 默认连接 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要预先准备连接配置、凭据和目标路径,优先通过平台连接配置而不是把密钥散落到节点超参数里。 + +## 数据通信与存储路径 + +- 该节点主要消费结构化 JSON payload,并把整理后的文档写入目标 Mongo 集合。 +- 平台主库只保留写入摘要、错误信息和执行状态,避免把已写入业务库的整批文档再次冗余保存。 + +## 典型上下游编排 + +- 通常接在 JSON 提取、映射、模板渲染或清洗节点之后,确保入库文档已经满足目标库 schema。 + +## 节点配置示例 + +```json +{ + "label": "MongoDB 写入", + "operatorId": "operator-builtin-mongodb-write", + "type": "builtin", + "category": "DATA", + "hyperparams": { + "MONGO_COLLECTION": "<按节点填写>", + "MONGO_MODE": "insert_many", + "MONGO_CONNECTION_ID": "<按节点填写>", + "MONGO_TARGET": "business", + "UPSERT_KEY": "_id", + "DOCUMENT_PATH": "<按节点填写>", + "MONGO_DB": "<按节点填写>", + "MONGO_URI": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:结构化 JSON 行列表、单文档对象或经过模板渲染后的入库 payload。 +- 典型输出:写入摘要、命中集合名、成功数量、失败数量和错误原因。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 +- 外部系统中存在真实副作用,例如对象已上传、文档已写入,而不只是平台内显示成功。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 若写库失败,先确认目标集合不在保留名单中,再核对连接串、认证信息和文档结构是否可序列化。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 +- 到目标 Mongo 集合查询最近写入数据,确认记录数量、关键字段和值类型都符合预期。 + +## 相关文档 + +- [DATA 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/README.md b/docs/operators/README.md new file mode 100644 index 00000000..434c4630 --- /dev/null +++ b/docs/operators/README.md @@ -0,0 +1,69 @@ +# 内建算子分册 + +当前目录按“每个内建算子一份 Markdown”组织,内容基于当前平台真实算子定义按算子名称去重生成。 + +- 当前收录数量:`25` +- 文档定位:单算子详细手册,重点覆盖输入输出契约、运行时路径、配置示例、排障与验收。 +- 配套总览:[`../builtin-operators-manual.zh-CN.md`](../builtin-operators-manual.zh-CN.md) + +## 使用建议 + +- 先阅读总手册理解平台级规则,再进入本目录阅读单算子分册。 +- 设计 DAG 时优先确认端口类型、artifact 语义、外部副作用和验收口径,而不是只看算子名称。 + +## 分类索引 + +### DATA + +- [CSV 转 JSON](CSV-转-JSON.md) +- [JSON 字段提取](JSON-字段提取.md) +- [JSON 映射重命名](JSON-映射重命名.md) +- [JSON 转 CSV](JSON-转-CSV.md) +- [JSON 过滤](JSON-过滤.md) +- [MongoDB 写入](MongoDB-写入.md) + +### DATASET + +- [X 平台数据集索引](X-平台数据集索引.md) +- [采集目录校验](采集目录校验.md) + +### HUMAN + +- [人工审核](人工审核.md) + +### IO + +- [BOS 增量扫描](BOS-增量扫描.md) +- [目录扫描](目录扫描.md) +- [真实数据源](真实数据源.md) +- [结果上传](结果上传.md) + +### LOGIC + +- [条件分流](条件分流.md) + +### MATH + +- [数据合并](数据合并.md) + +### QA + +- [抽检采样](抽检采样.md) + +### ROBOTICS + +- [LeRobot 数据集概览](LeRobot-数据集概览.md) +- [ROSBag 数据概览](ROSBag-数据概览.md) + +### UTIL + +- [Shell 命令执行](Shell-命令执行.md) +- [文本模板渲染](文本模板渲染.md) + +### VIDEO + +- [成品打包](成品打包.md) +- [数据过滤](数据过滤.md) +- [视频切片](视频切片.md) +- [视频标注](视频标注.md) +- [视频质检](视频质检.md) diff --git a/docs/operators/ROSBag-数据概览.md b/docs/operators/ROSBag-数据概览.md new file mode 100644 index 00000000..c7c51d4f --- /dev/null +++ b/docs/operators/ROSBag-数据概览.md @@ -0,0 +1,107 @@ +# ROSBag 数据概览 + +- 算子 ID:`operator-builtin-rosbag-summary` +- 类别:`ROBOTICS` +- 类型:`builtin` + +## 作用 + +读取真实 rosbag1 / rosbag2 / mcap / db3 数据并输出概览。 + +## 定位与职责边界 + +- 该算子属于 `ROBOTICS` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 + +## 平台语义 + +支持 rosbag db3 topic/message 概览,以及 bag/mcap 文件摘要。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_rosbag_summary(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[ROSBag 数据概览] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:bag 输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **summary** / `json` / `bus`:概览结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。bag/db3/mcap 文件或 rosbag2 目录 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "ROSBag 数据概览", + "operatorId": "operator-builtin-rosbag-summary", + "type": "builtin", + "category": "ROBOTICS", + "hyperparams": { + "SOURCE_PATH": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [ROBOTICS 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/Shell-命令执行.md b/docs/operators/Shell-命令执行.md new file mode 100644 index 00000000..19a13746 --- /dev/null +++ b/docs/operators/Shell-命令执行.md @@ -0,0 +1,111 @@ +# Shell 命令执行 + +- 算子 ID:`operator-builtin-shell-command` +- 类别:`UTIL` +- 类型:`builtin` + +## 作用 + +在 operator-runner 容器中真实执行 shell 命令并收集输出文件。 + +## 定位与职责边界 + +- 该算子属于 `UTIL` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 + +## 平台语义 + +执行 COMMAND 或 SCRIPT,自动收集输出目录产物并生成预览元数据。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_generic(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 +- 这类通用执行算子适合快速验证,但当逻辑稳定后应考虑沉淀为专用内建算子或受控自定义算子。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[Shell 命令执行] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:输入数据 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **result** / `file` / `bus`:执行结果文件 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `COMMAND`:类型 `string`,默认值 空字符串或由工作流显式填写。容器内执行的命令 通常建议在工作流中显式配置。 +- `SCRIPT`:类型 `string`,默认值 空字符串或由工作流显式填写。可执行脚本内容 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "Shell 命令执行", + "operatorId": "operator-builtin-shell-command", + "type": "builtin", + "category": "UTIL", + "hyperparams": { + "COMMAND": "<按节点填写>", + "SCRIPT": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [UTIL 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/X-平台数据集索引.md b/docs/operators/X-平台数据集索引.md new file mode 100644 index 00000000..a44de8b8 --- /dev/null +++ b/docs/operators/X-平台数据集索引.md @@ -0,0 +1,108 @@ +# X 平台数据集索引 + +- 算子 ID:`operator-builtin-xplatform-dataset-index` +- 类别:`DATASET` +- 类型:`builtin` + +## 作用 + +扫描 X-platform 采集目录,解析 meta.json、video_meta.json 和视频文件索引。 + +## 定位与职责边界 + +- 该算子属于 `DATASET` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 + +## 平台语义 + +面向 BJ_***/DJI_***/meta.json/video_meta.json 结构的真实数据集索引算子,输出采集目录和视频清单。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_xplatform_index(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[X 平台数据集索引] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:目录输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **dataset** / `json` / `bus`:数据集索引 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。X-platform 根目录或单个采集目录 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 上游通常是 `BOS 增量扫描` 或 `真实数据源`,下游再接 `采集目录校验`、`数据过滤`、`视频质检` 等节点。 +- 如果后续要做抽检或上传,建议把索引结果先标准化为结构化 manifest,再进入视频主链路。 + +## 节点配置示例 + +```json +{ + "label": "X 平台数据集索引", + "operatorId": "operator-builtin-xplatform-dataset-index", + "type": "builtin", + "category": "DATASET", + "hyperparams": { + "SOURCE_PATH": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [DATASET 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/人工审核.md b/docs/operators/人工审核.md new file mode 100644 index 00000000..526282b6 --- /dev/null +++ b/docs/operators/人工审核.md @@ -0,0 +1,110 @@ +# 人工审核 + +- 算子 ID:`operator-builtin-human-review` +- 类别:`HUMAN` +- 类型:`builtin` + +## 作用 + +将真实输入送入人工审核并等待审核动作恢复流程。 + +## 定位与职责边界 + +- 该算子属于 `HUMAN` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 + +## 平台语义 + +基于真实输入生成审核任务并等待人工处理。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_condition_router(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 +- 审核节点的职责是等待外部决策并路由分支,不应继续承担后置业务处理。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[人工审核] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **candidate** / `json` / `bus`:候选数据 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `approved` / **approved** / `json` / `bus`:审核通过 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 +- `rejected` / **rejected** / `json` / `bus`:审核拒绝 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要提前确认审核 API、审核页面或人工流程是否可用,否则工作流会停在等待状态。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 审核节点之后应明确分成 `approved` 和 `rejected` 两条支路,避免后续节点混收不同语义的数据。 + +## 节点配置示例 + +```json +{ + "label": "人工审核", + "operatorId": "operator-builtin-human-review", + "type": "builtin", + "category": "HUMAN", + "hyperparams": {} +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 +- 审核节点收到明确的 approve/reject 决策,并将正确的分支结果送达下游。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 节点停在等待状态通常不是故障,而是还没有收到人工决策。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [HUMAN 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/成品打包.md b/docs/operators/成品打包.md new file mode 100644 index 00000000..c3050a26 --- /dev/null +++ b/docs/operators/成品打包.md @@ -0,0 +1,108 @@ +# 成品打包 + +- 算子 ID:`operator-builtin-package` +- 类别:`VIDEO` +- 类型:`builtin` + +## 作用 + +将切片视频、质检结果、标注结果和配置文件组装成成品数据包。 + +## 定位与职责边界 + +- 该算子属于 `VIDEO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它位于视频主处理链路中,对共享 PVC、本地文件路径和 manifest 契约依赖较强。 + +## 平台语义 + +按 clip 组织真实 package 目录,输出 packages manifest 和所有成品文件。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_result_package(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[成品打包] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:切片/标注/质检结果 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **packages** / `json` / `bus`:成品包清单 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。 + +## 数据通信与存储路径 + +- 视频类处理主要依赖共享 PVC 的本地路径执行,必要时会把 `bos://...` 输入缓存到 `_bos_cache` 或当前节点工作目录。 +- 上下游之间传递的重点是 clip 路径、manifest、annotation JSON、package 目录等引用,而不是媒体字节本身。 + +## 典型上下游编排 + +- 建议在打包后单独接一个上传节点;如果同时上传 clip、标注和 package,应该拆成多条上传支路。 + +## 节点配置示例 + +```json +{ + "label": "成品打包", + "operatorId": "operator-builtin-package", + "type": "builtin", + "category": "VIDEO", + "hyperparams": {} +} +``` + +## 输入输出示例 + +- 典型输入:clip、annotation、quality、配置文件或传感器补充数据。 +- 典型输出:`packages-manifest.json` 和每个成品包目录。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 视频链路异常时,先在 runner 容器内用 `ffprobe`/`ffmpeg` 验证输入文件是否可读,再检查缓存路径和磁盘空间。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [VIDEO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/抽检采样.md b/docs/operators/抽检采样.md new file mode 100644 index 00000000..c10444aa --- /dev/null +++ b/docs/operators/抽检采样.md @@ -0,0 +1,107 @@ +# 抽检采样 + +- 算子 ID:`operator-builtin-audit-sample` +- 类别:`QA` +- 类型:`builtin` + +## 作用 + +按质检和标注规则对成品数据执行真实抽检采样。 + +## 定位与职责边界 + +- 该算子属于 `QA` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 + +## 平台语义 + +根据 RULES_JSON 对成品数据包做规则筛选和稳定抽样,输出抽检任务清单。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_audit_sampling(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[抽检采样] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:成品包输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **samples** / `json` / `bus`:抽检样本清单 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `RULES_JSON`:类型 `string`,默认值 `[]`。抽检规则数组,包含 field/operator/value/samplePercent 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "抽检采样", + "operatorId": "operator-builtin-audit-sample", + "type": "builtin", + "category": "QA", + "hyperparams": { + "RULES_JSON": "[]" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [QA 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/数据合并.md b/docs/operators/数据合并.md new file mode 100644 index 00000000..77c6192d --- /dev/null +++ b/docs/operators/数据合并.md @@ -0,0 +1,107 @@ +# 数据合并 + +- 算子 ID:`operator-builtin-merge` +- 类别:`MATH` +- 类型:`builtin` + +## 作用 + +将多个 JSON 输入合并成统一结果。 + +## 定位与职责边界 + +- 该算子属于 `MATH` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。 + +## 平台语义 + +对多个 JSON 输入执行真实合并。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_merge(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[数据合并] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **left** / `json` / `bus`:左输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 +- `in-2` / **right** / `json` / `bus`:右输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **merged** / `json` / `bus`:合并结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "数据合并", + "operatorId": "operator-builtin-merge", + "type": "builtin", + "category": "MATH", + "hyperparams": {} +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [MATH 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/数据过滤.md b/docs/operators/数据过滤.md new file mode 100644 index 00000000..9252219b --- /dev/null +++ b/docs/operators/数据过滤.md @@ -0,0 +1,119 @@ +# 数据过滤 + +- 算子 ID:`operator-builtin-video-filter` +- 类别:`VIDEO` +- 类型:`builtin` + +## 作用 + +按日期、地点、场景和质检阈值过滤有效视频。 + +## 定位与职责边界 + +- 该算子属于 `VIDEO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 + +## 平台语义 + +针对 X-platform 质检结果执行真实筛选,输出符合条件的视频列表。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_dataset_filter(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[数据过滤] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:视频与元数据输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **filtered** / `json` / `bus`:过滤结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `DATE_START`:类型 `string`,默认值 空字符串或由工作流显式填写。开始日期,支持 YYYY-MM-DD / YYYYMMDD 通常建议在工作流中显式配置。 +- `DATE_END`:类型 `string`,默认值 空字符串或由工作流显式填写。结束日期,支持 YYYY-MM-DD / YYYYMMDD 通常建议在工作流中显式配置。 +- `PLACE`:类型 `string`,默认值 空字符串或由工作流显式填写。采集地点 通常建议在工作流中显式配置。 +- `TYPE`:类型 `string`,默认值 空字符串或由工作流显式填写。采集场景类型 通常建议在工作流中显式配置。 +- `MIN_CLARITY`:类型 `string`,默认值 `0.8`。清晰度最低阈值 可直接使用默认值,也可在节点级覆盖。 +- `MAX_SHAKE`:类型 `string`,默认值 `0.1`。抖动度最高阈值 可直接使用默认值,也可在节点级覆盖。 +- `MAX_HAND_OCCLUSION`:类型 `string`,默认值 `0.1`。无手率最高阈值 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "数据过滤", + "operatorId": "operator-builtin-video-filter", + "type": "builtin", + "category": "VIDEO", + "hyperparams": { + "DATE_START": "<按节点填写>", + "DATE_END": "<按节点填写>", + "PLACE": "<按节点填写>", + "TYPE": "<按节点填写>", + "MIN_CLARITY": "0.8", + "MAX_SHAKE": "0.1", + "MAX_HAND_OCCLUSION": "0.1" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [VIDEO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/文本模板渲染.md b/docs/operators/文本模板渲染.md new file mode 100644 index 00000000..8fe6ada4 --- /dev/null +++ b/docs/operators/文本模板渲染.md @@ -0,0 +1,110 @@ +# 文本模板渲染 + +- 算子 ID:`operator-builtin-template-render` +- 类别:`UTIL` +- 类型:`builtin` + +## 作用 + +使用上游 JSON 上下文渲染真实文本模板文件。 + +## 定位与职责边界 + +- 该算子属于 `UTIL` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。 + +## 平台语义 + +支持 {{field}} 和 Python format 风格占位符,输出真实文本文件。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_text_template(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[文本模板渲染] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:JSON 输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **rendered** / `file` / `bus`:渲染后的文本文件 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `TEMPLATE_TEXT`:类型 `string`,默认值 空字符串或由工作流显式填写。模板文本 通常建议在工作流中显式配置。 +- `OUTPUT_NAME`:类型 `string`,默认值 `rendered.txt`。输出文件名 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "文本模板渲染", + "operatorId": "operator-builtin-template-render", + "type": "builtin", + "category": "UTIL", + "hyperparams": { + "TEMPLATE_TEXT": "<按节点填写>", + "OUTPUT_NAME": "rendered.txt" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [UTIL 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/条件分流.md b/docs/operators/条件分流.md new file mode 100644 index 00000000..76972e1c --- /dev/null +++ b/docs/operators/条件分流.md @@ -0,0 +1,115 @@ +# 条件分流 + +- 算子 ID:`operator-builtin-conditional-route` +- 类别:`LOGIC` +- 类型:`builtin` + +## 作用 + +按字段条件将输入拆分到 matched / unmatched 两个真实输出端口。 + +## 定位与职责边界 + +- 该算子属于 `LOGIC` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。 + +## 平台语义 + +按 FIELD_PATH + OPERATOR + EXPECTED 条件把真实数据分流到两个输出端口。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_condition_router(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[条件分流] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:JSON 输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `matched` / **matched** / `json` / `bus`:命中规则的数据 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 +- `unmatched` / **unmatched** / `json` / `bus`:未命中规则的数据 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `FIELD_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。待判断字段路径 通常建议在工作流中显式配置。 +- `OPERATOR`:类型 `string`,默认值 `eq`。eq/ne/contains/gt/gte/lt/lte/exists/missing 可直接使用默认值,也可在节点级覆盖。 +- `EXPECTED`:类型 `string`,默认值 空字符串或由工作流显式填写。字符串比较值 通常建议在工作流中显式配置。 +- `EXPECTED_JSON`:类型 `string`,默认值 空字符串或由工作流显式填写。JSON 格式比较值,优先于 EXPECTED 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "条件分流", + "operatorId": "operator-builtin-conditional-route", + "type": "builtin", + "category": "LOGIC", + "hyperparams": { + "FIELD_PATH": "<按节点填写>", + "OPERATOR": "eq", + "EXPECTED": "<按节点填写>", + "EXPECTED_JSON": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [LOGIC 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/目录扫描.md b/docs/operators/目录扫描.md new file mode 100644 index 00000000..0646ccfe --- /dev/null +++ b/docs/operators/目录扫描.md @@ -0,0 +1,108 @@ +# 目录扫描 + +- 算子 ID:`operator-builtin-directory-scan` +- 类别:`IO` +- 类型:`builtin` + +## 作用 + +对真实目录做文件统计、扩展名分布和样例文件扫描。 + +## 定位与职责边界 + +- 该算子属于 `IO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向工作流入口,职责是把外部世界的数据引用整理成平台统一 artifact。 + +## 平台语义 + +读取 SOURCE_PATH 或上游 source manifest,输出真实目录概览。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_directory_scan(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[目录扫描] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:目录输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **summary** / `json` / `bus`:目录概览 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。待扫描目录 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "目录扫描", + "operatorId": "operator-builtin-directory-scan", + "type": "builtin", + "category": "IO", + "hyperparams": { + "SOURCE_PATH": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [IO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/真实数据源.md b/docs/operators/真实数据源.md new file mode 100644 index 00000000..7a30c212 --- /dev/null +++ b/docs/operators/真实数据源.md @@ -0,0 +1,117 @@ +# 真实数据源 + +- 算子 ID:`operator-builtin-real-source` +- 类别:`IO` +- 类型:`builtin` + +## 作用 + +从 SOURCE_TEXT、SOURCE_JSON、SOURCE_PATH 或上游输入生成真实源数据清单。 + +## 定位与职责边界 + +- 该算子属于 `IO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向工作流入口,职责是把外部世界的数据引用整理成平台统一 artifact。 + +## 平台语义 + +工作流起点。读取真实输入源并生成结构化清单,供下游节点消费。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_source(...)`。 +- 该类源算子没有显式上游输入,主要依赖节点超参数和运行时注入的存储连接信息。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[真实数据源] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- 无显式输入端口。通常作为源算子使用,主要依赖节点超参数和运行时环境。 + +## 输出端口详解 + +- `out-1` / **manifest** / `json` / `bus`:源数据清单 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `SOURCE_TEXT`:类型 `string`,默认值 空字符串或由工作流显式填写。直接注入文本内容 通常建议在工作流中显式配置。 +- `SOURCE_JSON`:类型 `string`,默认值 空字符串或由工作流显式填写。直接注入 JSON 内容 通常建议在工作流中显式配置。 +- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。从路径读取真实数据 通常建议在工作流中显式配置。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 源算子通常先输出轻量级 manifest,记录 `entries`、`catalogEntries`、`bos://` 路径或摘要信息,而不是直接下载整个数据集。 +- 下游第一次真正读取文件内容时,才会通过 runner 的物化逻辑把远程对象缓存到共享 PVC 或本地工作目录。 + +## 典型上下游编排 + +- 常接 `X 平台数据集索引`、`目录扫描`、`模型推理` 或任意自定义 JSON 处理节点,作为工作流入口。 +- 当需要持续监听新增目录或新增对象时,应把它放在整条 DAG 的最上游,并让后续节点只消费本轮增量。 + +## 节点配置示例 + +```json +{ + "label": "真实数据源", + "operatorId": "operator-builtin-real-source", + "type": "builtin", + "category": "IO", + "hyperparams": { + "SOURCE_TEXT": "<按节点填写>", + "SOURCE_JSON": "<按节点填写>", + "SOURCE_PATH": "<按节点填写>" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 +- 输出 manifest 与真实源路径内容一致,没有漏扫、重扫或错误分类。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 对象存储异常时,先核对 `SOURCE_PATH`、bucket/prefix、endpoint、AK/SK 和目标对象是否真实存在。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 +- 对照真实源目录核验 `entries`、目录数、对象数和增量语义是否正确。 + +## 相关文档 + +- [IO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/结果上传.md b/docs/operators/结果上传.md new file mode 100644 index 00000000..7b5a97d2 --- /dev/null +++ b/docs/operators/结果上传.md @@ -0,0 +1,112 @@ +# 结果上传 + +- 算子 ID:`operator-builtin-upload` +- 类别:`IO` +- 类型:`builtin` + +## 作用 + +收集上游真实输出并生成上传结果清单。 + +## 定位与职责边界 + +- 该算子属于 `IO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它更偏向工作流出口,职责是把平台内部产物同步到外部对象存储或业务数据库。 + +## 平台语义 + +复制或汇总上游产物,生成上传结果清单。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_upload(...)`。 +- 该类末端算子通常没有输出端口,验收重点是外部对象存储中的真实写入结果。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 +- 上传节点只应负责收集并落地结果,不应顺便做复杂转换;转换应在上传前完成。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[结果上传] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **result** / `json` / `bus`:待上传结果 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- 无显式输出端口。通常意味着该节点以落库、上传或触发外部副作用为主。 + +## 关键超参数 + +- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要预先准备连接配置、凭据和目标路径,优先通过平台连接配置而不是把密钥散落到节点超参数里。 + +## 数据通信与存储路径 + +- 该节点读取上游的 `artifact.path`、`dataRef.payloadPath` 或本地文件目录,然后把真实文件上传到对象存储。 +- 平台运行记录里通常只保留上传摘要、目标路径和状态,不重复存放完整文件内容。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "结果上传", + "operatorId": "operator-builtin-upload", + "type": "builtin", + "category": "IO", + "hyperparams": {} +} +``` + +## 输入输出示例 + +- 典型输入:来自上游的 `packages-manifest.json`、`annotation.json`、`clip.mp4` 或任意 `dataRef.payloadPath`。 +- 典型输出:`uploaded-results.json`,记录每个对象的目标路径、对象大小、状态和错误信息。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 +- 外部系统中存在真实副作用,例如对象已上传、文档已写入,而不只是平台内显示成功。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 对象存储异常时,先核对 `SOURCE_PATH`、bucket/prefix、endpoint、AK/SK 和目标对象是否真实存在。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 +- 到目标对象存储前缀核对真实对象、对象大小和命名规则,确认并非只在平台侧生成了摘要。 + +## 相关文档 + +- [IO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/视频切片.md b/docs/operators/视频切片.md new file mode 100644 index 00000000..4a811625 --- /dev/null +++ b/docs/operators/视频切片.md @@ -0,0 +1,110 @@ +# 视频切片 + +- 算子 ID:`operator-builtin-video-slice` +- 类别:`VIDEO` +- 类型:`builtin` + +## 作用 + +将真实视频按固定时长切分为多个 mp4 片段。 + +## 定位与职责边界 + +- 该算子属于 `VIDEO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它位于视频主处理链路中,对共享 PVC、本地文件路径和 manifest 契约依赖较强。 + +## 平台语义 + +使用 ffmpeg/OpenCV 将真实视频切成固定长度片段,并输出 clip manifest。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_video_slicing(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[视频切片] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:视频输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **clips** / `json` / `bus`:切片结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `SEGMENT_DURATION_SECONDS`:类型 `string`,默认值 `300`。每个切片的秒数 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。 + +## 数据通信与存储路径 + +- 视频类处理主要依赖共享 PVC 的本地路径执行,必要时会把 `bos://...` 输入缓存到 `_bos_cache` 或当前节点工作目录。 +- 上下游之间传递的重点是 clip 路径、manifest、annotation JSON、package 目录等引用,而不是媒体字节本身。 + +## 典型上下游编排 + +- 常接在视频筛选之后,并把 clip 结果交给 `视频标注`、`成品打包` 或单独的 `结果上传`。 + +## 节点配置示例 + +```json +{ + "label": "视频切片", + "operatorId": "operator-builtin-video-slice", + "type": "builtin", + "category": "VIDEO", + "hyperparams": { + "SEGMENT_DURATION_SECONDS": "300" + } +} +``` + +## 输入输出示例 + +- 典型输入:带有视频路径的 manifest 或上游过滤结果。 +- 典型输出:`clips-manifest.json` 与多个 `*_clip001.mp4` 文件,clip 实体位于当前节点 `OUTPUT_DIR/clips/` 下。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 视频链路异常时,先在 runner 容器内用 `ffprobe`/`ffmpeg` 验证输入文件是否可读,再检查缓存路径和磁盘空间。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [VIDEO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/视频标注.md b/docs/operators/视频标注.md new file mode 100644 index 00000000..562766f3 --- /dev/null +++ b/docs/operators/视频标注.md @@ -0,0 +1,110 @@ +# 视频标注 + +- 算子 ID:`operator-builtin-video-annotate` +- 类别:`VIDEO` +- 类型:`builtin` + +## 作用 + +基于真实视频片段生成阶段划分、动作标签和对象提示。 + +## 定位与职责边界 + +- 该算子属于 `VIDEO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它位于视频主处理链路中,对共享 PVC、本地文件路径和 manifest 契约依赖较强。 + +## 平台语义 + +读取切片视频并做真实帧间运动分析,输出 annotation JSON 和动作分段。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_video_annotation(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[视频标注] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:切片输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **annotations** / `json` / `bus`:标注结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `FRAME_INTERVAL_SECONDS`:类型 `string`,默认值 `1.0`。采样帧间隔秒数 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。 + +## 数据通信与存储路径 + +- 视频类处理主要依赖共享 PVC 的本地路径执行,必要时会把 `bos://...` 输入缓存到 `_bos_cache` 或当前节点工作目录。 +- 上下游之间传递的重点是 clip 路径、manifest、annotation JSON、package 目录等引用,而不是媒体字节本身。 + +## 典型上下游编排 + +- 上游一般是 `视频切片`,下游则是 `成品打包`、`结果上传` 或人工质检支路。 + +## 节点配置示例 + +```json +{ + "label": "视频标注", + "operatorId": "operator-builtin-video-annotate", + "type": "builtin", + "category": "VIDEO", + "hyperparams": { + "FRAME_INTERVAL_SECONDS": "1.0" + } +} +``` + +## 输入输出示例 + +- 典型输入:来自切片节点的 clip 视频与 `clips-manifest.json`。 +- 典型输出:`annotations-manifest.json` 与逐 clip 的标注 JSON 文件。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 视频链路异常时,先在 runner 容器内用 `ffprobe`/`ffmpeg` 验证输入文件是否可读,再检查缓存路径和磁盘空间。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [VIDEO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/视频质检.md b/docs/operators/视频质检.md new file mode 100644 index 00000000..aa238729 --- /dev/null +++ b/docs/operators/视频质检.md @@ -0,0 +1,116 @@ +# 视频质检 + +- 算子 ID:`operator-builtin-video-quality` +- 类别:`VIDEO` +- 类型:`builtin` + +## 作用 + +基于真实视频帧计算清晰度、抖动度和无手率估计。 + +## 定位与职责边界 + +- 该算子属于 `VIDEO` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 +- 它位于视频主处理链路中,对共享 PVC、本地文件路径和 manifest 契约依赖较强。 + +## 平台语义 + +使用 OpenCV 对真实 mp4 做质检分析,输出 clarity / shake / handOcclusion / passed。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_video_quality(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[视频质检] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:视频清单 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **quality** / `json` / `bus`:质检结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `MIN_CLARITY`:类型 `string`,默认值 `0.8`。清晰度最低阈值 可直接使用默认值,也可在节点级覆盖。 +- `MAX_SHAKE`:类型 `string`,默认值 `0.1`。抖动度最高阈值 可直接使用默认值,也可在节点级覆盖。 +- `MAX_HAND_OCCLUSION`:类型 `string`,默认值 `0.1`。无手率最高阈值 可直接使用默认值,也可在节点级覆盖。 +- `CLARITY_SCALE`:类型 `string`,默认值 `35`。清晰度归一化尺度 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 +- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。 + +## 数据通信与存储路径 + +- 视频类处理主要依赖共享 PVC 的本地路径执行,必要时会把 `bos://...` 输入缓存到 `_bos_cache` 或当前节点工作目录。 +- 上下游之间传递的重点是 clip 路径、manifest、annotation JSON、package 目录等引用,而不是媒体字节本身。 + +## 典型上下游编排 + +- 常位于 `视频过滤` 或 `数据过滤` 之前,为后续筛选提供清晰度、抖动度、遮挡率等量化依据。 + +## 节点配置示例 + +```json +{ + "label": "视频质检", + "operatorId": "operator-builtin-video-quality", + "type": "builtin", + "category": "VIDEO", + "hyperparams": { + "MIN_CLARITY": "0.8", + "MAX_SHAKE": "0.1", + "MAX_HAND_OCCLUSION": "0.1", + "CLARITY_SCALE": "35" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 +- 视频链路异常时,先在 runner 容器内用 `ffprobe`/`ffmpeg` 验证输入文件是否可读,再检查缓存路径和磁盘空间。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [VIDEO 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/采集目录校验.md b/docs/operators/采集目录校验.md new file mode 100644 index 00000000..be644f6d --- /dev/null +++ b/docs/operators/采集目录校验.md @@ -0,0 +1,107 @@ +# 采集目录校验 + +- 算子 ID:`operator-builtin-xplatform-dataset-validate` +- 类别:`DATASET` +- 类型:`builtin` + +## 作用 + +校验采集目录是否包含 meta.json、video_meta.json,以及可选 intrinsics.json。 + +## 定位与职责边界 + +- 该算子属于 `DATASET` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。 + +## 平台语义 + +对 X-platform 数据集索引结果执行真实结构校验,输出每个采集目录的有效性报告。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前实现入口通常可追到 `backend/operator-runner/main.py` 中的 `builtin_collection_validate(...)`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[采集目录校验] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **input** / `json` / `bus`:数据集索引 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **validated** / `json` / `bus`:校验结果 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `REQUIRE_INTRINSICS`:类型 `string`,默认值 `false`。是否要求 intrinsics.json 必须存在 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 适合放在索引之后,用来把不完整目录尽早筛掉,减少后续视频处理成本。 + +## 节点配置示例 + +```json +{ + "label": "采集目录校验", + "operatorId": "operator-builtin-xplatform-dataset-validate", + "type": "builtin", + "category": "DATASET", + "hyperparams": { + "REQUIRE_INTRINSICS": "false" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [DATASET 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/plans/2026-04-08-droboticflow-design.md b/docs/plans/2026-04-08-droboticflow-design.md index 8b7f6d03..277c3239 100644 --- a/docs/plans/2026-04-08-droboticflow-design.md +++ b/docs/plans/2026-04-08-droboticflow-design.md @@ -1,4 +1,4 @@ -# DRoboticFlow Full-Stack Design +# Drobodata Full-Stack Design ## Goal diff --git a/docs/plans/2026-04-08-droboticflow-implementation.md b/docs/plans/2026-04-08-droboticflow-implementation.md index 0897b8b7..2cf5f089 100644 --- a/docs/plans/2026-04-08-droboticflow-implementation.md +++ b/docs/plans/2026-04-08-droboticflow-implementation.md @@ -1,4 +1,4 @@ -# DRoboticFlow Full-Stack Implementation Plan +# Drobodata Full-Stack Implementation Plan **Goal:** Replace the prototype backend with a Go-based MongoDB-backed API and worker stack, wire the frontend to it, and add Kubernetes plus Baidu registry deployment assets. diff --git a/docs/plans/2026-04-14-business-db-write-api-design.md b/docs/plans/2026-04-14-business-db-write-api-design.md new file mode 100644 index 00000000..da0014a2 --- /dev/null +++ b/docs/plans/2026-04-14-business-db-write-api-design.md @@ -0,0 +1,151 @@ +# Business DB External Write API Design + +## Goal + +Add a dedicated REST API that lets approved third parties write into a preconfigured business MongoDB connection using a simple header-based API key, without exposing raw Mongo credentials or the broader database browser surface. + +## Scope + +- add a dedicated external write API under `/api/external/business-db` +- require `X-API-Key` on every external write request +- support three write modes: + - create document + - patch document by `_id` + - upsert document by business keys +- bind each API key to: + - one or more preconfigured `connectionId` values + - a collection whitelist for each allowed connection +- add internal management APIs to create, list, update, rotate, and delete external write tokens +- document third-party usage in the API reference + +## Non-Goals + +- exposing arbitrary MongoDB query syntax to third parties +- allowing third parties to pass raw Mongo URI or database names +- adding frontend UI in this change +- implementing rate limits or IP allowlists + +## Architecture + +The new capability stays separate from the existing database browser endpoints. Internal token management and external write execution both live in the API service, but they use different route families and different trust boundaries. + +Token configuration is stored in the existing singleton settings document so the feature can ship without a new persistence collection or migration path. API keys are never stored in plaintext. The service generates a random key, stores only a SHA-256 hash, and returns the plaintext key once on creation or rotation. + +## Data Model + +### Settings Extension + +Add `externalBusinessWrite` to `SettingsRecord` with: + +- `tokens`: array of token records + +Each token record contains: + +- `id` +- `name` +- `description` +- `enabled` +- `tokenHash` +- `tokenPreview` +- `scopes` +- `lastUsedAt` +- `createdAt` +- `updatedAt` + +Each scope contains: + +- `connectionId` +- `collections` + +## API Design + +### Internal Token Management + +- `GET /api/external/business-db/tokens` +- `POST /api/external/business-db/tokens` +- `PUT /api/external/business-db/tokens/{id}` +- `POST /api/external/business-db/tokens/{id}/rotate` +- `DELETE /api/external/business-db/tokens/{id}` + +These routes manage token metadata and allowed scopes. Create and rotate responses include the newly generated plaintext API key once. List and get responses only expose the masked preview. + +### External Third-Party Write API + +All routes below require header `X-API-Key`. + +- `POST /api/external/business-db/documents` + - create a document +- `PATCH /api/external/business-db/documents/{documentId}` + - patch fields on an existing document by `_id` +- `POST /api/external/business-db/upserts` + - upsert a document by business keys + +All payloads require: + +- `connectionId` +- `collection` + +`PATCH` by `_id` also requires: + +- `patch` + +Upsert also requires: + +- `match` +- `patch` +- optional `setOnInsert` + +## Validation Rules + +- `connectionId` must resolve to an existing configured business Mongo connection +- requested collection must be allowed by the authenticated token scope for that `connectionId` +- `document`, `patch`, `match`, and `setOnInsert` must be JSON objects +- `_id` cannot be overwritten from `patch` +- `match` must be a plain equality object +- top-level keys beginning with `$` are rejected in third-party payloads + +## Security Model + +- third parties can only write through the dedicated external routes +- API keys are compared using constant-time comparison on SHA-256 digests +- raw keys are only shown once on create or rotate +- external routes always resolve database access through preconfigured `connectionId` +- no external route can touch the platform database + +## Store Changes + +Extend the database browser store contract with: + +- patch document by `_id` +- upsert document by business match + +Mongo implementation uses `$set` and `$setOnInsert` with `upsert=true`. Memory implementation mirrors the same semantics for tests. + +## Testing + +Add API tests for: + +- missing API key +- invalid API key +- disabled token +- unauthorized `connectionId` +- unauthorized collection +- create document success +- patch by `_id` success +- upsert create path +- upsert update path +- token create and rotate behavior + +Add store tests for: + +- patch preserves `_id` +- upsert creates when no match exists +- upsert updates when match exists + +## Acceptance Criteria + +- a third party can write to an allowed collection in an allowed `connectionId` using `X-API-Key` +- the same request fails with `401` for missing or invalid key +- the same request fails with `403` when the collection or connection is outside the token scope +- `_id` patch updates only allowed fields and preserves the document identity +- business-key upsert can create and update records without exposing Mongo internals diff --git a/docs/plans/2026-04-14-business-db-write-api-implementation.md b/docs/plans/2026-04-14-business-db-write-api-implementation.md new file mode 100644 index 00000000..c93b58af --- /dev/null +++ b/docs/plans/2026-04-14-business-db-write-api-implementation.md @@ -0,0 +1,164 @@ +# Business DB External Write API Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add a dedicated authenticated REST API for third-party writes into preconfigured business Mongo connections, with collection-scoped API keys and support for create, patch-by-id, and business-key upsert. + +**Architecture:** Extend the singleton settings document to persist hashed external write tokens and their scopes, add dedicated internal token-management endpoints plus dedicated external write endpoints under `/api/external/business-db`, and extend the Mongo/memory database store adapters with patch and upsert helpers. Keep the external API separate from the internal database browser routes so permissions stay narrow. + +**Tech Stack:** Go, chi, MongoDB Go driver, Go test, existing in-memory store test harness + +--- + +### Task 1: Save token configuration in domain and settings + +**Files:** +- Modify: `internal/domain/types.go` +- Modify: `internal/store/helpers.go` + +**Step 1: Write the failing test** + +Add an API test that creates an external write token and verifies the returned payload includes a one-time plaintext API key while the stored settings only keep a hash and preview. + +**Step 2: Run test to verify it fails** + +Run: `go test ./internal/api -run TestExternalBusinessWriteTokenCRUD -v` +Expected: FAIL because the token types and endpoints do not exist yet. + +**Step 3: Write minimal implementation** + +Add token and scope structs to `SettingsRecord`, plus helper defaults and sanitization-ready fields needed by the API. + +**Step 4: Run test to verify it passes** + +Run: `go test ./internal/api -run TestExternalBusinessWriteTokenCRUD -v` +Expected: still failing later in the request path, but no longer on missing types. + +### Task 2: Extend the store with patch and upsert support + +**Files:** +- Modify: `internal/store/store.go` +- Modify: `internal/store/mongo.go` +- Modify: `internal/store/memory.go` + +**Step 1: Write the failing test** + +Add store or API tests that expect: +- patch by `_id` to update selected fields without changing `_id` +- business-key upsert to create when no match exists +- business-key upsert to update when a match exists + +**Step 2: Run test to verify it fails** + +Run: `go test ./internal/api -run 'TestExternalBusinessWrite(DocumentPatch|Upsert)' -v` +Expected: FAIL because the store interface does not expose patch or upsert methods. + +**Step 3: Write minimal implementation** + +Add `PatchDatabaseDocument` and `UpsertDatabaseDocument` to the store interface and implement them in Mongo and memory stores. + +**Step 4: Run test to verify it passes** + +Run: `go test ./internal/api -run 'TestExternalBusinessWrite(DocumentPatch|Upsert)' -v` +Expected: still failing later in the API layer, but store calls now resolve. + +### Task 3: Add internal token management endpoints + +**Files:** +- Modify: `internal/api/server.go` +- Create or modify: `internal/api/database_external_write.go` +- Test: `internal/api/server_test.go` + +**Step 1: Write the failing test** + +Add tests for: +- create token +- list tokens +- rotate token +- reject unknown `connectionId` + +**Step 2: Run test to verify it fails** + +Run: `go test ./internal/api -run TestExternalBusinessWriteTokenCRUD -v` +Expected: FAIL with 404 or validation gaps. + +**Step 3: Write minimal implementation** + +Register token management routes and implement create/list/update/rotate/delete handlers against `SettingsRecord`. + +**Step 4: Run test to verify it passes** + +Run: `go test ./internal/api -run TestExternalBusinessWriteTokenCRUD -v` +Expected: PASS. + +### Task 4: Add authenticated external write endpoints + +**Files:** +- Modify: `internal/api/server.go` +- Create or modify: `internal/api/database_external_write.go` +- Test: `internal/api/server_test.go` + +**Step 1: Write the failing test** + +Add tests for: +- missing API key returns `401` +- invalid API key returns `401` +- disabled token returns `401` +- unauthorized collection returns `403` +- create document returns `201` +- patch by `_id` returns `200` +- business-key upsert returns created and updated outcomes + +**Step 2: Run test to verify it fails** + +Run: `go test ./internal/api -run 'TestExternalBusinessWrite(Auth|Create|DocumentPatch|Upsert)' -v` +Expected: FAIL because the middleware and routes do not exist. + +**Step 3: Write minimal implementation** + +Add: +- token hash generation and constant-time validation +- scope checks for `connectionId` and `collection` +- request validation for plain JSON objects only +- three external write handlers + +**Step 4: Run test to verify it passes** + +Run: `go test ./internal/api -run 'TestExternalBusinessWrite(Auth|Create|DocumentPatch|Upsert)' -v` +Expected: PASS. + +### Task 5: Update docs and verify end-to-end + +**Files:** +- Modify: `docs/api.md` + +**Step 1: Write the failing test** + +No code test needed; use verification commands. + +**Step 2: Run verification** + +Run: +- `go test ./internal/api ./internal/store` +- `go test ./...` +- `npm run build` + +Expected: +- backend tests pass +- repo build stays green + +**Step 3: Write minimal implementation** + +Document token-management and external write API payloads, headers, and sample requests. + +**Step 4: Re-run verification** + +Run: +- `go test ./...` +- `npm run build` +- `npm audit --omit=dev` + +Expected: +- all required tests pass +- build passes +- audit output reviewed for relevant issues diff --git a/docs/plans/2026-04-15-job-only-pvc-data-plane-design.md b/docs/plans/2026-04-15-job-only-pvc-data-plane-design.md new file mode 100644 index 00000000..47a0c438 --- /dev/null +++ b/docs/plans/2026-04-15-job-only-pvc-data-plane-design.md @@ -0,0 +1,155 @@ +# Job-Only PVC Data Plane Design + +## Goal + +在严格 `Kubernetes Job` 执行模型下,引入共享 PVC 和文件化数据面,提升多阶段大文件 DAG 的可用上限。 + +本次设计要解决的是: + +- 多个阶段重复消费同一批大文件时,避免重复从 BOS 全量拉取 +- 避免把大批量 `ExecutionArtifact[]` 塞进环境变量 +- 避免把完整执行结果塞进 stdout 单行 JSON +- 为后续 `dataset/partition/shuffle manifest` 演进建立统一目录约定 + +## Scope + +- 为 worker 和 operator-runner Job 引入共享工作目录语义 +- 为 Job 注入共享 PVC 名称和挂载路径配置 +- 将算子输入从 `INPUT_ARTIFACTS_JSON` 扩展为优先使用 `INPUT_ARTIFACTS_PATH` +- 将算子输出从仅支持 `DROBOTICFLOW_RESULT=` 扩展为优先使用 `RESULT_OUTPUT_PATH` +- 为每个节点输出目录生成 `stage-manifest.json` +- 更新设置页、设置 API、部署模板与文档,使 PVC 配置成为显式系统能力 + +## Non-Goals + +- 本次不实现真正的流式传输、细粒度 backpressure 或共享内存 +- 本次不把调度器改造成按 partition 调度 +- 本次不拆分 `RunRecord` 中的 artifact 持久化模型 +- 本次不删除现有 `ExecutionArtifact` 契约 + +## Constraints + +- 系统仍然只有 `Kubernetes Job` 一种执行模型 +- 不再考虑旧模式兼容或迁移辅助逻辑 +- 不为历史数据做自动修正 +- 对已有小数据节点,保留 `INPUT_ARTIFACTS_JSON` / `DROBOTICFLOW_RESULT` 作为进程内回退契约,但新主链路优先走文件 + +## Design + +### Shared Workspace + +系统引入共享工作区配置: + +- `workspacePVCName` +- `workspaceMountPath` + +默认挂载路径为 `/workspace`。 + +统一目录约定: + +```text +/workspace/runs//nodes// + input-artifacts.json + result.json + stage-manifest.json + outputs/ +``` + +说明: + +- `input-artifacts.json` 由 worker 在节点启动前写入 +- `result.json` 由 operator-runner 写回完整执行结果 +- `stage-manifest.json` 记录本节点输出文件和 artifact 摘要 +- `outputs/` 存放节点真实输出文件 + +### Worker Contract + +worker 执行节点时: + +1. 根据 DAG 收集上游 `ExecutionArtifact[]` +2. 将输入 artifact 写入 `input-artifacts.json` +3. 将 `INPUT_ARTIFACTS_PATH`、`RESULT_OUTPUT_PATH`、`STAGE_MANIFEST_PATH` 注入 Job +4. `OUTPUT_DIR` 指向 `.../outputs` +5. Job 完成后优先读取 `result.json`,stdout 仅保留轻量指针或日志 + +### Operator-Runner Contract + +operator-runner 启动后: + +1. 优先从 `INPUT_ARTIFACTS_PATH` 读取输入 +2. 若未配置文件输入,则回退到 `INPUT_ARTIFACTS_JSON` +3. 正常执行 builtin handler 或显式 command +4. 生成 artifact 列表后写入 `result.json` +5. 额外生成 `stage-manifest.json` +6. stdout 优先输出 `DROBOTICFLOW_RESULT_PATH=` + +### Stage Manifest + +`stage-manifest.json` 是面向后续文件化数据面的轻量摘要,不替代运行记录。 + +建议结构: + +```json +{ + "runId": "run-1", + "workflowId": "wf-1", + "nodeId": "node-1", + "nodeLabel": "视频切片", + "artifactCount": 3, + "artifacts": [ + { + "id": "artifact-1", + "name": "clips.json", + "type": "json", + "path": "/workspace/runs/run-1/nodes/node-1/outputs/clips.json", + "portId": "clips", + "portName": "切片清单" + } + ] +} +``` + +本次阶段 manifest 只做: + +- 可观察性 +- 调试 +- 后续 dataset/partition manifest 的落脚点 + +调度器暂不直接消费它。 + +### K8s Job Mounting + +`CreateJobManifest` 为 operator-runner Job 增加: + +- `volumes` +- `volumeMounts` + +使用单个 PVC,把共享工作区挂进容器。 + +worker 本身也需要挂同一 PVC,才能在启动 Job 前写入输入文件并在 Job 完成后读取结果文件。 + +### Settings and Deployment + +系统设置新增: + +- `k8s.workspacePVCName` +- `k8s.workspaceMountPath` + +部署层新增: + +- `WORKSPACE_PVC_NAME` +- `WORKSPACE_MOUNT_PATH` + +原则: + +- `workspacePVCName` 必须显式配置,表示共享工作区依赖已就绪 +- `workspaceMountPath` 默认 `/workspace` + +## Acceptance Criteria + +- worker 和 operator-runner Job 共享同一个 PVC 挂载 +- 节点输入主链路通过 `input-artifacts.json` 传递 +- 节点结果主链路通过 `result.json` 传递 +- 每个节点输出目录生成 `stage-manifest.json` +- 设置 API、前端设置页和部署模板都能配置 PVC 名称与挂载路径 +- 现有测试覆盖新的文件通道与 Job 挂载行为 diff --git a/docs/plans/2026-04-15-job-only-pvc-data-plane-implementation.md b/docs/plans/2026-04-15-job-only-pvc-data-plane-implementation.md new file mode 100644 index 00000000..1bc65534 --- /dev/null +++ b/docs/plans/2026-04-15-job-only-pvc-data-plane-implementation.md @@ -0,0 +1,93 @@ +# Job-Only PVC Data Plane Implementation Plan + +**Goal:** 在不改变 `Kubernetes Job` 执行模型的前提下,落地共享 PVC、文件化输入/输出通道和阶段 manifest,为大文件多阶段 DAG 建立可持续的数据面基础。 + +**Architecture:** worker 与 operator-runner Job 共享一个工作 PVC;worker 在共享目录写入 `input-artifacts.json`,operator-runner 在共享目录写入 `result.json` 与 `stage-manifest.json`;stdout 和环境变量只保留轻量控制信息。 + +**Tech Stack:** Go, React, Python, Kubernetes Job, PVC, Go test, Python unittest + +--- + +### Task 1: 增加共享工作区配置 + +**Files:** +- Modify: `internal/domain/types.go` +- Modify: `internal/store/helpers.go` +- Modify: `internal/api/server.go` +- Modify: `src/types.ts` +- Modify: `src/lib/settings.js` +- Modify: `src/pages/Settings.tsx` +- Modify: `internal/config/config.go` + +**Implementation:** + +- 在 `k8s` 设置中新增 `workspacePVCName` 和 `workspaceMountPath` +- 为默认值和 sanitize 逻辑补齐新字段 +- 设置页新增共享 PVC 配置项 +- 环境变量新增 `WORKSPACE_PVC_NAME` 与 `WORKSPACE_MOUNT_PATH` + +### Task 2: 为 Kubernetes Job 增加 PVC 挂载 + +**Files:** +- Modify: `internal/k8s/launcher.go` +- Modify: `internal/k8s/launcher_test.go` +- Modify: `k8s/droboticflow.template.yaml` +- Modify: `Dockerfile.api` +- Modify: `Dockerfile.operator-runner` + +**Implementation:** + +- 扩展 launcher 参数和 `LaunchInput` +- 为 operator-runner Job 注入 volume/volumeMount +- 为 worker 和 API deployment 模板挂载同一 PVC +- 为 Job manifest 增加挂载测试 + +### Task 3: 切换 worker 到文件化输入/结果通道 + +**Files:** +- Modify: `internal/worker/runtime.go` +- Modify: `internal/worker/executor.go` +- Modify: `internal/worker/executor_test.go` +- Modify: `internal/worker/runtime_test.go` + +**Implementation:** + +- runtime 新增共享工作区根路径 +- 输出目录改为工作区下的固定目录结构 +- 在节点启动前写入 `input-artifacts.json` +- 注入 `INPUT_ARTIFACTS_PATH`、`RESULT_OUTPUT_PATH`、`STAGE_MANIFEST_PATH` +- 节点完成后优先从结果文件读取执行结果 + +### Task 4: 扩展 operator-runner 文件通道与阶段 manifest + +**Files:** +- Modify: `backend/operator-runner/main.py` +- Modify: `backend/operator-runner/test_main.py` + +**Implementation:** + +- 优先从 `INPUT_ARTIFACTS_PATH` 读取输入 +- 支持把完整执行结果写入 `RESULT_OUTPUT_PATH` +- 新增 `DROBOTICFLOW_RESULT_PATH=` stdout 指针 +- 自动生成 `stage-manifest.json` +- 为文件输入、文件结果、stage manifest 补单测 + +### Task 5: 文档与验证 + +**Files:** +- Modify: `docs/operator-authoring.md` +- Modify: `docs/operations.md` +- Modify: `README.md` + +**Verification:** + +- `python3 -m unittest backend/operator-runner/test_main.py` +- `go test ./internal/k8s ./internal/worker` +- `go test ./...` +- `npm run lint` + +**Done When:** + +- 共享 PVC 配置贯通前后端与部署模板 +- Job 运行主链路不再依赖大体积 env/stdout 传递 artifact 明细 +- 节点输出目录稳定生成 `result.json` 与 `stage-manifest.json` diff --git a/docs/plans/2026-04-15-job-only-pvc-data-plane-phase2-design.md b/docs/plans/2026-04-15-job-only-pvc-data-plane-phase2-design.md new file mode 100644 index 00000000..2e10604d --- /dev/null +++ b/docs/plans/2026-04-15-job-only-pvc-data-plane-phase2-design.md @@ -0,0 +1,181 @@ +# Job-Only PVC Data Plane Phase 2 Design + +## Goal + +在严格 `Kubernetes Job` 模式下,把数据传输从“artifact 文件通道化”推进到“dataset/partition/shuffle manifest 驱动”。 + +本阶段的目标不是实现真正流式或共享内存,而是把当前系统提升到: + +- 大结构化 payload 不再内联进 Mongo run 文档 +- 下游算子默认通过 `dataRef` 在共享 PVC 上读取真实数据 +- 多阶段 DAG 重复消费同一批数据时,不再依赖 artifact 内联大 JSON +- 分配类算子显式产出可复用的 partition/shuffle manifest + +## Design Context + +当前已完成的阶段 1 能力: + +- worker / API / Job 已共享同一个 PVC +- 节点输入通过 `input-artifacts.json` 文件传递 +- 节点结果通过 `result.json` 文件传递 +- 每个节点输出 `stage-manifest.json` + +当前仍然存在的问题: + +- `ExecutionArtifact.Data` 仍可能携带完整大 payload +- 下游算子仍默认把 artifact 当内联 JSON 读取 +- `stage-manifest.json` 只是摘要,还不是数据面契约 +- `distribution` 只是按输出口切分 JSON,没有显式 partition/shuffle manifest +- 数据资产 API 大量依赖 `artifact.Data` 的完整内容 + +## Non-Goals + +- 不实现真正 streaming executor +- 不实现跨 Job 共享内存 +- 不重写调度器为“每个 partition 一个 Job”的执行器 +- 不拆分 Mongo 的 `runs` / `artifacts` 集合 + +## Data Model + +### Artifact DataRef + +对结构化大 payload,引入 `artifact.data.dataRef`: + +```json +{ + "kind": "dataset", + "datasetType": "x-platform-index", + "manifestPath": "/workspace/runs/run-1/nodes/index/outputs/_datasets/dataset-index/manifest.json", + "payloadPath": "/workspace/runs/run-1/nodes/index/outputs/dataset-index.json", + "partitionDir": "/workspace/runs/run-1/nodes/index/outputs/_datasets/dataset-index/partitions", + "partitionCount": 12, + "itemCount": 18214 +} +``` + +约束: + +- `payloadPath` 指向完整结构化 JSON +- `manifestPath` 指向 dataset manifest +- `partitionDir` 下是分区文件 +- `itemCount` / `partitionCount` 用于调试和调度上限评估 + +### Dataset Manifest + +`manifest.json` 记录: + +- `kind=dataset` +- `datasetType` +- `payloadPath` +- `partitionKeys` +- `partitionFiles` +- `summary` + +推荐结构: + +```json +{ + "kind": "dataset", + "datasetType": "x-platform-filtered", + "payloadPath": ".../filtered-videos.json", + "partitionKeys": ["items"], + "partitionFiles": { + "items": [ + ".../partitions/items-p000001.json", + ".../partitions/items-p000002.json" + ] + }, + "summary": { + "count": 1250, + "datasetType": "x-platform-filtered" + } +} +``` + +### Stage Manifest + +`stage-manifest.json` 从“摘要”升级为“阶段数据入口”: + +- 保留 artifact 摘要 +- 新增 `dataRefs` +- 新增 `primaryDataRefs` + +说明: + +- `dataRefs` 列出本节点所有可供后续消费的数据引用 +- `primaryDataRefs` 表示最常用的结构化输出 + +## Runtime Contract + +### Operator Runner Output + +`write_json_artifact(...)` 对以下 payload 一律外置: + +- 包含 `items` +- 包含 `entries` +- 包含 `rows` +- 包含 `collections` +- 包含 `catalogCollections` +- 包含 `catalogItems` +- 包含 `catalogEntries` + +外置后: + +- artifact 仍然保留 `path` +- artifact `data` 只保留小摘要与 `dataRef` +- 完整 payload 通过 `dataRef.payloadPath` 读取 + +### Operator Runner Input + +`load_artifact_payload(...)` 改为: + +1. 先看 `artifact.data.dataRef.payloadPath` +2. 再看 `artifact.data` +3. 最后才看 `artifact.path` + +因此所有内置算子无需知道 payload 是否被外置,读取 helper 会自动回到共享 PVC 上的真实文件。 + +### Shuffle / Distribution + +`builtin_distribution(...)` 输出仍按 output port 切分,但每个输出 payload 会自动得到: + +- dataset manifest +- partition files +- `dataRef` + +如果节点配置 `shuffle=false`,只做顺序切分;否则先洗牌再切分。 + +这意味着 fan-out/fan-in 仍在 Job 级 DAG 内进行,但数据面已经变成可复用的分区清单。 + +## API Contract + +面向 API 的原则: + +- 运行记录页和监控页默认读 artifact 摘要 +- 数据资产、源数据、成品数据等需要完整结构化内容的接口,优先通过 `dataRef.payloadPath` 读取真实 payload + +因此 API 需要新增统一 helper: + +- `artifactStructuredData(artifact)`:返回完整结构化 payload +- `artifactDataRef(artifact)`:返回 `dataRef` + +## Cleanup Rules + +为了保持整洁,本阶段删除这些回退: + +- operator-runner 不再把 `INPUT_ARTIFACTS_JSON` 作为主路径 +- worker 不再依赖 stdout 内联大 JSON 结果 +- 结构化大 payload 不再允许完整塞回 `artifact.Data` + +可以保留的唯一小回退: + +- 小 JSON 仍允许直接保存在 `artifact.Data` + +## Acceptance Criteria + +- 大结构化 payload 默认外置到共享 PVC,并以 `dataRef` 暴露 +- 下游 builtin handler 能透明读取 externalized payload +- `distribution` 输出携带 dataset/partition manifest +- `stage-manifest.json` 明确列出 `dataRefs` +- 数据资产与内容读取 API 可以解析 externalized payload +- 整仓测试通过,且新增测试覆盖 externalized dataset path diff --git a/docs/plans/2026-04-15-job-only-pvc-data-plane-phase2-implementation.md b/docs/plans/2026-04-15-job-only-pvc-data-plane-phase2-implementation.md new file mode 100644 index 00000000..d617f353 --- /dev/null +++ b/docs/plans/2026-04-15-job-only-pvc-data-plane-phase2-implementation.md @@ -0,0 +1,131 @@ +# Job-Only PVC Data Plane Phase 2 Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** 在现有共享 PVC 文件通道基础上,把大结构化 payload 改造成 dataset/partition/shuffle manifest 驱动的数据面。 + +**Architecture:** Python operator-runner 对大 JSON 输出自动 externalize 到共享 PVC,并为 artifact 注入 `dataRef`;Go worker 与 API 只持有轻量 artifact 摘要,真正需要完整 payload 时再通过 `dataRef.payloadPath` 读取;`distribution` 产物天然成为 partition/shuffle manifest。 + +**Tech Stack:** Go, Python, React, Kubernetes Job, PVC, Go test, Python unittest, Node test + +--- + +### Task 1: 扩展 phase 2 设计上下文 + +**Files:** +- Create: `docs/plans/2026-04-15-job-only-pvc-data-plane-phase2-design.md` +- Create: `docs/plans/2026-04-15-job-only-pvc-data-plane-phase2-implementation.md` + +**Step 1: 写设计与实施文档** + +- 明确 `dataRef` +- 明确 dataset manifest / partition manifest +- 明确 API 如何读取 externalized payload + +**Step 2: 保存到仓库** + +无需额外测试。 + +### Task 2: 让 operator-runner externalize 结构化大 payload + +**Files:** +- Modify: `backend/operator-runner/main.py` +- Modify: `backend/operator-runner/test_main.py` + +**Step 1: 写失败测试** + +- `write_json_artifact(...)` 对 dataset 类 payload 生成 `dataRef` +- `load_artifact_payload(...)` 能通过 `dataRef.payloadPath` 读回完整 payload +- `stage-manifest.json` 记录 `dataRefs` + +**Step 2: 跑测试确认失败** + +Run: `python3 -m unittest backend/operator-runner/test_main.py` + +**Step 3: 最小实现** + +- 增加 `externalize_structured_payload` +- 增加 `dataset manifest` / `partition files` +- 增加 `dataRef` +- 更新 `stage-manifest.json` + +**Step 4: 重新跑测试** + +Run: `python3 -m unittest backend/operator-runner/test_main.py` + +### Task 3: 扩展 Go 领域模型与 worker 记录 + +**Files:** +- Modify: `internal/domain/types.go` +- Modify: `internal/worker/executor.go` +- Modify: `internal/worker/runtime.go` +- Modify: `internal/worker/executor_test.go` + +**Step 1: 写失败测试** + +- 节点执行结果记录 `stageManifestPath` +- worker 能从 result/stage manifest 中保留 `dataRef` 摘要 + +**Step 2: 跑测试确认失败** + +Run: `go test ./internal/worker -run 'TestParseExecutionOutput|TestBuildNodeEnv'` + +**Step 3: 最小实现** + +- 给执行记录加 `StageManifestPath` +- 扩展执行结果解析 +- stage/data refs 成为运行时显式字段 + +**Step 4: 重新跑测试** + +Run: `go test ./internal/worker` + +### Task 4: 扩展 API 读取 externalized payload + +**Files:** +- Modify: `internal/api/data_platform.go` +- Modify: `internal/api/data_platform_test.go` + +**Step 1: 写失败测试** + +- `dataset-index.json` 只有 `dataRef` 时,源数据列表仍可返回完整 collections/items +- `packages-manifest.json` / `audit-samples.json` 只有 `dataRef` 时,成品数据与抽检接口仍可工作 + +**Step 2: 跑测试确认失败** + +Run: `go test ./internal/api -run 'TestSourceData|TestPackages|TestSampling'` + +**Step 3: 最小实现** + +- 增加 `artifactStructuredData` +- 所有关键数据资产入口统一经 helper 取 payload + +**Step 4: 重新跑测试** + +Run: `go test ./internal/api` + +### Task 5: 清理旧回退并同步文档 + +**Files:** +- Modify: `docs/operator-authoring.md` +- Modify: `docs/operations.md` +- Modify: `README.md` + +**Step 1: 移除旧主路径文案** + +- 不再把内联大 JSON 当主路径 +- 文档改成 `dataRef + dataset manifest` + +**Step 2: 整体验证** + +Run: +- `python3 -m unittest backend/operator-runner/test_main.py` +- `go test ./...` +- `npm run lint` + +**Done When:** + +- 大结构化 payload 默认 externalize 到 PVC +- 下游算子透明读取 externalized payload +- 数据资产 API 对 externalized payload 可用 +- tests and lint 全通过 diff --git a/docs/plans/2026-04-15-kubernetes-only-runtime-design.md b/docs/plans/2026-04-15-kubernetes-only-runtime-design.md new file mode 100644 index 00000000..35a8ccbe --- /dev/null +++ b/docs/plans/2026-04-15-kubernetes-only-runtime-design.md @@ -0,0 +1,73 @@ +# Kubernetes-Only Runtime Design + +## Goal + +Remove all local-mode execution code and product semantics from Drobodata so the system only supports Kubernetes Job execution. + +## Scope + +- remove local execution branches from the worker runtime +- remove `K8S_ENABLED` and `LOCAL_OPERATOR_RUNNER_CMD` from backend configuration +- remove `k8s.enabled` from persisted settings, API payloads, and frontend types/UI +- update worker startup to require Kubernetes launcher initialization +- update tests so they exercise the Kubernetes Job execution path only +- update docs and sample environment files to describe Kubernetes Job mode as the only supported model + +## Non-Goals + +- adding a data migration that deletes legacy `settings.k8s.enabled` fields from MongoDB +- changing the Python operator-runner contract +- redesigning artifact persistence or BOS semantics in this change + +## Design + +### Runtime + +The worker runtime becomes Kubernetes-only. `Runtime` no longer stores or accepts: + +- `mode` +- `localRunnerCommand` + +Node execution always goes through the Kubernetes launcher. If launcher initialization fails, worker startup fails immediately. + +### Launcher + +`internal/k8s.NewLauncher` no longer accepts an enabled flag. A constructed launcher is always active and backed by Kubernetes client configuration. + +For tests, the worker package will use a launcher interface so runtime tests can provide a Job-launch substitute without reintroducing a user-facing local mode. + +### Configuration + +Backend config removes: + +- `K8S_ENABLED` +- `LOCAL_OPERATOR_RUNNER_CMD` + +Frontend and API settings remove: + +- `k8s.enabled` + +The system still keeps Kubernetes connection fields: + +- `endpoint` +- `namespace` +- `kubeconfig` + +### Frontend + +The settings page no longer exposes a toggle to enable or disable Kubernetes scheduling. It only presents Kubernetes configuration fields. + +### Testing + +Production code will not keep a local execution fallback. Runtime tests will instead inject a launcher test double so all runtime behavior is exercised through the Kubernetes Job abstraction. + +Legacy persisted settings containing `k8s.enabled` remain tolerated in storage because the field can simply be ignored by the new code. + +## Acceptance Criteria + +- worker runtime contains no executable local-mode branch +- `K8S_ENABLED`, `LOCAL_OPERATOR_RUNNER_CMD`, and `k8s.enabled` are removed from supported configuration and UI +- worker startup fails if Kubernetes launcher initialization fails +- frontend settings no longer describe a local/Kubernetes choice +- tests pass using Kubernetes-only execution semantics +- public docs describe Kubernetes Job mode as the only supported execution model diff --git a/docs/plans/2026-04-16-wjzh-large-file-regression-design.md b/docs/plans/2026-04-16-wjzh-large-file-regression-design.md new file mode 100644 index 00000000..558d29d4 --- /dev/null +++ b/docs/plans/2026-04-16-wjzh-large-file-regression-design.md @@ -0,0 +1,72 @@ +# WJZH 大文件算子回归设计 + +## 目标 + +- 删除线上现有全部项目和工作流 +- 基于 `bos://wjzh-data-migration/wjzh-0330-Data-migration` 重建新的真实数据回归项目 +- 重点覆盖视频和大文件处理链路 +- 所有中间结果和最终结果都写回 `wjzh-data-migration` 桶的独立回归前缀 + +## 方案 + +回归项目固定为 1 个项目、4 条工作流: + +1. `01-索引校验上传` + - 自定义 BOS 数据源适配 + - X 平台数据集索引 + - 采集目录校验 + - 结果上传 + - 目的:验证全量目录扫描、`video_metadata.json` 兼容、全量 manifest 上传 + +2. `02-JSON工具链上传` + - 自定义 BOS 数据源适配 + - X 平台数据集索引 + - JSON 字段提取 / 模板渲染 / JSON 过滤 / 映射重命名 / JSON 转 CSV / CSV 转 JSON / 条件分流 / 数据合并 + - 结果上传 + - 目的:验证结构化清单在真实数据上的 JSON/CSV 工具链 + +3. `03-中大型视频切片上传` + - 自定义 BOS 数据源适配 + - X 平台数据集索引 + - 采集目录校验 + - JSON 过滤,精确选取一个真实中大型视频 + - 视频质检 + - 视频切片 + - 结果上传 + - 目的:验证真实视频下载、质检、切片和中间视频上传 + +4. `04-超大视频全链路上传` + - 自定义 BOS 数据源适配 + - X 平台数据集索引 + - 采集目录校验 + - JSON 过滤,精确选取一个 17GB 级真实视频 + - 视频质检 + - 数据过滤 + - 视频切片 + - 视频标注 + - 成品打包 + - 抽检采样 + - 结果上传 + - 目的:验证超大文件视频的完整处理链路 + +## 数据与存储约定 + +- 回归项目名使用时间戳,避免和历史对象混淆 +- 所有上传结果写入: + - `bos://wjzh-data-migration/wjzh-0330-operator-regression//01-index-validate/` + - `bos://wjzh-data-migration/wjzh-0330-operator-regression//02-json-toolchain/` + - `bos://wjzh-data-migration/wjzh-0330-operator-regression//03-large-video/` + - `bos://wjzh-data-migration/wjzh-0330-operator-regression//04-full-chain/` + +## 实现细节 + +- 源节点继续使用自定义 shell 适配器,直接兼容当前 `wjzh` 目录里的 `video_metadata.json` +- 真实大文件链路优先选择单个目标视频,而不是把所有大视频同时送入重处理节点 +- worker 侧把 `WORKSPACE_ROOT` 改为 run 级共享目录,避免同一条工作流里多个节点重复下载同一个 BOS 大视频 + +## 验收标准 + +- 线上项目列表只剩新建的回归项目 +- 新项目下 4 条工作流的最新运行全部为 `completed` +- 4 条工作流都在 `wjzh-data-migration` 桶的独立前缀下产出结果 +- `04-超大视频全链路上传` 能完成 17GB 级真实视频的完整处理 diff --git a/docs/plans/2026-04-17-drobodata-brand-surface-design.md b/docs/plans/2026-04-17-drobodata-brand-surface-design.md new file mode 100644 index 00000000..069ede6f --- /dev/null +++ b/docs/plans/2026-04-17-drobodata-brand-surface-design.md @@ -0,0 +1,38 @@ +# Drobodata Brand Surface Design + +## Goal + +Rename the user-visible product brand from `DroboticFlow` to `Drobodata` without changing internal technical identifiers. + +## In Scope + +- Frontend page titles and headings +- Logo text and product subtitle +- User-visible health badge copy +- Public-facing README and manuals +- Playwright scripts that assert visible page titles +- AI prompt copy that may influence user-visible generated wording + +## Out of Scope + +- Go module path `droboticflow` +- Kubernetes resource names +- Docker image names +- Service names and internal URLs +- Runtime protocol strings such as `DROBOTICFLOW_RESULT_PATH` +- Storage keys, local persistence keys, and database defaults + +## Decision + +Adopt a presentation-only rename: + +- Product name: `Drobodata` +- Product label: `Drobodata Platform` + +Where a runtime health badge previously surfaced internal service text, replace it with branded copy so the UI no longer exposes `droboticflow-api` directly. + +## Validation + +- Search for remaining `DroboticFlow` / `DRoboticFlow` references in user-facing files +- Run frontend lint +- Run existing frontend tests that cover template and scan workflow UI behavior diff --git a/docs/plans/2026-04-17-incremental-loop-scan-workflow-design.md b/docs/plans/2026-04-17-incremental-loop-scan-workflow-design.md new file mode 100644 index 00000000..f0fe8034 --- /dev/null +++ b/docs/plans/2026-04-17-incremental-loop-scan-workflow-design.md @@ -0,0 +1,67 @@ +# 通用增量循环工作流设计 + +**背景** + +当前平台已经具备两块独立能力: + +- `BOS 增量扫描` 节点可以基于 checkpoint 扫描增量对象,并在 run 成功后提交 source state。 +- `streaming` 类型工作流可以按 `schedule.interval` 周期性创建新的 `scheduled run`。 + +但这两块能力还没有在“任意完整 DAG”上打通。结果是: + +- 只有专用扫描工作流能稳定表现出“循环执行”语义。 +- 含 `BOS 增量扫描` 节点的完整处理 DAG,如果没有显式顶层 `scanConfig/schedule`,就会被保存成普通 `batch` 工作流。 +- 回归脚本因此退化成“重复手动运行同一个工作流”,而不是验证真正的周期增量调度。 + +**目标** + +实现并验收下面这类平台能力: + +- 一个工作流里以 `BOS 增量扫描` 作为源节点 +- 顶层配置一个调度间隔 +- worker 按间隔自动创建新的 `scheduled run` +- 每次 run 只把本轮 BOS 增量数据推给后续整条 DAG +- run 成功后提交 checkpoint,下一轮继续基于 source state 扫描 + +**设计** + +1. 后端把“图里包含增量源节点”视为一种扫描工作流候选。 + +- 在创建/更新工作流时,如果图中存在 `BOS 增量扫描` 节点,就允许从节点自身的 `hyperparams/data` 自动推导顶层 `scanConfig`。 +- 如果同时配置了 `schedule.interval > 0`,则自动把工作流归一化为: + - `kind=scan` + - `type=streaming` + - `schedule.sourcePath` 与源节点对齐 +- 如果没有开启周期执行,则仍保留为手动批处理,但顶层 `scanConfig` 仍要和节点保持一致,方便后续切换。 + +2. 前端编辑器把“图里存在增量源节点”识别为扫描型工作流。 + +- 不再要求必须先有顶层 `kind=scan/scanConfig` 才显示扫描配置区。 +- 只要图里存在 `BOS 增量扫描`,就显示: + - 触发方式 + - 调度间隔 + - Bucket / Prefix / Connection / Endpoint / 密钥 + - FULL / INCREMENTAL / Pattern +- 保存、运行、导出模板时统一先把图和顶层扫描配置对齐。 + +3. 模板与复制链路也支持这类工作流。 + +- 工作流导出模板、模板创建/更新,都要保留并归一化顶层 `kind/scanConfig/schedule`。 +- 模板实例化后仍是“可周期增量执行”的完整 DAG。 + +4. 真实回归改成“单工作流自动循环”验收。 + +- 回归脚本不再通过手动重复运行同一个 workflow ID 来模拟增量。 +- 改为创建一条 `streaming + interval` 的完整增量工作流。 +- 通过向 BOS 前缀分轮次注入新目录,验证: + - 第一轮:发现新增并处理 + - 第二轮:只处理新增加的目录 + - 第三轮:无新增时仍能完成空增量 run + +**验收标准** + +- 任意含 `BOS 增量扫描` 节点的完整工作流都可配置为周期执行。 +- worker 会自动周期创建新的 `scheduled run`,而非依赖人工重复点击运行。 +- 每轮只处理 BOS 增量数据,source state 正确提交和续扫。 +- 模板、复制、保存、运行链路全部支持该模型。 +- 使用 `bos://wjzh-data-migration/wjzh-0330-Data-migration` 做真实循环验收,所有相关工作流最终通过。 diff --git a/docs/plans/2026-04-17-incremental-loop-scan-workflow-implementation.md b/docs/plans/2026-04-17-incremental-loop-scan-workflow-implementation.md new file mode 100644 index 00000000..ec7daa4e --- /dev/null +++ b/docs/plans/2026-04-17-incremental-loop-scan-workflow-implementation.md @@ -0,0 +1,148 @@ +# 通用增量循环工作流 Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** 让任意包含 `BOS 增量扫描` 源节点的完整工作流都能作为周期性增量处理 DAG 自动运行。 + +**Architecture:** 后端在创建/更新工作流及模板时,从图中的增量源节点自动推导顶层 `scanConfig` 和调度属性;前端编辑器将“含增量源节点”的工作流视为扫描型工作流并同步配置;回归脚本改成真实周期调度而不是重复手动运行。 + +**Operational Notes (implemented):** +- 扫描型工作流默认 `maxConcurrentRuns=1`,避免同一增量链路多轮 `scheduled run` 重叠。 +- API 侧代理 `/worker-api/*` 到集群内 `droboticflow-worker` Service,便于统一入口下检查 worker 健康、stats 与手动 `process-once`。 + +**Tech Stack:** Go API/store, React workflow editor, Python regression script, Kubernetes worker, BOS + +--- + +### Task 1: 为通用增量循环工作流补失败测试 + +**Files:** +- Modify: `/Users/longtaowu/workspace/droboticflow/internal/api/server_test.go` +- Modify: `/Users/longtaowu/workspace/droboticflow/tests/frontend/scan-workflows.test.mjs` + +**Step 1: 写后端失败测试** + +- 创建一个包含 `BOS 增量扫描` 节点、带 `schedule.interval`、但没有显式顶层 `scanConfig` 的完整工作流 payload。 +- 期望 API 自动补出 `kind=scan`、`scanConfig`、`schedule.sourcePath`。 + +**Step 2: 跑后端测试确认失败** + +Run: `go test ./internal/api -run 'TestProjectWorkflowCreateInfersScanConfigFromIncrementalSourceNode|TestWorkflowUpdateInfersScanConfigFromIncrementalSourceNode|TestTemplateCreateInfersScanConfigFromIncrementalSourceNode' -v` + +**Step 3: 写前端失败测试** + +- 验证没有顶层 `scanConfig` 时,仍可从 `BOS 增量扫描` 节点推导扫描配置和 ready 状态。 + +**Step 4: 跑前端测试确认失败** + +Run: `node --test tests/frontend/scan-workflows.test.mjs` + +### Task 2: 实现后端工作流/模板自动推导扫描壳 + +**Files:** +- Modify: `/Users/longtaowu/workspace/droboticflow/internal/domain/workflow.go` +- Modify: `/Users/longtaowu/workspace/droboticflow/internal/api/server.go` + +**Step 1: 增加通用扫描源节点识别 helper** + +- 增加判断节点是否为 `BOS 增量扫描` 的公共 helper。 + +**Step 2: 实现从节点推导 scan config 的 helper** + +- 从节点 `hyperparams`、`bucket/prefix/pattern/connectionId`、`schedule.interval` 生成 `scanConfigRequest`。 + +**Step 3: 在工作流 create/update 中接入** + +- 含增量源节点时自动归一化: + - `kind` + - `scanConfig` + - `schedule` + - `type` + - `maxConcurrentRuns=1`(若用户未显式指定) + +**Step 4: 在模板 create/update 中接入** + +- 保持与工作流一致的归一化行为。 + - 同样默认 `maxConcurrentRuns=1` + +**Step 5: 跑后端测试** + +Run: `go test ./internal/api -v` + +### Task 3: 实现前端编辑器对通用增量循环工作流的支持 + +**Files:** +- Modify: `/Users/longtaowu/workspace/droboticflow/src/lib/scan-workflows.js` +- Modify: `/Users/longtaowu/workspace/droboticflow/src/pages/WorkflowEditor.tsx` +- Modify: `/Users/longtaowu/workspace/droboticflow/tests/frontend/scan-workflows.test.mjs` + +**Step 1: 从图节点推导扫描配置** + +- 在 `scan-workflows.js` 里增加从 `BOS 增量扫描` 节点推导 scan config 的 helper。 + +**Step 2: 调整编辑器识别逻辑** + +- 只要图里有增量源节点,就显示扫描配置 UI。 + +**Step 3: 调整保存/运行/导出模板逻辑** + +- 保存前统一同步顶层扫描壳和画布节点。 +- 前端草稿和扫描壳默认显示 `maxConcurrentRuns=1` + +**Step 4: 跑前端测试** + +Run: +- `node --test tests/frontend/scan-workflows.test.mjs tests/frontend/templates.test.mjs tests/frontend/template-settings.test.mjs` +- `npm run lint` + +### Task 4: 回归脚本改成真实周期增量验收 + +**Files:** +- Modify: `/Users/longtaowu/workspace/droboticflow/scripts/wjzh_operator_regression.py` + +**Step 1: 改掉重复手动运行同一 workflow ID 的逻辑** + +- 对增量验收工作流,改成真正的 `streaming + interval` 工作流。 + +**Step 2: 实现分轮次 BOS 注入和 run 观测** + +- round-1 注入新增目录并等待 scheduled run 完成。 +- round-2 再注入新增目录并验证仅处理新增。 +- round-3 不注入数据,验证空增量 run。 + +**Step 3: 本地脚本 dry-run / lint** + +Run: `python3 -m py_compile /Users/longtaowu/workspace/droboticflow/scripts/wjzh_operator_regression.py` + +### Task 5: 部署并做真实集群循环验收 + +**Files:** +- Modify as needed during bugfix loop + +**Step 1: 运行本地全量测试** + +Run: `go test ./...` + +**Step 2: 构建并部署需要更新的镜像** + +- API +- 前端(如果独立构建) +- 如需要再发布 worker / operator-runner +- 若 live worker 未对公网暴露,也可通过 API 入口使用 `/worker-api/health|stats|process-once` + +**Step 3: 真实执行回归** + +Run: + +```bash +API_BASE_URL=http://114.111.24.223:30080 \ +STORAGE_CONNECTION_ID=storage-d69ba0e9824a9abd216eced490f0aed3 \ +python3 /Users/longtaowu/workspace/droboticflow/scripts/wjzh_operator_regression.py --delete-regression-projects +``` + +**Step 4: 如失败则修复后重复执行,直到通过** + +**Step 5: 验收结果整理** + +- 给出最终项目 ID / 工作流 ID / 增量循环工作流 ID +- 给出各轮 scheduled run 结果和 BOS 输出路径 diff --git a/docs/plans/2026-04-20-directory-incremental-loop-acceptance.md b/docs/plans/2026-04-20-directory-incremental-loop-acceptance.md new file mode 100644 index 00000000..c32961ef --- /dev/null +++ b/docs/plans/2026-04-20-directory-incremental-loop-acceptance.md @@ -0,0 +1,153 @@ +# 目录级循环增量工作流验收记录 + +日期:`2026-04-20` + +## 目标 + +验证平台支持以下能力: + +- 一个工作流周期性执行 BOS 增量扫描 +- 增量单位为目录,而不是对象文件 +- 新目录首次出现时,将该目录下全部内容推给后续流程 +- 老目录新增单个文件时,不应重新触发整目录 + +## 最终验收对象 + +- 项目:`project-f06a3295cc32f3aaba2a0fcdc2a98d23` +- 工作流:`workflow-866f6e6fabb343c2fa6fd2171361cf7a` +- 名称:`12-循环增量目录验收` +- 数据源: + `bos://wjzh-data-migration/wjzh-0330-Data-migration/__codex_incremental_directory_acceptance__/20260420T0002` + +最终冻结后的工作流状态: + +- `type=batch` +- `status=active` +- `schedule.interval=0` +- `scanConfig.enabled=false` + +保留冻结版本的目的是避免继续自动触发,同时保留验收结果。 + +## 运行前修复 + +在真实验收前,定位到一个 live 环境问题: + +- worker 的 `chooseImage()` 会把节点里配置的 `droboticflow-operator-runner:*` 镜像统一替换成 worker 自己的默认 `OPERATOR_RUNNER_IMAGE` +- 这会导致工作流节点虽然配置了新 runner,实际运行仍使用旧镜像 +- 目录级增量逻辑因此无法真正生效 + +处理方式: + +- 构建并推送新镜像: + `ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:20260420-dirincrement-amd64` +- 将 live `droboticflow-worker` 的 `OPERATOR_RUNNER_IMAGE` 更新到该镜像 +- 滚动完成后重新创建验收工作流 + +## 工作流配置 + +源节点核心参数: + +- `DATA_SOURCE=BOS` +- `SOURCE_PATH=bos://wjzh-data-migration/wjzh-0330-Data-migration/__codex_incremental_directory_acceptance__/20260420T0002` +- `SCAN_MODE=INCREMENTAL` +- `INCREMENTAL_UNIT=DIRECTORY` +- `INCREMENTAL_DIRECTORY_DEPTH=1` + +顶层工作流参数: + +- `type=streaming` +- `kind=scan` +- `schedule.interval=20` +- `maxConcurrentRuns=1` +- `scanConfig.scanMode=INCREMENTAL` +- `scanConfig.incrementalUnit=DIRECTORY` +- `scanConfig.directoryDepth=1` + +节点链路: + +```mermaid +flowchart LR + A["BOS 增量扫描"] --> B["X 平台数据集索引"] + B --> C["采集目录校验"] +``` + +## 5 轮真实验收结果 + +### 第 1 轮 + +- Run:`run-34485343e7f4c373f04898f2cacc20dd` +- 操作:预先创建 2 个新目录 `loop-a-002`、`loop-b-002` +- 结果: + - `newCount=8` + - `listedCount=8` + - `incrementalUnit=DIRECTORY` + - `directoryDepth=1` + - `seenFingerprints=["loop-a-002","loop-b-002"]` + +### 第 2 轮 + +- Run:`run-453a9aca8a39e687ab9b300b5e9a3237` +- 操作:无新增目录 +- 结果: + - `newCount=0` + - `listedCount=8` + - `incrementalUnit=DIRECTORY` + - `directoryDepth=1` + +### 第 3 轮 + +- Run:`run-0653050fd3e04cde4fa936a1ab6e2ddd` +- 操作:在已有目录 `loop-a-002` 下新增 `notes.txt` +- 结果: + - `newCount=0` + - `listedCount=9` + - `incrementalUnit=DIRECTORY` + - `directoryDepth=1` + +说明: + +- 这轮是关键验证 +- 目录内新增文件不会把已有目录重新当成“新目录” + +### 第 4 轮 + +- Run:`run-cf6b308760ff66d152a036b149ab0c55` +- 操作:新增 2 个新目录 `loop-c-002`、`loop-d-002` +- 结果: + - `newCount=8` + - `listedCount=17` + - `incrementalUnit=DIRECTORY` + - `directoryDepth=1` + +### 第 5 轮 + +- Run:`run-57b4b0c25060b52221e54dd51f37a529` +- 操作:无新增目录 +- 结果: + - `newCount=0` + - `listedCount=17` + - `incrementalUnit=DIRECTORY` + - `directoryDepth=1` + +## 最终结论 + +目录级循环增量工作流已经真实通过验收,满足以下语义: + +- 新目录首次出现时,整目录内容会进入后续流程 +- 老目录中单独新增文件,不会重新触发整目录 +- 空增量轮次可以稳定完成,不会误报新增 +- source state 会按目录名记录 `seenFingerprints` + +## 清理结果 + +验收过程中创建过的旧临时工作流均已删除,不再残留: + +- `workflow-59898c1cbf431a156733cf6b869a8e1d` +- `workflow-976bcfc00e074fa2b9e6a56b6ec31d53` +- `workflow-dae6d63c04af0e7646b722aee42adfc6` + +验收收尾时确认: + +- `/api/runs?status=running` 为空 +- `/api/runs?status=queued` 为空 + diff --git a/docs/plans/2026-04-21-external-mongo-only-k8s-deploy-design.md b/docs/plans/2026-04-21-external-mongo-only-k8s-deploy-design.md new file mode 100644 index 00000000..f19f1f3c --- /dev/null +++ b/docs/plans/2026-04-21-external-mongo-only-k8s-deploy-design.md @@ -0,0 +1,120 @@ +# 外部 Mongo Only K8s 部署设计 + +## 背景 + +当前项目线上实际已经使用外部 MongoDB: + +- `MONGO_URI=mongodb://:@:27017/` +- `MONGO_DB=droboticflow` + +但仓库中的 Kubernetes 部署链路仍然保留了两套互相冲突的语义: + +1. `scripts/deploy-k8s.sh` 在未显式提供 `MONGO_URI` 时,会自动生成集群内地址格式的 URI:`mongodb://...@mongodb:27017/...` +2. `k8s/droboticflow.template.yaml` 仍然包含集群内 `mongodb` 的 Service、Deployment 和初始化脚本 + +这会带来两个问题: + +- 部署默认行为与线上真实架构不一致 +- 后续维护者容易误以为平台支持或推荐内置 Mongo 运行模式 + +## 目标 + +把 Kubernetes 部署链路收敛为单一语义: + +- 平台部署必须显式使用外部 MongoDB +- 不再生成或回退到集群内 `mongodb:27017` 风格的 `MONGO_URI` +- 不再在主部署模板中创建集群内 MongoDB 资源 + +## 非目标 + +本次不处理: + +- 本地开发默认值 `mongodb://127.0.0.1:27017` +- 测试专用的 `k8s/business-mongodb.test.yaml` +- 业务 Mongo 写入算子的外部连接管理能力 + +换句话说,这次只治理“主 Kubernetes 部署链路”,不扩大到所有运行场景。 + +## 方案 + +### 1. 部署脚本强制要求显式外部 `MONGO_URI` + +`scripts/deploy-k8s.sh` 改为: + +- 不再生成 `MONGO_ROOT_USERNAME`、`MONGO_ROOT_PASSWORD`、`MONGO_APP_USERNAME`、`MONGO_APP_PASSWORD` +- 不再生成 `mongodb://...@mongodb:27017/...` 回退 URI +- 如果未提供 `MONGO_URI`,直接失败 +- 如果 `MONGO_URI` 命中明显的集群内/本地地址模式,也直接失败 + +建议拦截的模式: + +- `mongodb://...@mongodb:27017` +- `mongodb://localhost` +- `mongodb://127.0.0.1` +- `mongodb://*.svc` + +这样做的目的不是做 URI 语法校验,而是避免再次把部署指向集群内临时库。 + +### 2. K8s 模板移除内置 Mongo 资源 + +`k8s/droboticflow.template.yaml` 中删除: + +- `mongodb-init` ConfigMap +- `mongodb` Service +- `mongodb` Deployment + +保留: + +- `droboticflow-mongo-secret` 中的 `MONGO_URI` +- API/worker 对 `MONGO_URI` 和 `MONGO_DB` 的环境变量注入 + +这样模板会更符合实际运行形态: + +- 平台应用连接外部数据库 +- 集群内不再承担数据库生命周期管理 + +### 3. 文档同步 + +需要同步更新: + +- `README.md` +- `docs/deployment.md` +- `.env.example` + +文档应明确说明: + +- K8s 部署必须显式提供外部 `MONGO_URI` +- 不再支持通过主部署脚本创建平台主库 Mongo 容器 +- `MONGO_IMAGE`、root/app 密码等变量不再属于主部署链路 + +## 风险与兼容性 + +### 风险 + +- 之前依赖脚本自动创建集群内 Mongo 的人,会在部署时直接失败 + +### 风险控制 + +- 这是故意的 fail-fast 行为 +- 比起静默连到错误数据库,部署时明确报错更安全 + +### 兼容性边界 + +- 已部署线上环境不会因代码修改自动切换数据库 +- 只有重新执行部署脚本时,才会受到新约束影响 + +## 测试策略 + +至少覆盖: + +1. 渲染后的 manifest 中不再包含 `mongodb` 资源 +2. `deploy-k8s.sh` 在缺少 `MONGO_URI` 时直接失败 +3. `deploy-k8s.sh` 在传入 `mongodb:27017` 等集群内地址时直接失败 +4. `deploy-k8s.sh` 在传入外部 URI 时能继续执行 secret 与 manifest 流程 +5. 文档不再提及主部署链路中的 `MONGO_IMAGE` + +## 预期结果 + +改完之后,项目主 Kubernetes 部署链路会明确表达一件事: + +“平台依赖外部 MongoDB,部署者必须显式提供它;仓库不再伪装支持集群内主库模式。” diff --git a/docs/plans/2026-04-21-external-mongo-only-k8s-deploy-implementation.md b/docs/plans/2026-04-21-external-mongo-only-k8s-deploy-implementation.md new file mode 100644 index 00000000..991b7d39 --- /dev/null +++ b/docs/plans/2026-04-21-external-mongo-only-k8s-deploy-implementation.md @@ -0,0 +1,233 @@ +# External Mongo Only K8s Deploy Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Remove cluster-local MongoDB from the main Kubernetes deployment path and require an explicit external `MONGO_URI`. + +**Architecture:** Keep API and worker runtime injection unchanged, but remove in-cluster Mongo resources from the manifest template and fail deployment early unless an external Mongo URI is provided. Update docs and examples so the repository communicates a single deployment model. + +**Tech Stack:** Bash, Kubernetes YAML, Go runtime config context, Markdown docs + +--- + +### Task 1: Remove in-cluster Mongo resources from the main manifest template + +**Files:** +- Modify: `/Users/longtaowu/workspace/droboticflow/k8s/droboticflow.template.yaml` +- Verify: `/Users/longtaowu/workspace/droboticflow/scripts/render-manifests.sh` + +**Step 1: Remove Mongo-specific resources** + +Delete these sections from the template: + +- `ConfigMap` named `mongodb-init` +- `Service` named `mongodb` +- `Deployment` named `mongodb` + +Keep these sections intact: + +- API env injection for `MONGO_URI` +- worker env injection for `MONGO_URI` +- config map values for `MONGO_DB` + +**Step 2: Render manifests and inspect output** + +Run: + +```bash +./scripts/render-manifests.sh | rg -n "^kind:|name: mongodb|mongodb-init|MONGO_URI" +``` + +Expected: + +- `MONGO_URI` env injection remains +- no `name: mongodb` +- no `mongodb-init` + +**Step 3: Commit-ready check** + +Confirm the rendered output no longer contains cluster-local Mongo resources. + +### Task 2: Make deployment fail fast without explicit external Mongo URI + +**Files:** +- Modify: `/Users/longtaowu/workspace/droboticflow/scripts/deploy-k8s.sh` + +**Step 1: Remove local Mongo fallback logic** + +Delete logic for: + +- `MONGO_IMAGE` +- `MONGO_ROOT_USERNAME` +- `MONGO_ROOT_PASSWORD` +- `MONGO_APP_USERNAME` +- `MONGO_APP_PASSWORD` +- default `mongodb://...@mongodb:27017/...` URI synthesis +- rollout wait for `deployment/mongodb` + +**Step 2: Add explicit validation** + +Add a bash validation block that: + +- errors if `MONGO_URI` is empty after reading existing secret and env +- errors if URI matches local/cluster patterns such as `mongodb:27017`, `localhost`, `127.0.0.1`, `.svc` + +Expected message style: + +- clear +- actionable +- mentions external MongoDB is required + +**Step 3: Keep secret update path** + +Keep secret creation/update for: + +- `MONGO_URI` + +Remove secret writes for: + +- root/app usernames and passwords + +**Step 4: Verify failure path** + +Run: + +```bash +env -i PATH="$PATH" HOME="$HOME" K8S_NAMESPACE=droboticflow bash /Users/longtaowu/workspace/droboticflow/scripts/deploy-k8s.sh +``` + +Expected: + +- non-zero exit +- explicit error that `MONGO_URI` is required + +### Task 3: Keep manifest rendering aligned with the new deploy model + +**Files:** +- Modify: `/Users/longtaowu/workspace/droboticflow/scripts/render-manifests.sh` + +**Step 1: Remove unused Mongo image input** + +Delete: + +- `MONGO_IMAGE` env default +- `__MONGO_IMAGE__` replacement + +**Step 2: Verify render script still succeeds** + +Run: + +```bash +/Users/longtaowu/workspace/droboticflow/scripts/render-manifests.sh >/tmp/droboticflow-rendered.yaml +``` + +Expected: + +- command succeeds +- rendered file contains API/worker resources only for app runtime plus existing non-Mongo resources + +### Task 4: Update developer-facing configuration examples + +**Files:** +- Modify: `/Users/longtaowu/workspace/droboticflow/.env.example` +- Modify: `/Users/longtaowu/workspace/droboticflow/README.md` +- Modify: `/Users/longtaowu/workspace/droboticflow/docs/deployment.md` + +**Step 1: Update `.env.example`** + +Replace local placeholder: + +- from `mongodb://127.0.0.1:27017` +- to an explicit external placeholder such as `mongodb://:@:27017/` + +Add a short note comment if needed. + +**Step 2: Update `README.md` deployment guidance** + +Remove references to: + +- mirroring `mongo:7` +- `MONGO_IMAGE` +- cluster-local Mongo deployment assumptions + +Add language that: + +- Kubernetes deployment requires external MongoDB +- `MONGO_URI` must be provided explicitly + +**Step 3: Update `docs/deployment.md`** + +Remove: + +- Mongo image build/push steps for the main deployment +- `MONGO_IMAGE` as a required deployment variable + +Add: + +- external Mongo requirement +- example export of `MONGO_URI` +- note that the main manifest no longer creates a platform Mongo instance + +### Task 5: Regression checks + +**Files:** +- Verify only + +**Step 1: Bash syntax checks** + +Run: + +```bash +bash -n /Users/longtaowu/workspace/droboticflow/scripts/deploy-k8s.sh +bash -n /Users/longtaowu/workspace/droboticflow/scripts/render-manifests.sh +``` + +Expected: + +- both commands exit `0` + +**Step 2: Render output check** + +Run: + +```bash +/Users/longtaowu/workspace/droboticflow/scripts/render-manifests.sh > /tmp/droboticflow-rendered.yaml +rg -n "name: mongodb|mongodb-init|__MONGO_IMAGE__" /tmp/droboticflow-rendered.yaml +``` + +Expected: + +- no matches + +**Step 3: Failure-mode check for deploy script** + +Run: + +```bash +env -i PATH="$PATH" HOME="$HOME" K8S_NAMESPACE=droboticflow bash /Users/longtaowu/workspace/droboticflow/scripts/deploy-k8s.sh +``` + +Expected: + +- non-zero exit +- clear `MONGO_URI` required error + +**Step 4: Safe positive-path dry run** + +Run with a placeholder external URI and mocked kubectl path if needed, or inspect the script logic directly to ensure it would proceed past validation. + +Expected: + +- validation accepts non-local Mongo URI format + +**Step 5: Final diff review** + +Run: + +```bash +git diff -- /Users/longtaowu/workspace/droboticflow/k8s/droboticflow.template.yaml /Users/longtaowu/workspace/droboticflow/scripts/deploy-k8s.sh /Users/longtaowu/workspace/droboticflow/scripts/render-manifests.sh /Users/longtaowu/workspace/droboticflow/.env.example /Users/longtaowu/workspace/droboticflow/README.md /Users/longtaowu/workspace/droboticflow/docs/deployment.md +``` + +Expected: + +- only the external-Mongo-only deployment changes appear diff --git a/docs/plans/2026-04-21-workflow-schedule-modes-and-run-env-design.md b/docs/plans/2026-04-21-workflow-schedule-modes-and-run-env-design.md index 3cf1d07c..28dd799a 100644 --- a/docs/plans/2026-04-21-workflow-schedule-modes-and-run-env-design.md +++ b/docs/plans/2026-04-21-workflow-schedule-modes-and-run-env-design.md @@ -84,13 +84,13 @@ - 建议新增: - `WORKFLOW_PREVIOUS_RUN_TIMESTAMP` - `WORKFLOW_CURRENT_RUN_TIMESTAMP` -- 格式统一为“Unix 秒级时间戳字符串” +- 格式统一为 `Asia/Shanghai` 时区下的 RFC3339 时间字符串 - 首次运行时: - - `WORKFLOW_PREVIOUS_RUN_TIMESTAMP=0` -- 选择 Unix 秒级字符串的原因: - - `0` 能作为首轮缺省值,格式与正常值一致 - - Python / Shell / Go 都容易消费 - - 不依赖展示时区格式 + - `WORKFLOW_PREVIOUS_RUN_TIMESTAMP=""` +- 选择可读时间字符串的原因: + - 前端展示和实际注入保持一致 + - 算子脚本无需自行把 Unix 时间戳转回本地时间 + - 排障时可以直接从环境变量看出真实触发时刻 6. 前端回显放在“工作流编辑器的节点配置面板”,而不是 Operators 管理页。 @@ -123,7 +123,7 @@ - 历史只含 `schedule.interval` 的工作流不回归 - worker 能正确创建“每 N 小时”与“每 N 天固定时刻”的 `scheduled run` - 首次运行时节点可读到: - - `WORKFLOW_PREVIOUS_RUN_TIMESTAMP=0` - - `WORKFLOW_CURRENT_RUN_TIMESTAMP=` + - `WORKFLOW_PREVIOUS_RUN_TIMESTAMP=""` + - `WORKFLOW_CURRENT_RUN_TIMESTAMP=` - 非首次运行时,两个变量值按 workflow 级运行顺序正确滚动 - WorkflowEditor 节点侧能够只读回显这两个系统环境变量 diff --git a/docs/plans/2026-04-21-workflow-schedule-modes-and-run-env-implementation.md b/docs/plans/2026-04-21-workflow-schedule-modes-and-run-env-implementation.md index f1cab4d2..599bf717 100644 --- a/docs/plans/2026-04-21-workflow-schedule-modes-and-run-env-implementation.md +++ b/docs/plans/2026-04-21-workflow-schedule-modes-and-run-env-implementation.md @@ -181,8 +181,8 @@ go test ./internal/worker -run 'TestEnqueueStreamingWorkflows|TestBuildNodeEnv' - 新增: - `WORKFLOW_PREVIOUS_RUN_TIMESTAMP` - `WORKFLOW_CURRENT_RUN_TIMESTAMP` -- 值统一转成 Unix 秒级字符串 -- `PreviousRunAt` 为空时,注入 `0` +- 值统一转成 `Asia/Shanghai` 时区下的 RFC3339 时间字符串 +- `PreviousRunAt` 为空时,注入空字符串 **Step 2: 文档同步** @@ -223,8 +223,8 @@ go test ./internal/worker -run 'TestBuildNodeEnv' -v - `WORKFLOW_PREVIOUS_RUN_TIMESTAMP` - `WORKFLOW_CURRENT_RUN_TIMESTAMP` - 标注: - - `Unix 秒级时间戳` - - `首次运行 previous=0` + - `Asia/Shanghai RFC3339 时间` + - `首次运行 previous=""` - `运行时自动注入` **Step 4: 跑前端测试** diff --git a/docs/plans/2026-04-22-scheduled-run-history-and-subtree-rerun-design.md b/docs/plans/2026-04-22-scheduled-run-history-and-subtree-rerun-design.md new file mode 100644 index 00000000..96501662 --- /dev/null +++ b/docs/plans/2026-04-22-scheduled-run-history-and-subtree-rerun-design.md @@ -0,0 +1,65 @@ +# Scheduled Run History And Subtree Rerun Design + +## Scope + +This change set covers three related behaviors: + +1. `WORKFLOW_PREVIOUS_RUN_TIMESTAMP` is only injected for `scheduled` runs. +2. Scheduled workflows expose a run-history entry inside `WorkflowEditor`. +3. A failed run can spawn a child run that re-executes from any selected node through its downstream subtree. + +## Env Variable Rules + +- `WORKFLOW_PREVIOUS_RUN_TIMESTAMP` + - Inject only when `run.trigger == scheduled` + - Value format: `Asia/Shanghai` RFC3339 string + - First scheduled run uses empty string +- `WORKFLOW_CURRENT_RUN_TIMESTAMP` + - Inject for all run triggers + - Value format: `Asia/Shanghai` RFC3339 string + +## Run History UX + +- Only workflows with enabled `schedule` show a `周期运行记录` entry. +- The entry opens an in-editor history panel. +- The panel lists runs for the current workflow sorted by `createdAt desc`. +- Clicking a run loads that run into the existing canvas/run-detail context. + +## Subtree Rerun Model + +- Subtree rerun never mutates the original failed run. +- The system creates a child run with: + - `trigger = rerun_subtree` + - `parentRunId = source failed run id` + - `restartFromNodeId = selected node id` +- Execution semantics: + - Re-run the selected node and every downstream reachable node. + - Reuse completed upstream executions from the source run when they are dependencies of the rerun subtree. + - Do not copy unrelated branches outside the dependency boundary. + +## Eligibility + +- The action is only exposed for failed runs. +- The user may select any node in the failed workflow graph. +- Backend validation rejects requests whose required upstream baseline cannot be satisfied from completed executions in the source run. + +## Data Model + +- Extend run trigger enum with `rerun_subtree`. +- Extend `RunRecord` and `RunSummary` with `restartFromNodeId`. + +## API + +- `GET /api/workflows/{id}/runs` + - List run summaries for one workflow +- `POST /api/runs/{id}/rerun-subtree` + - Body: `{ "nodeId": "" }` + - Returns the queued child run + +## Acceptance + +- Manual and retry runs do not receive `WORKFLOW_PREVIOUS_RUN_TIMESTAMP`. +- Scheduled runs still receive both workflow time envs. +- Scheduled workflows can browse their own run history inside the editor. +- Failed runs can create subtree rerun child runs from a selected node. +- The child run preserves upstream completed dependency context and re-executes only the chosen subtree. diff --git a/docs/plans/2026-04-22-scheduled-run-history-and-subtree-rerun-implementation.md b/docs/plans/2026-04-22-scheduled-run-history-and-subtree-rerun-implementation.md new file mode 100644 index 00000000..5c51895d --- /dev/null +++ b/docs/plans/2026-04-22-scheduled-run-history-and-subtree-rerun-implementation.md @@ -0,0 +1,68 @@ +# Scheduled Run History And Subtree Rerun Implementation + +## Step 1: Extend Run Domain + +- Add `RunTriggerRerunSubtree` +- Add `restartFromNodeId` to: + - `domain.RunRecord` + - `domain.RunSummary` + - frontend `ExecutionRun` + +## Step 2: Gate Scheduled Previous Timestamp + +- Update worker env injection: + - `WORKFLOW_PREVIOUS_RUN_TIMESTAMP` only for `scheduled` + - `WORKFLOW_CURRENT_RUN_TIMESTAMP` unchanged for all triggers +- Update frontend preview helper to mirror the same rule +- Update tests + +## Step 3: Add Workflow Run History API + +- Add `GET /api/workflows/{id}/runs` +- Reuse store run listing, then summarize and filter per workflow in API +- Include `restartFromNodeId` in summary payload + +## Step 4: Add Subtree Rerun API + +- Add `POST /api/runs/{id}/rerun-subtree` +- Validate: + - source run exists + - source run is failed + - workflow exists + - node exists in workflow +- Build dependency sets: + - rerun subtree + - required upstream baseline +- Ensure every required upstream node is completed in source run +- Create queued child run and seed copied baseline executions + +## Step 5: Update Worker Runtime Compatibility + +- No new execution engine is required. +- Seeded completed baseline executions must be sufficient for existing dependency resolution. +- Rebuild aggregated run logs/artifacts/resourceSeries from seeded executions. + +## Step 6: Workflow Editor UI + +- Add `周期运行记录` entry for scheduled workflows +- Add history panel with run list +- Selecting a run updates the current active run context +- On failed runs, expose `从该节点重新执行下游` in node detail + +## Step 7: Tests + +- Backend: + - scheduled env gating + - workflow run history listing + - subtree rerun creation and validation +- Frontend: + - env preview gating + - workflow run history data loading helpers if needed + +## Step 8: Verification + +- `go test ./internal/api ./internal/worker -count=1` +- `node --test tests/frontend/workflow-system-env.test.mjs` +- `npm run build` +- deploy to `wjzh` +- smoke: `/api/health`, `/api/operators` diff --git a/docs/platform-usage-manual.zh-CN.md b/docs/platform-usage-manual.zh-CN.md new file mode 100644 index 00000000..e4c9f4b0 --- /dev/null +++ b/docs/platform-usage-manual.zh-CN.md @@ -0,0 +1,302 @@ +# Drobodata 平台使用手册 + +本文档面向平台使用者,覆盖项目、工作流、模板、运行、数据管理、对象存储和常见排障。文中截图来自当前线上环境,流程说明基于当前代码与真实运行行为。 + +## 1. 平台结构总览 + +平台里最核心的 6 个对象是: + +- `项目`:一组相关工作流的容器 +- `工作流`:由多个算子节点组成的 DAG +- `模板`:可复用的工作流蓝本 +- `运行`:某次工作流执行实例 +- `Artifact`:算子之间传递的数据产物 +- `数据源`:BOS、目录、JSON、CSV、Mongo 等外部输入 + +典型处理链路如下: + +```mermaid +flowchart LR + A["项目"] --> B["工作流"] + B --> C["运行"] + C --> D["节点执行"] + D --> E["Artifacts / Stage Manifest"] + E --> F["上传结果 / 数据管理 / 成品包"] +``` + +## 2. 主要页面 + +### 2.1 项目页 + +项目页用于管理项目、查看项目内工作流,以及复制工作流。 + +![项目列表](./assets/manuals/platform-projects.png) + +常见操作: + +- 新建项目 +- 进入项目详情 +- 在项目内新建工作流 +- 复制已有工作流 +- 删除工作流 + +说明: + +- 现在“复制工作流”会完整保留工作流顶层参数 +- 包括 `kind`、`scanConfig`、并发、超时、重试等 +- 对扫描型工作流,数据源、`bucket/prefix`、`accessKeyId/secretAccessKey` 也会一并复制 + +### 2.2 模板页 + +模板页用于沉淀标准流程、复制模板,以及从模板创建工作流。 + +![模板页](./assets/manuals/platform-templates.png) + +常见操作: + +- 创建空模板 +- 将已有工作流保存为模板 +- 复制模板 +- 从模板创建新工作流 + +说明: + +- 现在“保存为模板”和“从模板创建工作流”都保留完整参数 +- 包括 `scanConfig`、执行参数、节点、边、调度配置 + +### 2.3 工作流编辑器 + +工作流编辑器用于配置节点、连接节点、保存和运行工作流。 + +![工作流编辑器](./assets/manuals/platform-workflow-editor.png) + +编辑器里的关键概念: + +- 左侧:算子面板 +- 中间:画布 +- 右侧或弹层:节点参数配置 +- 顶部:保存、运行、导出模板等动作 + +当前约束: + +- 同一个输入端口只允许一条入边 +- 如果需要上传多个不同结果,必须为每个结果添加独立的“结果上传”算子 + +### 2.4 数据管理页 + +数据管理页用于查看扫描导入的数据、目录索引、成品包等。 + +![数据管理](./assets/manuals/platform-data-assets.png) + +适合查看: + +- 采集目录 +- 视频、JSON、配置文件 +- 成品包 +- 运行产出 manifest +- 上传后的 BOS 路径 + +### 2.5 算子列表页 + +算子列表页用于查看当前内置算子、分类、描述和输入输出能力。 + +![算子列表](./assets/manuals/platform-operators.png) + +## 3. 典型使用流程 + +### 3.1 创建项目和工作流 + +1. 在项目页点击“新建项目” +2. 进入项目详情页 +3. 点击“新建工作流” +4. 打开工作流编辑器 +5. 拖入需要的算子并配置参数 +6. 保存工作流 + +### 3.2 保存为模板并复用 + +1. 在项目详情页选择现有工作流 +2. 点击“保存为模板” +3. 在模板页选择该模板 +4. 点击“使用模板” +5. 选择目标项目 +6. 系统会创建一条新的工作流实例 + +当前行为: + +- 工作流复制 +- 模板导出 +- 模板实例化 + +以上三条链路都完整保留: + +- `kind` +- `scanConfig` +- `bucket/prefix/endpoint` +- `accessKeyId/secretAccessKey` +- `maxConcurrentRuns` +- `nodeTimeoutSeconds` +- `runTimeoutSeconds` +- `maxRetryAttempts` +- `retryBackoffSeconds` +- `schedule` +- `nodes` +- `edges` + +### 3.3 执行工作流 + +1. 打开工作流 +2. 点击“运行” +3. 查看执行状态 +4. 进入运行详情页查看节点状态、日志和产物 + +运行状态通常有: + +- `pending` +- `running` +- `completed` +- `failed` + +## 4. 扫描型工作流 + +扫描型工作流用于轮询或增量嗅探 BOS 路径下的新目录/新文件。 + +```mermaid +flowchart LR + A["BOS 增量扫描"] --> B["数据集索引"] + B --> C["目录校验"] + C --> D["后续处理工作流"] +``` + +建议配置项: + +- `dataSource=BOS` +- `bucket` +- `prefix` +- `endpoint` +- `accessKeyId` +- `secretAccessKey` +- `scanMode=INCREMENTAL` + +当前平台能力: + +- 支持增量 source state +- 支持发现新目录后触发后续处理 +- 支持在共享 PVC 上保存阶段 manifest 和中间态 + +目录级循环增量工作流推荐配置: + +- `type=streaming` +- `kind=scan` +- `schedule.interval=20` 或更长 +- `maxConcurrentRuns=1` +- `scanMode=INCREMENTAL` +- `incrementalUnit=DIRECTORY` +- `directoryDepth=1` + +目录级模式语义: + +- `OBJECT`:按文件对象判增量,新增文件或覆盖更新文件都会触发 +- `DIRECTORY`:按目录判增量,只有新目录首次出现时才触发,并把该目录下全部内容推给后续节点 +- `DIRECTORY` 模式下,如果老目录里只新增了一个文件,不会重新把这个目录当成新目录触发 + +一个稳定的目录级循环链路通常是: + +```mermaid +flowchart LR + A["BOS 增量扫描 (DIRECTORY)"] --> B["X 平台数据集索引"] + B --> C["采集目录校验"] +``` + +真实验收结论: + +- 已在 `bos://wjzh-data-migration/wjzh-0330-Data-migration/__codex_incremental_directory_acceptance__/20260420T0002` 上完成 5 轮真实验收 +- 最终工作流:`workflow-866f6e6fabb343c2fa6fd2171361cf7a` +- 5 轮结果如下: + - 第 1 轮:新增 2 个新目录,`newCount=8`,`listedCount=8` + - 第 2 轮:无新增,`newCount=0`,`listedCount=8` + - 第 3 轮:老目录新增 `notes.txt`,仍然 `newCount=0`,`listedCount=9` + - 第 4 轮:新增 2 个新目录,`newCount=8`,`listedCount=17` + - 第 5 轮:无新增,`newCount=0`,`listedCount=17` + +这说明目录级模式已经满足“新增目录即触发整目录内容进入后续流程;老目录新增单文件不触发整目录重跑”的要求。 + +注意: + +- 增量扫描依赖 source state +- 如果你重置 source state,下一次会重新把已有对象视为新对象 +- 线上要确保 `droboticflow-worker` 的 `OPERATOR_RUNNER_IMAGE` 已切到支持目录级增量的 runner;否则节点里即使写了 `INCREMENTAL_UNIT=DIRECTORY`,实际运行也可能仍落到旧镜像 + +## 5. 运行和结果怎么看 + +每次运行至少有三层信息: + +1. `工作流级状态` +2. `节点级状态` +3. `artifact / manifest / 日志` + +建议排查顺序: + +1. 先看工作流是否 `failed` +2. 再看失败节点 +3. 再看该节点日志 +4. 再看该节点输出的 `stage-manifest` +5. 最后回溯上游 artifact + +## 6. 数据传输与上传原则 + +平台当前是 `Kubernetes Job + 共享 PVC + BOS` 的混合数据面: + +- 小型结构化数据:通过 JSON artifact 传递 +- 大文件和中间态:通过共享 PVC 上的 manifest / payloadPath 传递 +- 长期持久化结果:通过 BOS 上传保存 + +使用建议: + +- 不要让多个上游结果并到同一个输入端口 +- 需要多个上传结果时,拆成多个“结果上传”节点 +- 大视频/大 JSON 不要依赖浏览器直接预览全部内容 + +## 7. 常见问题 + +### 7.1 为什么复制工作流后参数丢了? + +旧版本前端只复制了基础字段,后端又把 `scanConfig` 里的密钥清空了。当前版本已经修复,复制/模板链路会完整保留参数。 + +### 7.2 为什么一个上传节点不能接多个上游结果? + +当前平台规定:一个输入端口只能有一条入边。多个结果请拆成多个上传节点,否则语义不清晰,也会被前后端拒绝保存/运行。 + +### 7.3 为什么大视频处理慢? + +大文件工作流主要受以下因素影响: + +- BOS 拉取速度 +- ffmpeg 切片模式 +- PVC 吞吐 +- 多节点并发 + +当前平台已经优先对切片走 `stream copy`,失败时再回退重编码。 + +### 7.4 为什么删除目录时会报 `NoSuchKey`? + +旧版本把 BOS 的目录占位 key 错误裁掉了末尾 `/`。当前版本已修复,递归删目录会按真实 key 删除。 + +## 8. 推荐实践 + +- 标准流程优先沉淀为模板 +- 需要复用数据源时,用扫描型工作流保存 `scanConfig` +- 对大文件结果,尽量上传到 BOS,再让下游消费远程 artifact +- 对多分支输出,给每个最终产物单独配上传节点 +- 对增量目录处理,优先使用 `BOS 增量扫描 + source state` + +## 9. 相关文档 + +- [API 文档](./api.md) +- [运维文档](./operations.md) +- [自定义算子手册](./custom-operator-manual.zh-CN.md) +- [自定义算子 Markdown 体系](./custom-operators/README.md) +- [内置算子手册](./builtin-operators-manual.zh-CN.md) +- [内置算子分册索引](./operators/README.md) +- [目录级循环增量工作流验收记录](./plans/2026-04-20-directory-incremental-loop-acceptance.md) +- [算子开发文档](./operator-authoring.md) diff --git a/docs/wjzh-operator-regression.md b/docs/wjzh-operator-regression.md new file mode 100644 index 00000000..4aa7d458 --- /dev/null +++ b/docs/wjzh-operator-regression.md @@ -0,0 +1,132 @@ +# WJZH 真实数据算子回归 + +这个回归入口面向真实生产环境,不再使用临时 shell 适配脚本里的明文 `AK/SK`,而是优先复用系统里的 BOS `storage connection`。 + +入口脚本: + +```bash +python3 scripts/wjzh_operator_regression.py +``` + +## 回归内容 + +脚本会创建 1 个回归项目,并依次执行 4 条工作流: + +1. `01-索引校验上传` +2. `02-JSON工具链上传` +3. `03-中大型视频切片上传` +4. `04-超大视频全链路上传` + +数据源固定为: + +```text +bos://wjzh-data-migration/wjzh-0330-Data-migration +``` + +默认输出前缀: + +```text +bos://wjzh-data-migration/wjzh-0330-operator-regression +``` + +## 生产前置 + +必须满足下面 3 项,否则脚本会在 preflight 或 runtime smoke 阶段直接失败: + +1. API / worker 已部署为当前仓库版本 + 关键标志:`/api/settings` 返回里应包含 `k8s.workspacePVCName`。 + +2. 共享工作区 PVC 已配置 + 至少要有: + - `WORKSPACE_PVC_NAME` + - `WORKSPACE_MOUNT_PATH` + +3. 已存在可用的 BOS storage connection + 连接需要指向 `wjzh-data-migration` 桶。 + +## 推荐执行顺序 + +先做纯配置预检: + +```bash +API_BASE_URL=http://:30080 \ +STORAGE_CONNECTION_ID= \ +python3 scripts/wjzh_operator_regression.py --preflight-only +``` + +预检通过后,再做带运行时烟测的正式回归: + +```bash +API_BASE_URL=http://:30080 \ +STORAGE_CONNECTION_ID= \ +python3 scripts/wjzh_operator_regression.py --delete-regression-projects +``` + +如果确实要先清空所有项目,再重建回归项目,显式加: + +```bash +--delete-all-projects +``` + +这个选项是破坏性的,默认不会启用。 + +## 连接配置方式 + +### 方式 1:使用已有 storage connection + +推荐: + +```bash +API_BASE_URL=http://:30080 \ +STORAGE_CONNECTION_ID= \ +python3 scripts/wjzh_operator_regression.py --preflight-only +``` + +也可以按名称查找: + +```bash +API_BASE_URL=http://:30080 \ +STORAGE_CONNECTION_NAME=wjzh-data-migration \ +python3 scripts/wjzh_operator_regression.py --preflight-only +``` + +### 方式 2:由脚本创建 storage connection + +仅当环境里还没有现成连接时使用: + +```bash +API_BASE_URL=http://:30080 \ +STORAGE_CONNECTION_NAME=wjzh-data-migration \ +CREATE_CONNECTION_IF_MISSING=1 \ +BOS_ENDPOINT=https://bj.bcebos.com \ +BOS_BUCKET=wjzh-data-migration \ +BOS_ACCESS_KEY_ID= \ +BOS_SECRET_ACCESS_KEY= \ +python3 scripts/wjzh_operator_regression.py --preflight-only +``` + +## 脚本做了什么 + +1. 校验 `/api/settings` 是否满足当前版本的数据面要求。 +2. 校验并解析 BOS storage connection。 +3. 校验回归所需内置算子是否齐全。 +4. 先创建一条最小 `connectionId` 烟测工作流: + `BOS 增量扫描 -> X 平台数据集索引 -> 采集目录校验 -> 结果上传` +5. 只有烟测通过后,才会继续创建 4 条正式回归工作流。 + +## 如果脚本失败 + +重点看两类错误: + +1. `settings.k8s.workspacePVCName is missing` + 说明线上还没部署到当前仓库这版,或者共享 PVC 没配好。 + +2. `runtime-smoke-failed` + 说明 API 里虽然已经存了 `connectionId`,但 worker / operator-runner 在真实运行时没有正确解析 BOS 连接,通常是线上部署版本落后于当前代码。 + +## 验收标准 + +1. 运行时烟测工作流成功。 +2. 4 条正式回归工作流全部 `completed`。 +3. 工作流定义里不再保存明文 `BOS_ACCESS_KEY_ID` / `BOS_SECRET_ACCESS_KEY`。 +4. 结果写入 `wjzh-data-migration` 桶下独立时间戳前缀。 diff --git a/go.mod b/go.mod index c9c591d0..a741456c 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( ) require ( + github.com/baidubce/bce-sdk-go v0.9.264 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect diff --git a/go.sum b/go.sum index f7aa3606..b128bb66 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/baidubce/bce-sdk-go v0.9.264 h1:qqOW8OEjNIqFOzULGTum7m0y6NBebG9+2rJBdBgdQso= +github.com/baidubce/bce-sdk-go v0.9.264/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/index.html b/index.html index 9742593b..04444ba5 100644 --- a/index.html +++ b/index.html @@ -3,7 +3,7 @@ - DroboticFlow + Drobodata
diff --git a/internal/ai/client.go b/internal/ai/client.go index 5e7f2650..b33a9727 100644 --- a/internal/ai/client.go +++ b/internal/ai/client.go @@ -224,19 +224,21 @@ func (client *client) GenerateWorkflow(ctx context.Context, description string, } systemPrompt := strings.TrimSpace(` -You are an expert workflow architect for DroboticFlow. +You are an expert workflow architect for Drobodata. Return a single JSON object only. Do not wrap it in markdown. Your task is to turn the user's requirement into a workflow PLAN using only the provided operators. Rules: 1. Use only operatorId values from the provided operator catalog. -2. Nodes must be valid for DroboticFlow execution and connected in a runnable order. +2. Nodes must be valid for Drobodata execution and connected in a runnable order. 3. For each edge, sourcePort and targetPort must match the real port names from the chosen operators. 4. Prefer batch workflows unless the user explicitly asks for streaming behavior. 5. Keep names short and production-oriented. 6. If a node needs operator-specific configuration, put it in hyperparamOverrides using real hyperparameter keys. -7. Output schema: +7. Never connect multiple upstream edges into the same target input port. One input port accepts only one incoming edge. +8. If the workflow needs to upload multiple result sets, create one upload node per result instead of merging them into the same upload input. +9. Output schema: { "name": "workflow short name", "description": "short summary", @@ -340,6 +342,9 @@ Rules: TargetHandle: targetHandle, }) } + if conflicts := domain.FindInputPortConflicts(nodes, edges); len(conflicts) > 0 { + return nil, fmt.Errorf("AI generated a workflow with multiple upstream edges feeding the same input port; split those branches or add separate upload operators for each result") + } return &GeneratedWorkflow{ Name: chooseValue(strings.TrimSpace(plan.Name), "AI 生成工作流"), @@ -355,7 +360,7 @@ func (client *client) GenerateOperator(ctx context.Context, description string) } systemPrompt := strings.TrimSpace(` -You are an expert DroboticFlow operator designer. +You are an expert Drobodata operator designer. Return one JSON object only. Do not wrap it in markdown. Allowed categories: IO, CV, AI, MATH, HUMAN, FLOW diff --git a/internal/api/builtin_operators.go b/internal/api/builtin_operators.go new file mode 100644 index 00000000..268d7178 --- /dev/null +++ b/internal/api/builtin_operators.go @@ -0,0 +1,690 @@ +package api + +import ( + "context" + "strings" + + "droboticflow/internal/domain" +) + +const ( + builtinRealSourceOperatorID = "operator-builtin-real-source" + builtinBOSScanOperatorID = "operator-builtin-bos-incremental-scan" + builtinModelOperatorID = "operator-builtin-model-inference" + builtinMergeOperatorID = "operator-builtin-merge" + builtinUploadOperatorID = "operator-builtin-upload" + builtinHumanReviewOperatorID = "operator-builtin-human-review" + builtinDirectoryScanOperatorID = "operator-builtin-directory-scan" + builtinCSVToJSONOperatorID = "operator-builtin-csv-to-json" + builtinJSONToCSVOperatorID = "operator-builtin-json-to-csv" + builtinJSONExtractOperatorID = "operator-builtin-json-field-extract" + builtinJSONFilterOperatorID = "operator-builtin-json-filter" + builtinJSONMapOperatorID = "operator-builtin-json-map-rename" + builtinTemplateRenderOperatorID = "operator-builtin-template-render" + builtinRouteOperatorID = "operator-builtin-conditional-route" + builtinDatasetIndexOperatorID = "operator-builtin-xplatform-dataset-index" + builtinDatasetValidateOperatorID = "operator-builtin-xplatform-dataset-validate" + builtinVideoQualityOperatorID = "operator-builtin-video-quality" + builtinVideoFilterOperatorID = "operator-builtin-video-filter" + builtinVideoSliceOperatorID = "operator-builtin-video-slice" + builtinVideoAnnotateOperatorID = "operator-builtin-video-annotate" + builtinPackageOperatorID = "operator-builtin-package" + builtinSamplingOperatorID = "operator-builtin-audit-sample" + builtinMongoWriteOperatorID = "operator-builtin-mongodb-write" + builtinLeRobotSummaryOperatorID = "operator-builtin-lerobot-summary" + builtinROSBagSummaryOperatorID = "operator-builtin-rosbag-summary" + builtinShellCommandOperatorID = "operator-builtin-shell-command" +) + +var reservedMongoCollections = map[string]struct{}{ + "projects": {}, + "operators": {}, + "operatorTombstones": {}, + "workflows": {}, + "templates": {}, + "runs": {}, + "sourceStates": {}, + "auditTasks": {}, + "samplingTasks": {}, + "users": {}, + "storageConnections": {}, + "mongoConnections": {}, + "settings": {}, +} + +func jsonPort(id string, name string, description string) domain.PortDefinition { + return domain.PortDefinition{ID: id, Name: name, Type: domain.PortTypeJSON, Transmission: "bus", Description: description} +} + +func filePort(id string, name string, description string) domain.PortDefinition { + return domain.PortDefinition{ID: id, Name: name, Type: domain.PortTypeFile, Transmission: "bus", Description: description} +} + +func builtinOperatorRecords(now string, runnerImage string) []domain.OperatorRecord { + return []domain.OperatorRecord{ + { + ID: builtinRealSourceOperatorID, + Name: "真实数据源", + Type: "builtin", + Category: "IO", + Description: "从 SOURCE_TEXT、SOURCE_JSON、SOURCE_PATH 或上游输入生成真实源数据清单。", + Inputs: nil, + Outputs: []domain.PortDefinition{jsonPort("out-1", "manifest", "源数据清单")}, + Image: runnerImage, + SkillDefinition: "工作流起点。读取真实输入源并生成结构化清单,供下游节点消费。", + Hyperparams: []domain.HyperParam{ + {Key: "SOURCE_TEXT", Value: "", Description: "直接注入文本内容", Type: "string"}, + {Key: "SOURCE_JSON", Value: "", Description: "直接注入 JSON 内容", Type: "string"}, + {Key: "SOURCE_PATH", Value: "", Description: "从路径读取真实数据", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinBOSScanOperatorID, + Name: "BOS 增量扫描", + Type: "builtin", + Category: "IO", + Description: "按 bucket/prefix 增量扫描 BOS,仅输出本次新增对象并提交 checkpoint 给 runtime。", + Inputs: nil, + Outputs: []domain.PortDefinition{jsonPort("out-1", "manifest", "增量对象清单")}, + Image: runnerImage, + SkillDefinition: "统一工作流里的有状态源算子。每次 run 只产出 BOS 中新增的对象,状态由 runtime 在 run 成功后提交。", + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "BOS", Description: "固定为 BOS", Type: "string"}, + {Key: "SOURCE_PATH", Value: "", Description: "完整 bos://bucket/prefix 路径", Type: "string"}, + {Key: "BOS_ENDPOINT", Value: "https://bj.bcebos.com", Description: "BOS Endpoint", Type: "string"}, + {Key: "BOS_ACCESS_KEY_ID", Value: "", Description: "BOS Access Key ID", Type: "string"}, + {Key: "BOS_SECRET_ACCESS_KEY", Value: "", Description: "BOS Secret Access Key", Type: "string"}, + {Key: "SCAN_MODE", Value: "INCREMENTAL", Description: "固定为 INCREMENTAL", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinModelOperatorID, + Name: "模型推理", + Type: "builtin", + Category: "AI", + Description: "执行真实节点输入汇总、命令调用或模型镜像推理。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "输入数据")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "result", "推理结果")}, + Image: runnerImage, + SkillDefinition: "执行通用推理节点;若提供 COMMAND 或 SCRIPT,会真实执行命令并收集输出。", + Hyperparams: []domain.HyperParam{ + {Key: "COMMAND", Value: "", Description: "容器内执行的命令", Type: "string"}, + {Key: "SCRIPT", Value: "", Description: "可执行脚本内容", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinMergeOperatorID, + Name: "数据合并", + Type: "builtin", + Category: "MATH", + Description: "将多个 JSON 输入合并成统一结果。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "left", "左输入"), jsonPort("in-2", "right", "右输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "merged", "合并结果")}, + Image: runnerImage, + SkillDefinition: "对多个 JSON 输入执行真实合并。", + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinUploadOperatorID, + Name: "结果上传", + Type: "builtin", + Category: "IO", + Description: "收集上游真实输出并生成上传结果清单。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "result", "待上传结果")}, + Outputs: nil, + Image: runnerImage, + SkillDefinition: "复制或汇总上游产物,生成上传结果清单。", + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinHumanReviewOperatorID, + Name: "人工审核", + Type: "builtin", + Category: "HUMAN", + Description: "将真实输入送入人工审核并等待审核动作恢复流程。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "candidate", "候选数据")}, + Outputs: []domain.PortDefinition{ + jsonPort("approved", "approved", "审核通过"), + jsonPort("rejected", "rejected", "审核拒绝"), + }, + Image: runnerImage, + SkillDefinition: "基于真实输入生成审核任务并等待人工处理。", + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinDirectoryScanOperatorID, + Name: "目录扫描", + Type: "builtin", + Category: "IO", + Description: "对真实目录做文件统计、扩展名分布和样例文件扫描。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "目录输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "summary", "目录概览")}, + Image: runnerImage, + SkillDefinition: "读取 SOURCE_PATH 或上游 source manifest,输出真实目录概览。", + Hyperparams: []domain.HyperParam{ + {Key: "SOURCE_PATH", Value: "", Description: "待扫描目录", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinCSVToJSONOperatorID, + Name: "CSV 转 JSON", + Type: "builtin", + Category: "DATA", + Description: "将真实 CSV/TSV 文件转换成结构化 JSON 行列表。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "CSV/TSV 输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "table", "JSON 行列表")}, + Image: runnerImage, + SkillDefinition: "读取 SOURCE_PATH 或上游 source manifest 指向的 csv/tsv 文件,输出完整 JSON 行数据。", + Hyperparams: []domain.HyperParam{ + {Key: "SOURCE_PATH", Value: "", Description: "csv/tsv 文件路径", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinJSONToCSVOperatorID, + Name: "JSON 转 CSV", + Type: "builtin", + Category: "DATA", + Description: "将 JSON 行数据导出为真实 CSV 文件。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 行列表")}, + Outputs: []domain.PortDefinition{filePort("out-1", "csv", "CSV 文件")}, + Image: runnerImage, + SkillDefinition: "把上游 JSON items/entries 或单条对象导出为真实 CSV 文件,并附带摘要。", + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinJSONExtractOperatorID, + Name: "JSON 字段提取", + Type: "builtin", + Category: "DATA", + Description: "按字段路径从 JSON 输入中提取指定值。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "extracted", "提取结果")}, + Image: runnerImage, + SkillDefinition: "根据 FIELD_PATH 从 JSON 输入中提取字段。", + Hyperparams: []domain.HyperParam{ + {Key: "FIELD_PATH", Value: "", Description: "点分字段路径,例如 metrics.score", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinJSONFilterOperatorID, + Name: "JSON 过滤", + Type: "builtin", + Category: "DATA", + Description: "按字段条件过滤 JSON 列表。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "filtered", "过滤结果")}, + Image: runnerImage, + SkillDefinition: "按 FIELD_PATH + OPERATOR + EXPECTED 条件真实过滤 JSON items。", + Hyperparams: []domain.HyperParam{ + {Key: "FIELD_PATH", Value: "", Description: "待判断字段路径", Type: "string"}, + {Key: "OPERATOR", Value: "eq", Description: "eq/ne/contains/gt/gte/lt/lte/exists/missing", Type: "string"}, + {Key: "EXPECTED", Value: "", Description: "字符串比较值", Type: "string"}, + {Key: "EXPECTED_JSON", Value: "", Description: "JSON 格式比较值,优先于 EXPECTED", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinJSONMapOperatorID, + Name: "JSON 映射重命名", + Type: "builtin", + Category: "DATA", + Description: "按映射表批量重命名 JSON 字段。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "mapped", "映射结果")}, + Image: runnerImage, + SkillDefinition: "根据 FIELD_MAP_JSON 真实执行字段名重映射。", + Hyperparams: []domain.HyperParam{ + {Key: "FIELD_MAP_JSON", Value: "{}", Description: "JSON 字段映射,例如 {\"old\":\"new\"}", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinTemplateRenderOperatorID, + Name: "文本模板渲染", + Type: "builtin", + Category: "UTIL", + Description: "使用上游 JSON 上下文渲染真实文本模板文件。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")}, + Outputs: []domain.PortDefinition{filePort("out-1", "rendered", "渲染后的文本文件")}, + Image: runnerImage, + SkillDefinition: "支持 {{field}} 和 Python format 风格占位符,输出真实文本文件。", + Hyperparams: []domain.HyperParam{ + {Key: "TEMPLATE_TEXT", Value: "", Description: "模板文本", Type: "string"}, + {Key: "OUTPUT_NAME", Value: "rendered.txt", Description: "输出文件名", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinRouteOperatorID, + Name: "条件分流", + Type: "builtin", + Category: "LOGIC", + Description: "按字段条件将输入拆分到 matched / unmatched 两个真实输出端口。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")}, + Outputs: []domain.PortDefinition{ + jsonPort("matched", "matched", "命中规则的数据"), + jsonPort("unmatched", "unmatched", "未命中规则的数据"), + }, + Image: runnerImage, + SkillDefinition: "按 FIELD_PATH + OPERATOR + EXPECTED 条件把真实数据分流到两个输出端口。", + Hyperparams: []domain.HyperParam{ + {Key: "FIELD_PATH", Value: "", Description: "待判断字段路径", Type: "string"}, + {Key: "OPERATOR", Value: "eq", Description: "eq/ne/contains/gt/gte/lt/lte/exists/missing", Type: "string"}, + {Key: "EXPECTED", Value: "", Description: "字符串比较值", Type: "string"}, + {Key: "EXPECTED_JSON", Value: "", Description: "JSON 格式比较值,优先于 EXPECTED", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinDatasetIndexOperatorID, + Name: "X 平台数据集索引", + Type: "builtin", + Category: "DATASET", + Description: "扫描 X-platform 采集目录,解析 meta.json、video_meta.json 和视频文件索引。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "目录输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "dataset", "数据集索引")}, + Image: runnerImage, + SkillDefinition: "面向 BJ_***/DJI_***/meta.json/video_meta.json 结构的真实数据集索引算子,输出采集目录和视频清单。", + Hyperparams: []domain.HyperParam{ + {Key: "SOURCE_PATH", Value: "", Description: "X-platform 根目录或单个采集目录", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinDatasetValidateOperatorID, + Name: "采集目录校验", + Type: "builtin", + Category: "DATASET", + Description: "校验采集目录是否包含 meta.json、video_meta.json,以及可选 intrinsics.json。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "数据集索引")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "validated", "校验结果")}, + Image: runnerImage, + SkillDefinition: "对 X-platform 数据集索引结果执行真实结构校验,输出每个采集目录的有效性报告。", + Hyperparams: []domain.HyperParam{ + {Key: "REQUIRE_INTRINSICS", Value: "false", Description: "是否要求 intrinsics.json 必须存在", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinVideoQualityOperatorID, + Name: "视频质检", + Type: "builtin", + Category: "VIDEO", + Description: "基于真实视频帧计算清晰度、抖动度和无手率估计。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "视频清单")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "quality", "质检结果")}, + Image: runnerImage, + SkillDefinition: "使用 OpenCV 对真实 mp4 做质检分析,输出 clarity / shake / handOcclusion / passed。", + Hyperparams: []domain.HyperParam{ + {Key: "MIN_CLARITY", Value: "0.8", Description: "清晰度最低阈值", Type: "string"}, + {Key: "MAX_SHAKE", Value: "0.1", Description: "抖动度最高阈值", Type: "string"}, + {Key: "MAX_HAND_OCCLUSION", Value: "0.1", Description: "无手率最高阈值", Type: "string"}, + {Key: "CLARITY_SCALE", Value: "35", Description: "清晰度归一化尺度", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinVideoFilterOperatorID, + Name: "数据过滤", + Type: "builtin", + Category: "VIDEO", + Description: "按日期、地点、场景和质检阈值过滤有效视频。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "视频与元数据输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "filtered", "过滤结果")}, + Image: runnerImage, + SkillDefinition: "针对 X-platform 质检结果执行真实筛选,输出符合条件的视频列表。", + Hyperparams: []domain.HyperParam{ + {Key: "DATE_START", Value: "", Description: "开始日期,支持 YYYY-MM-DD / YYYYMMDD", Type: "string"}, + {Key: "DATE_END", Value: "", Description: "结束日期,支持 YYYY-MM-DD / YYYYMMDD", Type: "string"}, + {Key: "PLACE", Value: "", Description: "采集地点", Type: "string"}, + {Key: "TYPE", Value: "", Description: "采集场景类型", Type: "string"}, + {Key: "MIN_CLARITY", Value: "0.8", Description: "清晰度最低阈值", Type: "string"}, + {Key: "MAX_SHAKE", Value: "0.1", Description: "抖动度最高阈值", Type: "string"}, + {Key: "MAX_HAND_OCCLUSION", Value: "0.1", Description: "无手率最高阈值", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinVideoSliceOperatorID, + Name: "视频切片", + Type: "builtin", + Category: "VIDEO", + Description: "将真实视频按固定时长切分为多个 mp4 片段。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "视频输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "clips", "切片结果")}, + Image: runnerImage, + SkillDefinition: "使用 ffmpeg/OpenCV 将真实视频切成固定长度片段,并输出 clip manifest。", + Hyperparams: []domain.HyperParam{ + {Key: "SEGMENT_DURATION_SECONDS", Value: "300", Description: "每个切片的秒数", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinVideoAnnotateOperatorID, + Name: "视频标注", + Type: "builtin", + Category: "VIDEO", + Description: "基于真实视频片段生成阶段划分、动作标签和对象提示。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "切片输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "annotations", "标注结果")}, + Image: runnerImage, + SkillDefinition: "读取切片视频并做真实帧间运动分析,输出 annotation JSON 和动作分段。", + Hyperparams: []domain.HyperParam{ + {Key: "FRAME_INTERVAL_SECONDS", Value: "1.0", Description: "采样帧间隔秒数", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinPackageOperatorID, + Name: "成品打包", + Type: "builtin", + Category: "VIDEO", + Description: "将切片视频、质检结果、标注结果和配置文件组装成成品数据包。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "切片/标注/质检结果")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "packages", "成品包清单")}, + Image: runnerImage, + SkillDefinition: "按 clip 组织真实 package 目录,输出 packages manifest 和所有成品文件。", + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinSamplingOperatorID, + Name: "抽检采样", + Type: "builtin", + Category: "QA", + Description: "按质检和标注规则对成品数据执行真实抽检采样。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "成品包输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "samples", "抽检样本清单")}, + Image: runnerImage, + SkillDefinition: "根据 RULES_JSON 对成品数据包做规则筛选和稳定抽样,输出抽检任务清单。", + Hyperparams: []domain.HyperParam{ + {Key: "RULES_JSON", Value: "[]", Description: "抽检规则数组,包含 field/operator/value/samplePercent", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinMongoWriteOperatorID, + Name: "MongoDB 写入", + Type: "builtin", + Category: "DATA", + Description: "将上游 JSON 数据写入用户配置的业务 MongoDB,支持批量插入和按键 upsert,并允许显式覆盖连接。", + Inputs: []domain.PortDefinition{ + jsonPort("input", "input", "待写入的 JSON 文档或 items 列表"), + }, + Outputs: []domain.PortDefinition{ + jsonPort("result", "result", "写入结果摘要"), + }, + Image: runnerImage, + SkillDefinition: "将 JSON 文档写入业务 MongoDB 集合。", + Hyperparams: []domain.HyperParam{ + {Key: "MONGO_COLLECTION", Value: "", Description: "目标集合名称(禁止写系统保留集合)", Type: "string"}, + {Key: "MONGO_MODE", Value: "insert_many", Description: "写入模式:insert_many 或 upsert", Type: "string"}, + {Key: "MONGO_CONNECTION_ID", Value: "", Description: "业务 MongoDB 连接 ID;推荐优先配置", Type: "string"}, + {Key: "MONGO_TARGET", Value: "business", Description: "兼容字段;默认 business。若显式传入 URI/DB,会优先使用显式连接", Type: "string"}, + {Key: "UPSERT_KEY", Value: "_id", Description: "upsert 模式使用的唯一键", Type: "string"}, + {Key: "DOCUMENT_PATH", Value: "", Description: "从输入 payload 中提取待写入文档的字段路径", Type: "string"}, + {Key: "MONGO_DB", Value: "", Description: "可选数据库名;显式填写后会覆盖 business/platform 默认连接", Type: "string"}, + {Key: "MONGO_URI", Value: "", Description: "可选 Mongo URI;显式填写后会覆盖 business/platform 默认连接", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinLeRobotSummaryOperatorID, + Name: "LeRobot 数据集概览", + Type: "builtin", + Category: "ROBOTICS", + Description: "读取真实 LeRobot 数据集目录,输出 meta/tasks/parquet/videos 概览。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "目录输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "summary", "概览结果")}, + Image: runnerImage, + SkillDefinition: "面向 LeRobot 数据集目录的真实概览算子,适合机器人示教数据检查。", + Hyperparams: []domain.HyperParam{ + {Key: "SOURCE_PATH", Value: "", Description: "LeRobot 数据集根目录", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinROSBagSummaryOperatorID, + Name: "ROSBag 数据概览", + Type: "builtin", + Category: "ROBOTICS", + Description: "读取真实 rosbag1 / rosbag2 / mcap / db3 数据并输出概览。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "bag 输入")}, + Outputs: []domain.PortDefinition{jsonPort("out-1", "summary", "概览结果")}, + Image: runnerImage, + SkillDefinition: "支持 rosbag db3 topic/message 概览,以及 bag/mcap 文件摘要。", + Hyperparams: []domain.HyperParam{ + {Key: "SOURCE_PATH", Value: "", Description: "bag/db3/mcap 文件或 rosbag2 目录", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: builtinShellCommandOperatorID, + Name: "Shell 命令执行", + Type: "builtin", + Category: "UTIL", + Description: "在 operator-runner 容器中真实执行 shell 命令并收集输出文件。", + Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "输入数据")}, + Outputs: []domain.PortDefinition{filePort("out-1", "result", "执行结果文件")}, + Image: runnerImage, + SkillDefinition: "执行 COMMAND 或 SCRIPT,自动收集输出目录产物并生成预览元数据。", + Hyperparams: []domain.HyperParam{ + {Key: "COMMAND", Value: "", Description: "容器内执行的命令", Type: "string"}, + {Key: "SCRIPT", Value: "", Description: "可执行脚本内容", Type: "string"}, + }, + CreatedAt: now, + UpdatedAt: now, + }, + } +} + +func findBuiltinOperator(items []domain.OperatorRecord, id string, name string) *domain.OperatorRecord { + for index := range items { + item := items[index] + if item.ID == id || strings.TrimSpace(item.Name) == strings.TrimSpace(name) { + copy := item + return © + } + } + return nil +} + +type builtinOperatorDeletionSet struct { + ids map[string]struct{} + names map[string]struct{} +} + +func normalizeBuiltinOperatorDeletionValue(value string) string { + return strings.ToLower(strings.TrimSpace(value)) +} + +func builtinOperatorDeletionKey(id string, name string) string { + if normalizedName := normalizeBuiltinOperatorDeletionValue(name); normalizedName != "" { + return "name:" + normalizedName + } + if normalizedID := normalizeBuiltinOperatorDeletionValue(id); normalizedID != "" { + return "id:" + normalizedID + } + return "" +} + +func newBuiltinOperatorDeletionSet(items []domain.BuiltinOperatorDeletionRecord) builtinOperatorDeletionSet { + set := builtinOperatorDeletionSet{ + ids: make(map[string]struct{}), + names: make(map[string]struct{}), + } + for _, item := range items { + if normalizedID := normalizeBuiltinOperatorDeletionValue(item.OperatorID); normalizedID != "" { + set.ids[normalizedID] = struct{}{} + } + if normalizedName := normalizeBuiltinOperatorDeletionValue(item.OperatorName); normalizedName != "" { + set.names[normalizedName] = struct{}{} + } + } + return set +} + +func (set builtinOperatorDeletionSet) Has(item domain.OperatorRecord) bool { + if normalizedID := normalizeBuiltinOperatorDeletionValue(item.ID); normalizedID != "" { + if _, ok := set.ids[normalizedID]; ok { + return true + } + } + if normalizedName := normalizeBuiltinOperatorDeletionValue(item.Name); normalizedName != "" { + if _, ok := set.names[normalizedName]; ok { + return true + } + } + return false +} + +func filterDeletedBuiltinOperators(items []domain.OperatorRecord, deletions builtinOperatorDeletionSet) []domain.OperatorRecord { + filtered := make([]domain.OperatorRecord, 0, len(items)) + for _, item := range items { + if strings.EqualFold(strings.TrimSpace(item.Type), "builtin") && deletions.Has(item) { + continue + } + filtered = append(filtered, item) + } + return filtered +} + +func upsertBuiltinOperator(ctx context.Context, server *server, items []domain.OperatorRecord, builtin domain.OperatorRecord) ([]domain.OperatorRecord, error) { + existing := findBuiltinOperator(items, builtin.ID, builtin.Name) + if existing == nil { + if err := server.store.UpsertOperator(ctx, builtin); err != nil { + return items, err + } + return append(items, builtin), nil + } + + updated := *existing + updated.ID = builtin.ID + updated.Name = builtin.Name + updated.Type = builtin.Type + updated.Category = builtin.Category + updated.Description = builtin.Description + updated.Inputs = builtin.Inputs + updated.Outputs = builtin.Outputs + updated.ConfigSchema = builtin.ConfigSchema + updated.Image = builtin.Image + updated.SkillDefinition = builtin.SkillDefinition + updated.Hyperparams = builtin.Hyperparams + updated.Resources = builtin.Resources + updated.UpdatedAt = builtin.UpdatedAt + if strings.TrimSpace(updated.CreatedAt) == "" { + updated.CreatedAt = builtin.CreatedAt + } + if err := server.store.UpsertOperator(ctx, updated); err != nil { + return items, err + } + + for index := range items { + if items[index].ID == existing.ID || strings.TrimSpace(items[index].Name) == strings.TrimSpace(existing.Name) { + items[index] = updated + return items, nil + } + } + return append(items, updated), nil +} + +func (server *server) ensureBuiltinOperators(ctx context.Context) error { + items, err := server.store.ListOperators(ctx) + if err != nil { + return err + } + deletions, err := server.store.ListBuiltinOperatorDeletions(ctx) + if err != nil { + return err + } + deletionSet := newBuiltinOperatorDeletionSet(deletions) + + now := nowISO() + for _, builtin := range builtinOperatorRecords(now, server.operatorRunnerImage) { + if deletionSet.Has(builtin) { + continue + } + items, err = upsertBuiltinOperator(ctx, server, items, builtin) + if err != nil { + return err + } + } + return server.refreshBuiltinOperatorImages(ctx, deletionSet) +} + +func (server *server) refreshBuiltinOperatorImages(ctx context.Context, deletions builtinOperatorDeletionSet) error { + runnerImage := strings.TrimSpace(server.operatorRunnerImage) + if runnerImage == "" { + return nil + } + items, err := server.store.ListOperators(ctx) + if err != nil { + return err + } + now := nowISO() + for _, item := range items { + if strings.TrimSpace(strings.ToLower(item.Type)) != "builtin" { + continue + } + if deletions.Has(item) { + continue + } + if strings.TrimSpace(item.Image) == runnerImage { + continue + } + item.Image = runnerImage + item.UpdatedAt = now + if strings.TrimSpace(item.CreatedAt) == "" { + item.CreatedAt = now + } + if err := server.store.UpsertOperator(ctx, item); err != nil { + return err + } + } + return nil +} + +func (server *server) listAvailableOperators(ctx context.Context) ([]domain.OperatorRecord, error) { + if err := server.ensureBuiltinOperators(ctx); err != nil { + return nil, err + } + items, err := server.store.ListOperators(ctx) + if err != nil { + return nil, err + } + deletions, err := server.store.ListBuiltinOperatorDeletions(ctx) + if err != nil { + return nil, err + } + return filterDeletedBuiltinOperators(items, newBuiltinOperatorDeletionSet(deletions)), nil +} diff --git a/internal/api/data_platform.go b/internal/api/data_platform.go new file mode 100644 index 00000000..5de31fcc --- /dev/null +++ b/internal/api/data_platform.go @@ -0,0 +1,2660 @@ +package api + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "time" + + "droboticflow/internal/domain" + + "github.com/baidubce/bce-sdk-go/services/bos" + "github.com/go-chi/chi/v5" +) + +const ( + defaultBOSEndpoint = "https://bj.bcebos.com" +) + +type sourceBrowserPreviewRequest struct { + FileName string + Payload []byte + RedirectURL string +} + +type flushResponseWriter struct { + writer http.ResponseWriter + flusher http.Flusher +} + +func (target flushResponseWriter) Write(payload []byte) (int, error) { + count, err := target.writer.Write(payload) + if count > 0 { + target.flusher.Flush() + } + return count, err +} + +var sourceBrowserVideoPreviewPath = locateSourceBrowserVideoPreviewPath +var sourceBrowserVideoPreviewWriter = streamSourceBrowserVideoPreview +var sourceCollectionSignedPreviewURL = buildSourceCollectionSignedPreviewURL + +type scanConfigRequest struct { + Name string `json:"name"` + Description string `json:"description"` + SourcePath string `json:"sourcePath"` + ConnectionID string `json:"connectionId,omitempty"` + Bucket string `json:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + AccessKeyID string `json:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty"` + ScanMode string `json:"scanMode,omitempty"` + IncrementalUnit string `json:"incrementalUnit,omitempty"` + DirectoryDepth int `json:"directoryDepth,omitempty"` + Pattern string `json:"pattern,omitempty"` + Interval int `json:"interval"` + Enabled bool `json:"enabled"` + DataSource string `json:"dataSource,omitempty"` + Notifications domain.ScanNotificationSettings `json:"notifications,omitempty"` +} + +type samplingReviewRequest struct { + VideoQuality string `json:"videoQuality"` + AnnotationAccuracy string `json:"annotationAccuracy"` + AnnotationCompleteness string `json:"annotationCompleteness"` + OverallGrade string `json:"overallGrade"` + Notes string `json:"notes,omitempty"` +} + +func isScanWorkflow(item domain.WorkflowRecord) bool { + return item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil +} + +func chooseWorkflowKind(value domain.WorkflowKind) domain.WorkflowKind { + if value == "" { + return domain.WorkflowKindStandard + } + return value +} + +func chooseDataSource(value string) string { + _ = value + return "BOS" +} + +func cleanBOSPrefix(value string) string { + return strings.Trim(strings.TrimSpace(value), "/") +} + +func chooseBOSEndpoint(value string) string { + if strings.TrimSpace(value) == "" { + return defaultBOSEndpoint + } + return strings.TrimSpace(value) +} + +func chooseWorkflowScanMode(value string) string { + switch strings.ToUpper(strings.TrimSpace(value)) { + case "FULL": + return "FULL" + default: + return "INCREMENTAL" + } +} + +func chooseWorkflowIncrementalUnit(value string) string { + switch strings.ToUpper(strings.TrimSpace(value)) { + case "DIRECTORY": + return "DIRECTORY" + default: + return "OBJECT" + } +} + +func chooseWorkflowDirectoryDepth(value int) int { + if value > 0 { + return value + } + return 1 +} + +func scanSourcePath(dataSource string, sourcePath string, bucket string, prefix string) string { + if chooseDataSource(dataSource) == "BOS" { + if explicit := strings.TrimSpace(sourcePath); explicit != "" { + return explicit + } + if strings.TrimSpace(bucket) == "" { + return "" + } + canonical := "bos://" + strings.TrimSpace(bucket) + if cleanedPrefix := cleanBOSPrefix(prefix); cleanedPrefix != "" { + canonical += "/" + cleanedPrefix + } + return canonical + } + return strings.TrimSpace(sourcePath) +} + +func mergeScanSecrets(existing *domain.ScanConfig, payload scanConfigRequest) (string, string) { + accessKeyID := strings.TrimSpace(payload.AccessKeyID) + secretAccessKey := strings.TrimSpace(payload.SecretAccessKey) + if accessKeyID == "" && existing != nil { + accessKeyID = existing.AccessKeyID + } + if secretAccessKey == "" && existing != nil { + secretAccessKey = existing.SecretAccessKey + } + return accessKeyID, secretAccessKey +} + +func mergeScanConnectionID(existing *domain.ScanConfig, payload scanConfigRequest) string { + connectionID := strings.TrimSpace(payload.ConnectionID) + if connectionID == "" && + existing != nil && + strings.TrimSpace(existing.ConnectionID) != "" && + strings.TrimSpace(payload.AccessKeyID) == "" && + strings.TrimSpace(payload.SecretAccessKey) == "" { + connectionID = strings.TrimSpace(existing.ConnectionID) + } + return connectionID +} + +func (server *server) getStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) { + connectionID := strings.TrimSpace(id) + if connectionID == "" { + return nil, nil + } + item, err := server.store.GetStorageConnection(ctx, connectionID) + if err != nil { + return nil, err + } + if item == nil { + return nil, fmt.Errorf("storage connection %s not found", connectionID) + } + return item, nil +} + +func (server *server) getMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) { + connectionID := strings.TrimSpace(id) + if connectionID == "" { + return nil, nil + } + item, err := server.store.GetMongoConnection(ctx, connectionID) + if err != nil { + return nil, err + } + if item == nil { + return nil, fmt.Errorf("mongo connection %s not found", connectionID) + } + return item, nil +} + +func (server *server) resolveScanConfig(ctx context.Context, scanConfig *domain.ScanConfig) (*domain.ScanConfig, error) { + if scanConfig == nil { + return nil, nil + } + copy := *scanConfig + connection, err := server.getStorageConnection(ctx, copy.ConnectionID) + if err != nil { + return nil, err + } + if connection == nil { + return ©, nil + } + copy.Endpoint = storageEndpointForProvider(connection.Provider, connection.Endpoint) + if strings.TrimSpace(copy.AccessKeyID) == "" { + copy.AccessKeyID = connection.AccessKeyID + } + if strings.TrimSpace(copy.SecretAccessKey) == "" { + copy.SecretAccessKey = connection.SecretAccessKey + } + return ©, nil +} + +func cloneHyperparams(items []domain.HyperParam) []domain.HyperParam { + out := make([]domain.HyperParam, len(items)) + copy(out, items) + return out +} + +func clonePorts(items []domain.PortDefinition) []domain.PortDefinition { + out := make([]domain.PortDefinition, len(items)) + copy(out, items) + return out +} + +func cloneOperatorIntoNode(operator domain.OperatorRecord, nodeID string, x float64, y float64, overrides map[string]string) domain.WorkflowNode { + hyperparams := cloneHyperparams(operator.Hyperparams) + seen := make(map[string]int, len(hyperparams)) + for index, item := range hyperparams { + seen[item.Key] = index + } + for key, value := range overrides { + if index, ok := seen[key]; ok { + hyperparams[index].Value = value + continue + } + hyperparams = append(hyperparams, domain.HyperParam{Key: key, Value: value}) + } + return domain.WorkflowNode{ + ID: nodeID, + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: operator.Name, + Category: operator.Category, + Description: operator.Description, + Inputs: clonePorts(operator.Inputs), + Outputs: clonePorts(operator.Outputs), + Image: operator.Image, + SkillDefinition: operator.SkillDefinition, + Hyperparams: hyperparams, + }, + Position: domain.Position{X: x, Y: y}, + } +} + +func operatorByName(items []domain.OperatorRecord, name string) (*domain.OperatorRecord, bool) { + for _, item := range items { + if item.Name == name { + copy := item + return ©, true + } + } + return nil, false +} + +func (server *server) buildScanWorkflowInput(ctx context.Context, projectID string, payload scanConfigRequest, existing *domain.ScanConfig) (domain.CreateWorkflowInput, error) { + operators, err := server.listAvailableOperators(ctx) + if err != nil { + return domain.CreateWorkflowInput{}, err + } + dataSource := chooseDataSource(payload.DataSource) + sourceOperatorName := "BOS 增量扫描" + sourceOperator, ok := operatorByName(operators, sourceOperatorName) + if !ok { + return domain.CreateWorkflowInput{}, fmt.Errorf("operator %s not found", sourceOperatorName) + } + indexOperator, ok := operatorByName(operators, "X 平台数据集索引") + if !ok { + return domain.CreateWorkflowInput{}, fmt.Errorf("operator X 平台数据集索引 not found") + } + validateOperator, ok := operatorByName(operators, "采集目录校验") + if !ok { + return domain.CreateWorkflowInput{}, fmt.Errorf("operator 采集目录校验 not found") + } + + sourcePath := scanSourcePath(dataSource, payload.SourcePath, payload.Bucket, payload.Prefix) + connectionID := mergeScanConnectionID(existing, payload) + var connection *domain.StorageConnectionRecord + if connectionID != "" { + connection, err = server.getStorageConnection(ctx, connectionID) + if err != nil { + return domain.CreateWorkflowInput{}, err + } + } + endpoint := "" + accessKeyID := "" + secretAccessKey := "" + if connection != nil { + endpoint = storageEndpointForProvider(connection.Provider, connection.Endpoint) + } else if dataSource == "BOS" { + endpoint = chooseBOSEndpoint(payload.Endpoint) + accessKeyID, secretAccessKey = mergeScanSecrets(existing, payload) + } + + sourceOverrides := map[string]string{ + "DATA_SOURCE": dataSource, + "SCAN_MODE": chooseWorkflowScanMode(payload.ScanMode), + "INCREMENTAL_UNIT": chooseWorkflowIncrementalUnit(payload.IncrementalUnit), + "INCREMENTAL_DIRECTORY_DEPTH": strconv.Itoa(chooseWorkflowDirectoryDepth(payload.DirectoryDepth)), + } + if dataSource == "BOS" { + sourceOverrides["SOURCE_PATH"] = sourcePath + sourceOverrides["BOS_ENDPOINT"] = endpoint + } else { + sourceOverrides["SOURCE_PATH"] = sourcePath + } + sourceNode := cloneOperatorIntoNode(*sourceOperator, "scan-source", 0, 0, sourceOverrides) + sourceNode.Data.Bucket = strings.TrimSpace(payload.Bucket) + sourceNode.Data.Prefix = cleanBOSPrefix(payload.Prefix) + sourceNode.Data.Pattern = strings.TrimSpace(payload.Pattern) + indexNode := cloneOperatorIntoNode(*indexOperator, "scan-index", 260, 0, nil) + validateNode := cloneOperatorIntoNode(*validateOperator, "scan-validate", 520, 0, nil) + + schedule := &domain.WorkflowSchedule{SourcePath: sourcePath} + workflowType := domain.WorkflowTypeBatch + if payload.Enabled && payload.Interval > 0 { + schedule.Interval = payload.Interval + workflowType = domain.WorkflowTypeStreaming + } + + return domain.CreateWorkflowInput{ + ProjectID: projectID, + Name: strings.TrimSpace(payload.Name), + Description: strings.TrimSpace(payload.Description), + Type: workflowType, + Kind: domain.WorkflowKindScan, + ScanConfig: &domain.ScanConfig{ + DataSource: dataSource, + SourcePath: sourcePath, + ConnectionID: connectionID, + Bucket: strings.TrimSpace(payload.Bucket), + Prefix: cleanBOSPrefix(payload.Prefix), + Endpoint: endpoint, + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + ScanMode: chooseWorkflowScanMode(payload.ScanMode), + IncrementalUnit: chooseWorkflowIncrementalUnit(payload.IncrementalUnit), + DirectoryDepth: chooseWorkflowDirectoryDepth(payload.DirectoryDepth), + Pattern: strings.TrimSpace(payload.Pattern), + Enabled: payload.Enabled && payload.Interval > 0, + Notifications: payload.Notifications, + }, + Schedule: schedule, + Nodes: []domain.WorkflowNode{ + sourceNode, + indexNode, + validateNode, + }, + Edges: []domain.WorkflowEdge{ + {ID: "scan-source-index", Source: sourceNode.ID, SourceHandle: "out-1", Target: indexNode.ID, TargetHandle: "in-1"}, + {ID: "scan-index-validate", Source: indexNode.ID, SourceHandle: "out-1", Target: validateNode.ID, TargetHandle: "in-1"}, + }, + }, nil +} + +func validateScanConfigDraftPayload(payload scanConfigRequest) []validationIssue { + issues := make([]validationIssue, 0, 4) + if strings.TrimSpace(payload.Name) == "" { + issues = append(issues, validationIssue{Field: "name", Message: "name is required"}) + } + if payload.Interval < 0 { + issues = append(issues, validationIssue{Field: "interval", Message: "interval must be zero or greater"}) + } + if scanMode := strings.ToUpper(strings.TrimSpace(payload.ScanMode)); scanMode != "" && scanMode != "FULL" && scanMode != "INCREMENTAL" { + issues = append(issues, validationIssue{Field: "scanMode", Message: "scanMode must be FULL or INCREMENTAL"}) + } + if incrementalUnit := strings.ToUpper(strings.TrimSpace(payload.IncrementalUnit)); incrementalUnit != "" && incrementalUnit != "OBJECT" && incrementalUnit != "DIRECTORY" { + issues = append(issues, validationIssue{Field: "incrementalUnit", Message: "incrementalUnit must be OBJECT or DIRECTORY"}) + } + if payload.DirectoryDepth < 0 { + issues = append(issues, validationIssue{Field: "directoryDepth", Message: "directoryDepth must be zero or greater"}) + } + return issues +} + +func scanConfigPayloadReady(payload scanConfigRequest, existing *domain.ScanConfig) bool { + switch chooseDataSource(payload.DataSource) { + case "BOS": + connectionID := mergeScanConnectionID(existing, payload) + accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload) + hasTarget := strings.TrimSpace(payload.SourcePath) != "" || + (strings.TrimSpace(payload.Bucket) != "" && cleanBOSPrefix(payload.Prefix) != "") + return hasTarget && + (strings.TrimSpace(connectionID) != "" || + (strings.TrimSpace(accessKeyID) != "" && strings.TrimSpace(secretAccessKey) != "")) + default: + return strings.TrimSpace(payload.SourcePath) != "" + } +} + +func validateScanConfigPayload(payload scanConfigRequest, existing *domain.ScanConfig) []validationIssue { + issues := validateScanConfigDraftPayload(payload) + switch chooseDataSource(payload.DataSource) { + case "BOS": + if strings.TrimSpace(payload.SourcePath) == "" { + if strings.TrimSpace(payload.Bucket) == "" { + issues = append(issues, validationIssue{Field: "bucket", Message: "bucket is required for BOS scan configs when sourcePath is empty"}) + } + if cleanBOSPrefix(payload.Prefix) == "" { + issues = append(issues, validationIssue{Field: "prefix", Message: "prefix is required for BOS scan configs when sourcePath is empty"}) + } + } + connectionID := mergeScanConnectionID(existing, payload) + if strings.TrimSpace(connectionID) == "" { + accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload) + if strings.TrimSpace(accessKeyID) == "" { + issues = append(issues, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required for BOS scan configs"}) + } + if strings.TrimSpace(secretAccessKey) == "" { + issues = append(issues, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required for BOS scan configs"}) + } + } + default: + if strings.TrimSpace(payload.SourcePath) == "" { + issues = append(issues, validationIssue{Field: "sourcePath", Message: "sourcePath is required"}) + } + } + return issues +} + +func (server *server) scanWorkflows(ctx context.Context) ([]domain.WorkflowRecord, error) { + items, err := server.store.ListAllWorkflows(ctx) + if err != nil { + return nil, err + } + filtered := make([]domain.WorkflowRecord, 0) + for _, item := range items { + if isScanWorkflow(item) { + item.Kind = chooseWorkflowKind(item.Kind) + filtered = append(filtered, item) + } + } + return filtered, nil +} + +func (server *server) handleScanConfigsList(writer http.ResponseWriter, request *http.Request) { + items, err := server.scanWorkflows(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)}) +} + +func (server *server) handleScanConfigsCreate(writer http.ResponseWriter, request *http.Request) { + var payload scanConfigRequest + if !decodeJSON(writer, request, &payload) { + return + } + if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if payload.Interval > 0 && !scanConfigPayloadReady(payload, nil) { + renderValidationError(writer, validateScanConfigPayload(payload, nil)...) + return + } + if issues := server.validateScanConnectionReference(request.Context(), normalizedScanConfigFromRequest(nil, payload)); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + input, err := server.buildScanWorkflowInput(request.Context(), "", payload, nil) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + item, err := server.store.CreateWorkflow(request.Context(), input) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item)) +} + +func (server *server) handleScanConfigsGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil || !isScanWorkflow(*item) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"}) + return + } + item.Kind = chooseWorkflowKind(item.Kind) + renderJSON(writer, http.StatusOK, sanitizeWorkflow(item)) +} + +func (server *server) handleScanConfigsUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil || !isScanWorkflow(*item) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"}) + return + } + var payload scanConfigRequest + if !decodeJSON(writer, request, &payload) { + return + } + if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if payload.Interval > 0 && !scanConfigPayloadReady(payload, item.ScanConfig) { + renderValidationError(writer, validateScanConfigPayload(payload, item.ScanConfig)...) + return + } + if issues := server.validateScanConnectionReference(request.Context(), normalizedScanConfigFromRequest(item.ScanConfig, payload)); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + input, err := server.buildScanWorkflowInput(request.Context(), item.ProjectID, payload, item.ScanConfig) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + item.Name = input.Name + item.Description = input.Description + item.Type = input.Type + item.Kind = input.Kind + item.ScanConfig = input.ScanConfig + item.Schedule = input.Schedule + item.Nodes = input.Nodes + item.Edges = input.Edges + item.UpdatedAt = nowISO() + if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, sanitizeWorkflow(item)) +} + +func (server *server) handleScanConfigsDelete(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil || !isScanWorkflow(*item) { + writer.WriteHeader(http.StatusNotFound) + return + } + deleted, err := server.store.DeleteWorkflow(request.Context(), item.ID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + writer.WriteHeader(http.StatusNotFound) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (server *server) handleScanConfigsRun(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil || !isScanWorkflow(*item) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"}) + return + } + if issues := server.validateScanConnectionReference(request.Context(), item.ScanConfig); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + run, err := server.store.CreateRun(request.Context(), item.ID, domain.RunTriggerManual) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusAccepted, run) +} + +func artifactByName(run domain.RunRecord, name string) *domain.ExecutionArtifact { + for index := range run.Artifacts { + if run.Artifacts[index].Name == name { + return &run.Artifacts[index] + } + } + return nil +} + +func mapValue(value any) map[string]any { + item, _ := normalizeStructuredValue(value).(map[string]any) + return item +} + +func listValue(value any) []any { + items, _ := normalizeStructuredValue(value).([]any) + return items +} + +func normalizeStructuredValue(value any) any { + switch typed := value.(type) { + case nil: + return nil + case map[string]any: + items := make(map[string]any, len(typed)) + for key, item := range typed { + items[key] = normalizeStructuredValue(item) + } + return items + case []any: + items := make([]any, len(typed)) + for index, item := range typed { + items[index] = normalizeStructuredValue(item) + } + return items + } + + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Map: + if rv.Type().Key().Kind() != reflect.String { + return value + } + items := make(map[string]any, rv.Len()) + iter := rv.MapRange() + for iter.Next() { + items[iter.Key().String()] = normalizeStructuredValue(iter.Value().Interface()) + } + return items + case reflect.Slice, reflect.Array: + items := make([]any, rv.Len()) + for index := 0; index < rv.Len(); index++ { + items[index] = normalizeStructuredValue(rv.Index(index).Interface()) + } + return items + default: + return value + } +} + +func stringValue(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + default: + text := strings.TrimSpace(fmt.Sprintf("%v", typed)) + if text == "" { + return "" + } + return text + } +} + +func boolValue(value any) bool { + typed, ok := value.(bool) + return ok && typed +} + +func floatValue(value any) float64 { + switch typed := value.(type) { + case float64: + return typed + case float32: + return float64(typed) + case int: + return float64(typed) + case int64: + return float64(typed) + default: + return 0 + } +} + +func intValue(value any) int { + return int(floatValue(value)) +} + +func int64Value(value any) int64 { + return int64(floatValue(value)) +} + +func artifactDataRef(artifact *domain.ExecutionArtifact) map[string]any { + if artifact == nil || artifact.Data == nil { + return nil + } + return mapValue(artifact.Data["dataRef"]) +} + +func artifactStructuredData(artifact *domain.ExecutionArtifact) map[string]any { + if artifact == nil { + return nil + } + if dataRef := artifactDataRef(artifact); dataRef != nil { + payloadPath := strings.TrimSpace(stringValue(dataRef["payloadPath"])) + if payloadPath != "" && !strings.Contains(payloadPath, "://") { + payload, err := os.ReadFile(payloadPath) + if err == nil { + var decoded any + if err := json.Unmarshal(payload, &decoded); err == nil { + if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok { + return structured + } + } + } + } + } + if artifact.Data != nil { + return artifact.Data + } + if artifact.Type == domain.PortTypeJSON { + filePath := strings.TrimSpace(artifact.Path) + if filePath != "" && !strings.Contains(filePath, "://") { + payload, err := os.ReadFile(filePath) + if err == nil { + var decoded any + if err := json.Unmarshal(payload, &decoded); err == nil { + if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok { + return structured + } + } + } + } + } + return nil +} + +func sourceArtifactRelativePath(path string) string { + normalized := filepath.ToSlash(path) + marker := "/_source_tree/" + index := strings.Index(normalized, marker) + if index < 0 { + return "" + } + return strings.TrimPrefix(normalized[index+len(marker):], "/") +} + +func sourceArtifactPath(artifact domain.ExecutionArtifact) string { + if payload := artifactStructuredData(&artifact); payload != nil { + if relative := stringValue(payload["relativePath"]); relative != "" { + return filepath.ToSlash(strings.Trim(relative, "/")) + } + } + return sourceArtifactRelativePath(artifact.Path) +} + +func packageArtifactRelativePath(path string, packageName string) string { + normalized := filepath.ToSlash(path) + marker := "/packages/" + packageName + "/" + index := strings.Index(normalized, marker) + if index < 0 { + return "" + } + return strings.TrimPrefix(normalized[index+len(marker):], "/") +} + +func packageContentURL(runID string, packageName string, relativePath string) string { + return fmt.Sprintf( + "/api/packages/%s/%s/content?relativePath=%s", + url.PathEscape(runID), + url.PathEscape(packageName), + url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")), + ) +} + +func packageArtifactPreviewURL(artifact domain.ExecutionArtifact) string { + payload := artifactStructuredData(&artifact) + preview := mapValue(payload["preview"]) + if preview != nil { + if value := strings.TrimSpace(stringValue(preview["url"])); value != "" { + return value + } + } + source := mapValue(payload["source"]) + if source != nil { + if value := strings.TrimSpace(stringValue(source["previewURL"])); value != "" { + return value + } + } + return "" +} + +func packageArtifactResolvedRelativePath(artifact domain.ExecutionArtifact, packageName string) string { + if relativePath := packageArtifactRelativePath(artifact.Path, packageName); relativePath != "" { + return relativePath + } + payload := artifactStructuredData(&artifact) + if payload == nil { + return "" + } + relativePath := strings.Trim(filepath.ToSlash(stringValue(payload["relativePath"])), "/") + if relativePath == "" { + return "" + } + packagePrefix := strings.Trim(filepath.ToSlash(path.Join("packages", packageName)), "/") + if packagePrefix == "" { + return "" + } + if relativePath == packagePrefix { + return "" + } + if strings.HasPrefix(relativePath, packagePrefix+"/") { + return strings.TrimPrefix(relativePath, packagePrefix+"/") + } + return "" +} + +func packageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact { + copy := artifact + if copy.Data == nil { + copy.Data = make(map[string]any) + } + relativePath := packageArtifactResolvedRelativePath(artifact, packageName) + if relativePath == "" { + relativePath = filepath.ToSlash(strings.Trim(copy.Path, "/")) + } + if relativePath != "" { + copy.Path = relativePath + } + preview := mapValue(copy.Data["preview"]) + if preview == nil { + preview = make(map[string]any) + } + kind := strings.TrimSpace(stringValue(preview["kind"])) + mimeType := "" + if copy.Metadata != nil { + mimeType = strings.TrimSpace(copy.Metadata.MimeType) + } + if kind == "video" || copy.Type == "video" || strings.HasPrefix(mimeType, "video/") { + preview["kind"] = "video" + preview["url"] = packageContentURL(runID, packageName, copy.Path) + copy.Data["preview"] = preview + } + return copy +} + +func canonicalBOSPath(bucket string, object string) string { + bucket = strings.TrimSpace(bucket) + object = strings.Trim(strings.TrimSpace(object), "/") + if bucket == "" { + return "" + } + if object == "" { + return "bos://" + bucket + } + return "bos://" + bucket + "/" + object +} + +func packageArtifactPriority(artifact domain.ExecutionArtifact) int { + score := 0 + if artifact.NodeID == "upload" { + score += 40 + } + artifactPath := strings.TrimSpace(artifact.Path) + if strings.HasPrefix(strings.ToLower(artifactPath), "bos://") { + score += 30 + } else if artifactPath != "" && !strings.Contains(artifactPath, "://") { + score += 10 + } + payload := artifactStructuredData(&artifact) + source := mapValue(payload["source"]) + sourcePath := strings.TrimSpace(stringValue(source["path"])) + if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") { + score += 30 + } + if strings.TrimSpace(packageArtifactPreviewURL(artifact)) != "" { + score += 5 + } + return score +} + +func packageArtifactStoragePath(artifact domain.ExecutionArtifact, packageName string) string { + payload := artifactStructuredData(&artifact) + source := mapValue(payload["source"]) + if sourcePath := strings.TrimSpace(stringValue(source["path"])); sourcePath != "" { + if bucket, object, ok := parseBOSPath(sourcePath); ok { + return canonicalBOSPath(bucket, path.Dir(object)) + } + directory := filepath.ToSlash(filepath.Dir(sourcePath)) + if directory != "." { + return directory + } + } + artifactPath := strings.TrimSpace(artifact.Path) + if bucket, object, ok := parseBOSPath(artifactPath); ok { + return canonicalBOSPath(bucket, path.Dir(object)) + } + if artifactPath == "" { + return "" + } + directory := filepath.ToSlash(filepath.Dir(artifactPath)) + if directory == "." { + return "" + } + return directory +} + +func dedupePackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact { + bestByRelativePath := make(map[string]domain.ExecutionArtifact) + for _, artifact := range run.Artifacts { + relativePath := packageArtifactResolvedRelativePath(artifact, packageName) + if relativePath == "" { + continue + } + current, exists := bestByRelativePath[relativePath] + if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) { + bestByRelativePath[relativePath] = artifact + } + } + items := make([]domain.ExecutionArtifact, 0, len(bestByRelativePath)) + for _, artifact := range bestByRelativePath { + items = append(items, artifact) + } + sort.Slice(items, func(left int, right int) bool { + leftPath := packageArtifactResolvedRelativePath(items[left], packageName) + rightPath := packageArtifactResolvedRelativePath(items[right], packageName) + return leftPath < rightPath + }) + return items +} + +func packageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact { + items := make([]domain.ExecutionArtifact, 0) + for _, artifact := range dedupePackageArtifacts(run, packageName) { + items = append(items, packageArtifactForResponse(run.ID, packageName, artifact)) + } + return items +} + +func choosePackagePreview(files []domain.ExecutionArtifact) *domain.ExecutionArtifact { + for _, file := range files { + if strings.HasSuffix(strings.ToLower(file.Name), ".mp4") { + copy := file + return © + } + } + if len(files) == 0 { + return nil + } + copy := files[0] + return © +} + +func sourceDatasetList(data map[string]any, catalogKey string, fallbackKey string) []any { + if _, ok := data[catalogKey]; ok { + return listValue(data[catalogKey]) + } + return listValue(data[fallbackKey]) +} + +func sourceDatasetHasCatalog(data map[string]any) bool { + _, hasCollections := data["catalogCollections"] + _, hasItems := data["catalogItems"] + _, hasEntries := data["catalogEntries"] + return hasCollections || hasItems || hasEntries +} + +func sourceDatasetIsDisplayable(data map[string]any) bool { + if len(sourceDatasetList(data, "catalogCollections", "collections")) > 0 { + return true + } + if len(sourceDatasetList(data, "catalogEntries", "entries")) > 0 { + return true + } + return intValue(data["catalogCollectionCount"]) > 0 || intValue(data["collectionCount"]) > 0 +} + +func runSourceManifest(run domain.RunRecord) *domain.ExecutionArtifact { + manifest := artifactByName(run, "source-manifest.json") + if manifest == nil || artifactStructuredData(manifest) == nil { + return nil + } + return manifest +} + +func runDatasetIndex(run domain.RunRecord) *domain.ExecutionArtifact { + artifact := artifactByName(run, "dataset-index.json") + if artifact == nil || artifactStructuredData(artifact) == nil { + return nil + } + return artifact +} + +func runLooksLikeSourceDataset(workflow domain.WorkflowRecord, run domain.RunRecord) bool { + if manifest := runSourceManifest(run); manifest != nil { + payload := artifactStructuredData(manifest) + if strings.TrimSpace(stringValue(payload["sourcePath"])) != "" || strings.TrimSpace(stringValue(payload["dataSource"])) != "" { + return true + } + } + index := runDatasetIndex(run) + if index == nil { + return false + } + if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil { + return true + } + indexPayload := artifactStructuredData(index) + sourcePath := strings.TrimSpace(stringValue(indexPayload["sourcePath"])) + rootPath := strings.TrimSpace(stringValue(indexPayload["rootPath"])) + dataSource := strings.TrimSpace(stringValue(indexPayload["dataSource"])) + if dataSource != "" || strings.HasPrefix(sourcePath, "bos://") || strings.HasPrefix(rootPath, "bos://") { + return true + } + if len(mapValue(indexPayload["archiveSources"])) > 0 && strings.Contains(rootPath, "_materialized_inputs") { + return false + } + return sourceDatasetHasCatalog(indexPayload) +} + +func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]domain.WorkflowRecord, map[string]domain.RunRecord, error) { + workflows, err := server.store.ListAllWorkflows(ctx) + if err != nil { + return nil, nil, err + } + workflowByID := make(map[string]domain.WorkflowRecord, len(workflows)) + for _, workflow := range workflows { + workflowByID[workflow.ID] = workflow + } + runs, err := server.store.ListRuns(ctx, string(domain.RunStatusCompleted)) + if err != nil { + return nil, nil, err + } + latestByWorkflow := make(map[string]domain.RunRecord) + for _, run := range runs { + workflow, ok := workflowByID[run.WorkflowID] + if !ok { + continue + } + index := runDatasetIndex(run) + if index == nil { + continue + } + if !runLooksLikeSourceDataset(workflow, run) { + continue + } + if !sourceDatasetIsDisplayable(artifactStructuredData(index)) { + continue + } + current, exists := latestByWorkflow[run.WorkflowID] + if !exists || run.CreatedAt > current.CreatedAt { + latestByWorkflow[run.WorkflowID] = run + } + } + return workflowByID, latestByWorkflow, nil +} + +func parseBOSPath(sourcePath string) (string, string, bool) { + normalized := strings.TrimSpace(sourcePath) + if !strings.HasPrefix(strings.ToLower(normalized), "bos://") { + return "", "", false + } + trimmed := strings.TrimPrefix(normalized, "bos://") + trimmed = strings.TrimPrefix(trimmed, "BOS://") + bucket, object, found := strings.Cut(trimmed, "/") + bucket = strings.TrimSpace(bucket) + object = strings.Trim(strings.TrimSpace(object), "/") + if !found { + object = "" + } + if bucket == "" { + return "", "", false + } + return bucket, object, true +} + +func normalizeLocalDirectory(value string) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "" + } + directory := filepath.ToSlash(filepath.Dir(trimmed)) + if directory == "." { + return "" + } + return directory +} + +func assetStorageLocationFromPaths(rootPath string, currentPath string, endpoint string) domain.AssetStorageLocation { + location := domain.AssetStorageLocation{ + Path: strings.TrimSpace(currentPath), + } + + currentBucket, currentObject, hasCurrentBOS := parseBOSPath(currentPath) + rootBucket, rootObject, hasRootBOS := parseBOSPath(rootPath) + + switch { + case hasCurrentBOS: + location.Provider = "bos" + location.Endpoint = chooseBOSEndpoint(endpoint) + location.Bucket = currentBucket + if hasRootBOS && rootBucket == currentBucket { + location.Prefix = rootObject + } else { + location.Prefix = path.Dir(currentObject) + if location.Prefix == "." { + location.Prefix = "" + } + } + case hasRootBOS: + location.Provider = "bos" + location.Endpoint = chooseBOSEndpoint(endpoint) + location.Bucket = rootBucket + location.Prefix = rootObject + if location.Path == "" { + location.Path = strings.TrimSpace(rootPath) + } + case strings.TrimSpace(currentPath) != "": + location.Provider = "k8s-job" + location.Prefix = normalizeLocalDirectory(currentPath) + case strings.TrimSpace(rootPath) != "": + location.Provider = "k8s-job" + location.Path = strings.TrimSpace(rootPath) + location.Prefix = normalizeLocalDirectory(rootPath) + default: + location.Provider = "unknown" + } + + if location.Prefix == "." { + location.Prefix = "" + } + return location +} + +func sourceRootPathForRun(run domain.RunRecord, scanConfig *domain.ScanConfig) string { + if scanConfig != nil { + if value := scanSourcePath(scanConfig.DataSource, scanConfig.SourcePath, scanConfig.Bucket, scanConfig.Prefix); value != "" { + return value + } + } + if manifest := runSourceManifest(run); manifest != nil { + payload := artifactStructuredData(manifest) + if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" { + return value + } + } + if index := runDatasetIndex(run); index != nil { + payload := artifactStructuredData(index) + if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" { + return value + } + if value := strings.TrimSpace(stringValue(payload["rootPath"])); value != "" { + return value + } + } + return "" +} + +func (server *server) buildSourceCollectionRecord(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, collection map[string]any) domain.SourceCollectionRecord { + meta := mapValue(collection["meta"]) + requiredFiles := make([]string, 0) + for _, item := range listValue(collection["requiredFiles"]) { + requiredFiles = append(requiredFiles, stringValue(item)) + } + missingFiles := make([]string, 0) + for _, item := range listValue(collection["missingRequiredFiles"]) { + missingFiles = append(missingFiles, stringValue(item)) + } + collectionName := stringValue(collection["name"]) + if collectionName == "" { + if projectID := stringValue(meta["project_id"]); projectID != "" { + collectionName = projectID + } else if relativePath := strings.Trim(stringValue(collection["relativePath"]), "/"); relativePath != "" && relativePath != "." { + collectionName = path.Base(relativePath) + } else { + collectionName = stringValue(collection["id"]) + } + } + var resolvedScanConfig *domain.ScanConfig + if workflow != nil { + if scanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig); err == nil { + resolvedScanConfig = scanConfig + } + } + storage := assetStorageLocationFromPaths( + sourceRootPathForRun(run, resolvedScanConfig), + stringValue(collection["path"]), + func() string { + if resolvedScanConfig == nil { + return "" + } + return resolvedScanConfig.Endpoint + }(), + ) + return domain.SourceCollectionRecord{ + RunID: run.ID, + WorkflowID: run.WorkflowID, + WorkflowName: run.WorkflowName, + ProjectID: run.ProjectID, + ProjectName: run.ProjectName, + CollectionID: stringValue(collection["id"]), + CollectionName: collectionName, + Path: stringValue(collection["path"]), + RelativePath: stringValue(collection["relativePath"]), + Meta: meta, + HasIntrinsics: boolValue(collection["hasIntrinsics"]), + HasVideoMeta: boolValue(collection["hasVideoMeta"]), + RequiredFiles: requiredFiles, + MissingFiles: missingFiles, + VideoCount: intValue(collection["videoCount"]), + FileCount: intValue(collection["fileCount"]), + TotalBytes: int64Value(collection["totalBytes"]), + Valid: boolValue(collection["valid"]), + CreatedAt: run.CreatedAt, + Storage: storage, + } +} + +type sourceCollectionPayloadRecord struct { + Run *domain.RunRecord + Workflow *domain.WorkflowRecord + Collection map[string]any + Items []map[string]any + Entries []map[string]any + ArchiveSource map[string]any +} + +func (server *server) sourceCollectionPayload(ctx context.Context, runID string, collectionID string) (*sourceCollectionPayloadRecord, error) { + run, err := server.store.GetRun(ctx, runID) + if err != nil || run == nil { + return &sourceCollectionPayloadRecord{Run: run}, err + } + workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID) + if err != nil { + return &sourceCollectionPayloadRecord{Run: run}, err + } + artifact := artifactByName(*run, "dataset-index.json") + payload := artifactStructuredData(artifact) + if artifact == nil || payload == nil { + return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil + } + collections := sourceDatasetList(payload, "catalogCollections", "collections") + items := sourceDatasetList(payload, "catalogItems", "items") + entries := sourceDatasetList(payload, "catalogEntries", "entries") + var matchedCollection map[string]any + for _, item := range collections { + collection := mapValue(item) + if stringValue(collection["id"]) == collectionID { + matchedCollection = collection + break + } + } + if matchedCollection == nil { + return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil + } + matchedItems := make([]map[string]any, 0) + for _, item := range items { + row := mapValue(item) + if stringValue(row["collectionId"]) == collectionID { + matchedItems = append(matchedItems, row) + } + } + matchedEntries := make([]map[string]any, 0) + for _, item := range entries { + row := mapValue(item) + relativePath := strings.Trim(filepath.ToSlash(stringValue(row["relativePath"])), "/") + collectionRelativePath := strings.Trim(filepath.ToSlash(stringValue(matchedCollection["relativePath"])), "/") + if collectionRelativePath == "" || collectionRelativePath == "." { + if relativePath == "" { + continue + } + } else if relativePath != collectionRelativePath && !strings.HasPrefix(relativePath, collectionRelativePath+"/") { + continue + } + matchedEntries = append(matchedEntries, row) + } + return &sourceCollectionPayloadRecord{ + Run: run, + Workflow: workflow, + Collection: matchedCollection, + Items: matchedItems, + Entries: matchedEntries, + ArchiveSource: mapValue(payload["archiveSources"]), + }, nil +} + +func sourceEntryPayloadKind(relativePath string, mimeType string) string { + extension := strings.ToLower(path.Ext(relativePath)) + switch { + case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi": + return "video" + case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp": + return "image" + case strings.HasPrefix(mimeType, "audio/") || extension == ".mp3" || extension == ".wav" || extension == ".m4a": + return "audio" + default: + return "binary" + } +} + +func sourceEntryArtifactType(relativePath string, mimeType string) domain.PortType { + extension := strings.ToLower(path.Ext(relativePath)) + switch { + case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp": + return domain.PortTypeImage + case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi": + return domain.PortTypeVideo + case extension == ".json": + return domain.PortTypeJSON + default: + return domain.PortTypeFile + } +} + +func formatArtifactSize(sizeBytes int64) string { + switch { + case sizeBytes <= 0: + return "0 B" + case sizeBytes < 1024: + return fmt.Sprintf("%d B", sizeBytes) + case sizeBytes < 1024*1024: + return fmt.Sprintf("%.1f KiB", float64(sizeBytes)/1024) + case sizeBytes < 1024*1024*1024: + return fmt.Sprintf("%.1f MiB", float64(sizeBytes)/(1024*1024)) + default: + return fmt.Sprintf("%.1f GiB", float64(sizeBytes)/(1024*1024*1024)) + } +} + +func sourceCollectionContentURL(runID string, collectionID string, relativePath string) string { + return fmt.Sprintf( + "/api/source-data/collections/%s/%s/content?relativePath=%s", + url.PathEscape(runID), + url.PathEscape(collectionID), + url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")), + ) +} + +func datasetIndexEntrySource(entry map[string]any, archiveSources map[string]any) map[string]any { + source := mapValue(entry["source"]) + if len(source) == 0 { + source = make(map[string]any) + } + relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/") + if relativePath != "" && stringValue(source["relativePath"]) == "" { + source["relativePath"] = relativePath + } + mimeType := stringValue(entry["mimeType"]) + if mimeType == "" { + mimeType = stringValue(source["mimeType"]) + } + if mimeType == "" && relativePath != "" { + mimeType = mime.TypeByExtension(path.Ext(relativePath)) + } + if mimeType != "" { + source["mimeType"] = mimeType + } + if stringValue(source["previewURL"]) == "" { + archiveKey := stringValue(source["archiveKey"]) + archiveSource := mapValue(archiveSources[archiveKey]) + if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" { + source["previewURL"] = previewURL + } + if archiveName := stringValue(archiveSource["name"]); archiveName != "" && stringValue(source["archiveName"]) == "" { + source["archiveName"] = archiveName + } + } + if pathValue := stringValue(entry["path"]); pathValue != "" && stringValue(source["path"]) == "" { + source["path"] = pathValue + } + return source +} + +func sourceEntryArtifact(run domain.RunRecord, collectionID string, entry map[string]any, archiveSources map[string]any) *domain.ExecutionArtifact { + relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/") + if relativePath == "" { + relativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/") + } + if relativePath == "" { + return nil + } + source := datasetIndexEntrySource(entry, archiveSources) + mimeType := stringValue(source["mimeType"]) + sizeBytes := int64Value(entry["sizeBytes"]) + artifactType := sourceEntryArtifactType(relativePath, mimeType) + payload := make(map[string]any) + if value, ok := entry["data"]; ok { + payload["preview"] = map[string]any{ + "kind": "json", + "value": normalizeStructuredValue(value), + } + } + if previewURL := stringValue(source["previewURL"]); previewURL != "" && len(payload) == 0 { + payload["preview"] = map[string]any{ + "kind": sourceEntryPayloadKind(relativePath, mimeType), + "format": strings.TrimPrefix(strings.ToLower(path.Ext(relativePath)), "."), + "url": sourceCollectionContentURL(run.ID, collectionID, relativePath), + } + } + payload["relativePath"] = relativePath + if len(source) > 0 { + if previewURL := sourceCollectionContentURL(run.ID, collectionID, relativePath); previewURL != "" { + source["previewURL"] = previewURL + } + payload["source"] = source + } + return &domain.ExecutionArtifact{ + ID: fmt.Sprintf("%s-source-%s", run.ID, strings.ReplaceAll(relativePath, "/", "-")), + RunID: run.ID, + NodeID: "dataset-index", + Name: path.Base(relativePath), + Type: artifactType, + Size: formatArtifactSize(sizeBytes), + Path: func() string { + if sourcePath := stringValue(source["path"]); sourcePath != "" { + return sourcePath + } + return relativePath + }(), + Metadata: &domain.ArtifactMetadata{MimeType: mimeType}, + Data: payload, + } +} + +func contentBytesForDataURL(previewURL string) ([]byte, string, error) { + if !strings.HasPrefix(previewURL, "data:") { + return nil, "", fmt.Errorf("not a data url") + } + header, encoded, found := strings.Cut(previewURL, ",") + if !found || !strings.Contains(header, ";base64") { + return nil, "", fmt.Errorf("invalid data url") + } + mimeType := strings.TrimPrefix(strings.SplitN(header, ";", 2)[0], "data:") + payload, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, "", err + } + return payload, mimeType, nil +} + +func contentBytesForArchiveMember(archiveName string, archivePreviewURL string, memberPath string) ([]byte, error) { + payload, _, err := contentBytesForDataURL(archivePreviewURL) + if err != nil { + return nil, err + } + memberPath = strings.Trim(filepath.ToSlash(memberPath), "/") + if memberPath == "" { + return nil, fmt.Errorf("missing archive member path") + } + if strings.HasSuffix(strings.ToLower(archiveName), ".zip") { + archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload))) + if err != nil { + return nil, err + } + for _, file := range archive.File { + if strings.Trim(filepath.ToSlash(file.Name), "/") != memberPath { + continue + } + handle, err := file.Open() + if err != nil { + return nil, err + } + defer handle.Close() + return io.ReadAll(handle) + } + return nil, os.ErrNotExist + } + reader := bytes.NewReader(payload) + var tarReader *tar.Reader + if strings.HasSuffix(strings.ToLower(archiveName), ".tgz") || strings.HasSuffix(strings.ToLower(archiveName), ".tar.gz") || (len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b) { + gzipReader, err := gzip.NewReader(reader) + if err == nil { + defer gzipReader.Close() + tarReader = tar.NewReader(gzipReader) + } else { + reader = bytes.NewReader(payload) + tarReader = tar.NewReader(reader) + } + } else { + tarReader = tar.NewReader(reader) + } + for { + header, err := tarReader.Next() + if err == io.EOF { + return nil, os.ErrNotExist + } + if err != nil { + return nil, err + } + if header.FileInfo().IsDir() { + continue + } + if strings.Trim(filepath.ToSlash(header.Name), "/") != memberPath { + continue + } + return io.ReadAll(tarReader) + } +} + +func sourceCollectionEntryPayload(collection domain.SourceCollectionRecord, payloadEntries []map[string]any, relativePath string) map[string]any { + target := strings.Trim(filepath.ToSlash(relativePath), "/") + if target == "" { + return nil + } + collectionPrefix := strings.Trim(filepath.ToSlash(collection.RelativePath), "/") + fullTarget := target + if collectionPrefix != "" && target != collectionPrefix && !strings.HasPrefix(target, collectionPrefix+"/") { + fullTarget = collectionPrefix + "/" + target + } + for _, entry := range payloadEntries { + entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/") + if entryRelativePath == "" { + entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/") + } + if entryRelativePath == fullTarget || entryRelativePath == target { + return entry + } + } + return nil +} + +func writeSourceContent(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, payload []byte) { + if mimeType == "" { + mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName))) + } + if mimeType == "" && len(payload) > 0 { + mimeType = http.DetectContentType(payload) + } + if mimeType != "" { + writer.Header().Set("Content-Type", mimeType) + } + http.ServeContent(writer, request, fileName, time.Time{}, bytes.NewReader(payload)) +} + +func writeSourceContentFile(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, filePath string) error { + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + fileInfo, err := file.Stat() + if err != nil { + return err + } + + if mimeType == "" { + mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName))) + } + if mimeType == "" { + header := make([]byte, 512) + count, readErr := file.Read(header) + if readErr != nil && readErr != io.EOF { + return readErr + } + mimeType = http.DetectContentType(header[:count]) + if _, err := file.Seek(0, io.SeekStart); err != nil { + return err + } + } + if mimeType != "" { + writer.Header().Set("Content-Type", mimeType) + } + http.ServeContent(writer, request, fileName, fileInfo.ModTime(), file) + return nil +} + +func sourceBrowserPreviewCacheKey(request sourceBrowserPreviewRequest) string { + hash := sha256.New() + hash.Write([]byte("browser-preview-v2")) + hash.Write([]byte{0}) + hash.Write([]byte(strings.TrimSpace(request.FileName))) + hash.Write([]byte{0}) + if request.RedirectURL != "" { + hash.Write([]byte("redirect")) + hash.Write([]byte{0}) + hash.Write([]byte(strings.TrimSpace(request.RedirectURL))) + } + if len(request.Payload) > 0 { + payloadHash := sha256.Sum256(request.Payload) + hash.Write([]byte("payload")) + hash.Write([]byte{0}) + hash.Write(payloadHash[:]) + } + return hex.EncodeToString(hash.Sum(nil)) +} + +func sourceBrowserPreviewCacheDir() (string, error) { + cacheDir := filepath.Join(os.TempDir(), "droboticflow", "source-video-previews") + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", err + } + return cacheDir, nil +} + +func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string, error) { + cacheDir, err := sourceBrowserPreviewCacheDir() + if err != nil { + return "", err + } + cacheKey := sourceBrowserPreviewCacheKey(request) + return filepath.Join(cacheDir, cacheKey+".mp4"), nil +} + +func parseBOSObjectPath(sourcePath string) (string, string, bool) { + bucket, object, ok := parseBOSPath(sourcePath) + if !ok || object == "" { + return "", "", false + } + return bucket, object, true +} + +func buildSourceCollectionSignedPreviewURL(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { + if scanConfig == nil { + return "", fmt.Errorf("scan config is unavailable") + } + if chooseDataSource(scanConfig.DataSource) != "BOS" { + return "", fmt.Errorf("scan config is not BOS") + } + bucket, object, ok := parseBOSObjectPath(sourcePath) + if !ok { + return "", fmt.Errorf("invalid BOS source path: %s", sourcePath) + } + accessKeyID := strings.TrimSpace(scanConfig.AccessKeyID) + secretAccessKey := strings.TrimSpace(scanConfig.SecretAccessKey) + if accessKeyID == "" || secretAccessKey == "" { + return "", fmt.Errorf("BOS credentials are unavailable") + } + client, err := bos.NewClient(accessKeyID, secretAccessKey, chooseBOSEndpoint(scanConfig.Endpoint)) + if err != nil { + return "", err + } + previewURL := strings.TrimSpace(client.BasicGeneratePresignedUrl(bucket, object, 3600)) + if previewURL == "" { + return "", fmt.Errorf("failed to generate BOS preview url") + } + return previewURL, nil +} + +func locateSourceBrowserVideoPreviewPath(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { + finalPath, err := sourceBrowserPreviewCachePath(request) + if err != nil { + return "", err + } + if fileInfo, err := os.Stat(finalPath); err == nil && fileInfo.Size() > 0 { + return finalPath, nil + } + return "", os.ErrNotExist +} + +func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWriter, request sourceBrowserPreviewRequest) error { + if strings.TrimSpace(request.RedirectURL) == "" && len(request.Payload) == 0 { + return fmt.Errorf("source video content is unavailable") + } + + finalPath, err := sourceBrowserPreviewCachePath(request) + if err != nil { + return err + } + inputRef := strings.TrimSpace(request.RedirectURL) + var cleanupInput func() + if inputRef == "" { + extension := path.Ext(request.FileName) + if extension == "" { + extension = ".bin" + } + cacheDir := filepath.Dir(finalPath) + inputFile, err := os.CreateTemp(cacheDir, "source-video-*"+extension) + if err != nil { + return err + } + if _, err := inputFile.Write(request.Payload); err != nil { + inputFile.Close() + os.Remove(inputFile.Name()) + return err + } + if err := inputFile.Close(); err != nil { + os.Remove(inputFile.Name()) + return err + } + inputRef = inputFile.Name() + cleanupInput = func() { + _ = os.Remove(inputFile.Name()) + } + } + if cleanupInput != nil { + defer cleanupInput() + } + + cacheDir := filepath.Dir(finalPath) + outputFile, err := os.CreateTemp(cacheDir, sourceBrowserPreviewCacheKey(request)+"-*.mp4") + if err != nil { + return err + } + tempOutputPath := outputFile.Name() + stderr := &bytes.Buffer{} + command := exec.CommandContext( + ctx, + "ffmpeg", + "-y", + "-hide_banner", + "-loglevel", "error", + "-i", inputRef, + "-map", "0:v:0", + "-map", "0:a?", + "-threads", "1", + "-probesize", "32M", + "-analyzeduration", "0", + "-c:v", "libx264", + "-preset", "ultrafast", + "-tune", "zerolatency", + "-crf", "30", + "-vf", "fps=12,scale=min(960\\,iw):-2", + "-g", "12", + "-keyint_min", "12", + "-pix_fmt", "yuv420p", + "-movflags", "frag_keyframe+empty_moov+default_base_moof", + "-c:a", "aac", + "-b:a", "96k", + "-ac", "1", + "-f", "mp4", + "pipe:1", + ) + command.Stderr = stderr + stdout, err := command.StdoutPipe() + if err != nil { + outputFile.Close() + _ = os.Remove(tempOutputPath) + return err + } + if err := command.Start(); err != nil { + outputFile.Close() + _ = os.Remove(tempOutputPath) + message := strings.TrimSpace(stderr.String()) + if message != "" { + return fmt.Errorf("generate browser video preview: %s", message) + } + return err + } + streamTarget := io.Writer(writer) + if flusher, ok := writer.(http.Flusher); ok { + streamTarget = flushResponseWriter{writer: writer, flusher: flusher} + } + if _, err := io.Copy(io.MultiWriter(streamTarget, outputFile), stdout); err != nil { + outputFile.Close() + _ = os.Remove(tempOutputPath) + _ = command.Wait() + return err + } + if err := outputFile.Close(); err != nil { + _ = os.Remove(tempOutputPath) + _ = command.Wait() + return err + } + if err := command.Wait(); err != nil { + message := strings.TrimSpace(stderr.String()) + _ = os.Remove(tempOutputPath) + if message != "" { + return fmt.Errorf("generate browser video preview: %s", message) + } + return fmt.Errorf("generate browser video preview: %w", err) + } + + if err := os.Rename(tempOutputPath, finalPath); err != nil { + if fileInfo, statErr := os.Stat(finalPath); statErr == nil && fileInfo.Size() > 0 { + _ = os.Remove(tempOutputPath) + return nil + } + _ = os.Remove(tempOutputPath) + return err + } + return nil +} + +func syntheticSourceEntries(collection domain.SourceCollectionRecord, payloadItems []map[string]any) []map[string]any { + entries := make([]map[string]any, 0, len(payloadItems)+2) + if len(collection.Meta) > 0 { + entries = append(entries, map[string]any{ + "collectionId": collection.CollectionID, + "relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/meta.json", "/"), + "sizeBytes": len(fmt.Sprintf("%v", collection.Meta)), + "mimeType": "application/json", + "data": collection.Meta, + }) + } + if collection.HasVideoMeta && len(payloadItems) > 0 { + videoRows := make([]map[string]any, 0, len(payloadItems)) + for _, item := range payloadItems { + videoRows = append(videoRows, map[string]any{ + "filename": stringValue(item["filename"]), + "codec": stringValue(item["codec"]), + "resolution": stringValue(item["resolution"]), + "frameRate": floatValue(item["frameRate"]), + "duration": floatValue(item["duration"]), + "startTime": item["startTime"], + }) + } + entries = append(entries, map[string]any{ + "collectionId": collection.CollectionID, + "relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/video_meta.json", "/"), + "mimeType": "application/json", + "data": videoRows, + }) + } + for _, item := range payloadItems { + entry := map[string]any{ + "collectionId": collection.CollectionID, + "relativePath": stringValue(item["relativePath"]), + "path": stringValue(item["path"]), + "sizeBytes": int64Value(item["sizeBytes"]), + "mimeType": stringValue(mapValue(item["source"])["mimeType"]), + "source": mapValue(item["source"]), + "codec": stringValue(item["codec"]), + "resolution": stringValue(item["resolution"]), + "frameRate": floatValue(item["frameRate"]), + "duration": floatValue(item["duration"]), + } + entries = append(entries, entry) + } + return entries +} + +func buildSourceEntries(run domain.RunRecord, collection domain.SourceCollectionRecord, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, prefix string) []domain.SourceCollectionEntry { + if len(payloadEntries) == 0 { + payloadEntries = syntheticSourceEntries(collection, payloadItems) + } + videoMetaByRelative := make(map[string]map[string]any, len(payloadItems)) + for _, item := range payloadItems { + videoMetaByRelative[stringValue(item["relativePath"])] = item + } + normalizedPrefix := strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/") + prefixSlash := normalizedPrefix + if prefixSlash != "" { + prefixSlash += "/" + } + + directories := make(map[string]domain.SourceCollectionEntry) + files := make([]domain.SourceCollectionEntry, 0) + collectionPrefix := strings.Trim(collection.RelativePath, "/") + if collectionPrefix != "" { + collectionPrefix += "/" + } + + for _, entryPayload := range payloadEntries { + relativePath := strings.Trim(filepath.ToSlash(stringValue(entryPayload["relativePath"])), "/") + if relativePath == "" { + relativePath = strings.Trim(filepath.ToSlash(stringValue(entryPayload["path"])), "/") + } + if relativePath == "" { + continue + } + if collectionPrefix != "" && relativePath != strings.TrimSuffix(collectionPrefix, "/") && !strings.HasPrefix(relativePath, collectionPrefix) { + continue + } + entryPath := relativePath + if collectionPrefix != "" { + entryPath = strings.TrimPrefix(relativePath, collectionPrefix) + } + if entryPath == "" { + continue + } + if normalizedPrefix != "" { + if entryPath == normalizedPrefix { + // exact file match + } else if !strings.HasPrefix(entryPath, prefixSlash) { + continue + } else { + entryPath = strings.TrimPrefix(entryPath, prefixSlash) + } + } + if entryPath == "" { + continue + } + parts := strings.Split(entryPath, "/") + if len(parts) > 1 { + dirPath := parts[0] + displayPath := dirPath + if normalizedPrefix != "" { + displayPath = normalizedPrefix + "/" + dirPath + } + if _, exists := directories[displayPath]; !exists { + directories[displayPath] = domain.SourceCollectionEntry{ + Path: displayPath, + Name: dirPath, + Kind: "directory", + CollectionName: collection.CollectionName, + } + } + continue + } + displayPath := parts[0] + if normalizedPrefix != "" && displayPath != normalizedPrefix { + displayPath = normalizedPrefix + "/" + displayPath + } + artifact := sourceEntryArtifact(run, collection.CollectionID, entryPayload, archiveSources) + videoMeta := videoMetaByRelative[filepath.ToSlash(strings.Trim(collection.RelativePath, "/")+"/"+displayPath)] + entry := domain.SourceCollectionEntry{ + Path: displayPath, + Name: parts[0], + Kind: "file", + CollectionName: collection.CollectionName, + SizeBytes: int64Value(entryPayload["sizeBytes"]), + Artifact: artifact, + } + if videoMeta != nil { + entry.Kind = "video" + entry.Codec = stringValue(videoMeta["codec"]) + entry.Resolution = stringValue(videoMeta["resolution"]) + entry.FrameRate = floatValue(videoMeta["frameRate"]) + entry.Duration = floatValue(videoMeta["duration"]) + entry.Metadata = videoMeta + if entry.SizeBytes == 0 { + entry.SizeBytes = int64Value(videoMeta["sizeBytes"]) + } + } else if strings.HasSuffix(strings.ToLower(entry.Name), ".json") { + entry.Kind = "json" + } else if strings.HasSuffix(strings.ToLower(entry.Name), ".mp4") { + entry.Kind = "video" + } + if entry.SizeBytes == 0 { + entry.SizeBytes = int64Value(entryPayload["sizeBytes"]) + } + files = append(files, entry) + } + + items := make([]domain.SourceCollectionEntry, 0, len(directories)+len(files)) + for _, item := range directories { + items = append(items, item) + } + sort.Slice(items, func(left int, right int) bool { return items[left].Path < items[right].Path }) + sort.Slice(files, func(left int, right int) bool { return files[left].Path < files[right].Path }) + items = append(items, files...) + return items +} + +func sourceCollectionContentPayload(collection domain.SourceCollectionRecord, scanConfig *domain.ScanConfig, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, relativePath string) ([]byte, string, string, string, error) { + if len(payloadEntries) == 0 { + payloadEntries = syntheticSourceEntries(collection, payloadItems) + } + entry := sourceCollectionEntryPayload(collection, payloadEntries, relativePath) + if entry == nil { + return nil, "", "", "", os.ErrNotExist + } + entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/") + if entryRelativePath == "" { + entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/") + } + fileName := path.Base(entryRelativePath) + source := datasetIndexEntrySource(entry, archiveSources) + mimeType := stringValue(source["mimeType"]) + if mimeType == "" { + mimeType = stringValue(entry["mimeType"]) + } + if mimeType == "" { + mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName))) + } + if value, ok := entry["data"]; ok { + payload, err := json.MarshalIndent(normalizeStructuredValue(value), "", " ") + if err != nil { + return nil, "", "", "", err + } + if mimeType == "" { + mimeType = "application/json" + } + return payload, mimeType, fileName, "", nil + } + if sourcePath := stringValue(source["path"]); sourcePath != "" && !strings.Contains(sourcePath, "://") { + if payload, err := os.ReadFile(sourcePath); err == nil { + return payload, mimeType, fileName, "", nil + } + } + if sourcePath := stringValue(source["path"]); strings.HasPrefix(strings.ToLower(sourcePath), "bos://") && scanConfig != nil { + if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" { + source["previewURL"] = previewURL + } + } + archiveKey := stringValue(source["archiveKey"]) + archiveMemberPath := stringValue(source["archiveMemberPath"]) + if archiveKey != "" && archiveMemberPath != "" { + archiveSource := mapValue(archiveSources[archiveKey]) + if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" { + payload, err := contentBytesForArchiveMember(stringValue(archiveSource["name"]), previewURL, archiveMemberPath) + if err == nil { + return payload, mimeType, fileName, "", nil + } + } + } + if previewURL := stringValue(source["previewURL"]); previewURL != "" { + if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") { + return nil, "", fileName, previewURL, nil + } + payload, decodedMimeType, err := contentBytesForDataURL(previewURL) + if err == nil { + if mimeType == "" { + mimeType = decodedMimeType + } + return payload, mimeType, fileName, "", nil + } + } + return nil, "", "", "", os.ErrNotExist +} + +func (server *server) handleSourceCollectionsList(writer http.ResponseWriter, request *http.Request) { + workflowByID, latestRuns, err := server.latestCompletedScanRuns(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + items := make([]domain.SourceCollectionRecord, 0) + for _, run := range latestRuns { + artifact := artifactByName(run, "dataset-index.json") + payload := artifactStructuredData(artifact) + if artifact == nil || payload == nil { + continue + } + for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") { + collection := mapValue(raw) + if collection == nil { + continue + } + workflow := workflowByID[run.WorkflowID] + items = append(items, server.buildSourceCollectionRecord(request.Context(), run, &workflow, collection)) + } + } + sort.Slice(items, func(left int, right int) bool { + if items[left].CreatedAt == items[right].CreatedAt { + if items[left].WorkflowName == items[right].WorkflowName { + return items[left].CollectionID < items[right].CollectionID + } + return items[left].WorkflowName < items[right].WorkflowName + } + return items[left].CreatedAt > items[right].CreatedAt + }) + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleSourceCollectionDetail(writer http.ResponseWriter, request *http.Request) { + payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if payload.Run == nil || payload.Collection == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"}) + return + } + record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection) + detail := domain.SourceCollectionDetail{ + Collection: record, + Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, ""), + } + renderJSON(writer, http.StatusOK, detail) +} + +func (server *server) handleSourceCollectionEntries(writer http.ResponseWriter, request *http.Request) { + payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if payload.Run == nil || payload.Collection == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"}) + return + } + record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection) + prefix := request.URL.Query().Get("prefix") + detail := domain.SourceCollectionDetail{ + Collection: record, + Prefix: prefix, + Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, prefix), + } + renderJSON(writer, http.StatusOK, detail) +} + +func (server *server) handleSourceCollectionContent(writer http.ResponseWriter, request *http.Request) { + payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if payload.Run == nil || payload.Collection == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"}) + return + } + record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection) + relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath")) + if relativePath == "" { + renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"}) + return + } + var effectiveScanConfig *domain.ScanConfig + if payload.Workflow != nil { + effectiveScanConfig, err = server.resolveScanConfig(request.Context(), payload.Workflow.ScanConfig) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + } + content, mimeType, fileName, redirectURL, err := sourceCollectionContentPayload(record, effectiveScanConfig, payload.Entries, payload.Items, payload.ArchiveSource, relativePath) + if err != nil { + if os.IsNotExist(err) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source file not found"}) + return + } + renderError(writer, http.StatusInternalServerError, err) + return + } + if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" { + previewRequest := sourceBrowserPreviewRequest{ + FileName: fileName, + Payload: content, + RedirectURL: redirectURL, + } + if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" { + if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil { + renderError(writer, http.StatusInternalServerError, err) + } + return + } + writer.Header().Set("Content-Type", "video/mp4") + writer.Header().Set("Cache-Control", "no-store") + writer.WriteHeader(http.StatusOK) + if flusher, ok := writer.(http.Flusher); ok { + flusher.Flush() + } + if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil { + fmt.Printf("[api] source browser preview stream failed: %v\n", err) + } + return + } + if redirectURL != "" { + http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) + return + } + writeSourceContent(writer, request, fileName, mimeType, content) +} + +func packageArtifactCandidatesByRelativePath(run domain.RunRecord, packageName string, relativePath string) []domain.ExecutionArtifact { + target := strings.Trim(filepath.ToSlash(relativePath), "/") + if target == "" { + return nil + } + items := make([]domain.ExecutionArtifact, 0) + for _, artifact := range run.Artifacts { + if packageArtifactResolvedRelativePath(artifact, packageName) == target { + items = append(items, artifact) + } + } + sort.SliceStable(items, func(left int, right int) bool { + return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right]) + }) + return items +} + +func workflowNodeHyperparamsMap(node domain.WorkflowNode) map[string]string { + values := make(map[string]string, len(node.Data.Hyperparams)) + for _, item := range node.Data.Hyperparams { + values[item.Key] = item.Value + } + return values +} + +func (server *server) packageArtifactScanConfig(ctx context.Context, workflow *domain.WorkflowRecord, artifact domain.ExecutionArtifact) (*domain.ScanConfig, error) { + node := workflowNodeByID(workflow, artifact.NodeID) + if node == nil { + return nil, nil + } + params := workflowNodeHyperparamsMap(*node) + dataSource := chooseDataSource(params["DATA_SOURCE"]) + sourcePath := strings.TrimSpace(params["SOURCE_PATH"]) + if dataSource != "BOS" && strings.HasPrefix(strings.ToLower(sourcePath), "bos://") { + dataSource = "BOS" + } + if dataSource != "BOS" && strings.TrimSpace(node.Data.ConnectionID) != "" { + dataSource = "BOS" + } + if dataSource != "BOS" { + return nil, nil + } + scanConfig := &domain.ScanConfig{ + DataSource: "BOS", + SourcePath: sourcePath, + ConnectionID: strings.TrimSpace(node.Data.ConnectionID), + Bucket: strings.TrimSpace(node.Data.Bucket), + Prefix: cleanBOSPrefix(node.Data.Prefix), + Endpoint: strings.TrimSpace(params["BOS_ENDPOINT"]), + AccessKeyID: strings.TrimSpace(params["BOS_ACCESS_KEY_ID"]), + SecretAccessKey: strings.TrimSpace(params["BOS_SECRET_ACCESS_KEY"]), + } + return server.resolveScanConfig(ctx, scanConfig) +} + +func (server *server) packageContentPayload(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, packageName string, relativePath string) ([]byte, string, string, string, string, error) { + artifacts := packageArtifactCandidatesByRelativePath(run, packageName, relativePath) + if len(artifacts) == 0 { + return nil, "", "", "", "", os.ErrNotExist + } + fileName := path.Base(strings.Trim(filepath.ToSlash(relativePath), "/")) + for _, artifact := range artifacts { + mimeType := "" + if artifact.Metadata != nil { + mimeType = strings.TrimSpace(artifact.Metadata.MimeType) + } + if mimeType == "" { + mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName))) + } + filePath := strings.TrimSpace(artifact.Path) + if filePath != "" && !strings.Contains(filePath, "://") { + if _, err := os.Stat(filePath); err == nil { + return nil, mimeType, fileName, filePath, "", nil + } + } + structuredData := artifactStructuredData(&artifact) + if artifact.Type == "json" && structuredData != nil { + payload, err := json.MarshalIndent(normalizeStructuredValue(structuredData), "", " ") + if err != nil { + return nil, "", "", "", "", err + } + if mimeType == "" { + mimeType = "application/json" + } + return payload, mimeType, fileName, "", "", nil + } + source := mapValue(structuredData["source"]) + sourcePath := stringValue(source["path"]) + if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") { + scanConfig, err := server.packageArtifactScanConfig(ctx, workflow, artifact) + if err != nil { + return nil, "", "", "", "", err + } + if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" { + return nil, mimeType, fileName, "", previewURL, nil + } + } + if previewURL := packageArtifactPreviewURL(artifact); previewURL != "" { + if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") { + return nil, mimeType, fileName, "", previewURL, nil + } + payload, decodedMimeType, err := contentBytesForDataURL(previewURL) + if err == nil { + if mimeType == "" { + mimeType = decodedMimeType + } + return payload, mimeType, fileName, "", "", nil + } + } + } + return nil, "", "", "", "", os.ErrNotExist +} + +func (server *server) handlePackageContent(writer http.ResponseWriter, request *http.Request) { + run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if run == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) + return + } + packageName := chi.URLParam(request, "packageName") + relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath")) + if relativePath == "" { + renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"}) + return + } + workflow, err := server.store.GetWorkflow(request.Context(), run.WorkflowID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + content, mimeType, fileName, filePath, redirectURL, err := server.packageContentPayload(request.Context(), *run, workflow, packageName, relativePath) + if err != nil { + if os.IsNotExist(err) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package file not found"}) + return + } + renderError(writer, http.StatusInternalServerError, err) + return + } + if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" { + previewRequest := sourceBrowserPreviewRequest{ + FileName: fileName, + Payload: content, + } + if filePath != "" { + previewRequest.RedirectURL = filePath + } else { + previewRequest.RedirectURL = redirectURL + } + if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" { + if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil { + renderError(writer, http.StatusInternalServerError, err) + } + return + } + writer.Header().Set("Content-Type", "video/mp4") + writer.Header().Set("Cache-Control", "no-store") + writer.WriteHeader(http.StatusOK) + if flusher, ok := writer.(http.Flusher); ok { + flusher.Flush() + } + if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil { + fmt.Printf("[api] package browser preview stream failed: %v\n", err) + } + return + } + if filePath != "" { + if err := writeSourceContentFile(writer, request, fileName, mimeType, filePath); err != nil { + renderError(writer, http.StatusInternalServerError, err) + } + return + } + if redirectURL != "" { + http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) + return + } + writeSourceContent(writer, request, fileName, mimeType, content) +} + +func packageRecordFromManifest(run domain.RunRecord, item map[string]any, files []domain.ExecutionArtifact, rawFiles []domain.ExecutionArtifact) domain.PackageRecord { + pipelineTags := make([]string, 0) + for _, value := range listValue(item["pipelineTags"]) { + pipelineTags = append(pipelineTags, stringValue(value)) + } + storagePath := stringValue(item["storagePath"]) + for _, artifact := range rawFiles { + if candidate := packageArtifactStoragePath(artifact, stringValue(item["packageName"])); candidate != "" { + storagePath = candidate + break + } + } + fileCount := intValue(item["fileCount"]) + if len(files) > 0 { + fileCount = len(files) + } + return domain.PackageRecord{ + RunID: run.ID, + WorkflowID: run.WorkflowID, + WorkflowName: run.WorkflowName, + ProjectID: run.ProjectID, + ProjectName: run.ProjectName, + PackageName: stringValue(item["packageName"]), + ClipID: stringValue(item["clipId"]), + SourceVideo: stringValue(item["sourceVideo"]), + ClipDuration: floatValue(item["clipDuration"]), + FileCount: fileCount, + PipelineTags: pipelineTags, + StoragePath: storagePath, + Quality: mapValue(item["quality"]), + Annotation: mapValue(item["annotation"]), + CreatedAt: run.CreatedAt, + PreviewArtifact: choosePackagePreview(files), + Storage: assetStorageLocationFromPaths("", storagePath, ""), + } +} + +func (server *server) packageDetail(ctx context.Context, runID string, packageName string) (*domain.PackageDetail, error) { + run, err := server.store.GetRun(ctx, runID) + if err != nil || run == nil { + return nil, err + } + manifest := artifactByName(*run, "packages-manifest.json") + manifestPayload := artifactStructuredData(manifest) + if manifest == nil || manifestPayload == nil { + return nil, nil + } + rawFiles := dedupePackageArtifacts(*run, packageName) + files := make([]domain.ExecutionArtifact, 0, len(rawFiles)) + for _, artifact := range rawFiles { + files = append(files, packageArtifactForResponse(run.ID, packageName, artifact)) + } + for _, raw := range listValue(manifestPayload["items"]) { + item := mapValue(raw) + if stringValue(item["packageName"]) != packageName { + continue + } + record := packageRecordFromManifest(*run, item, files, rawFiles) + return &domain.PackageDetail{Package: record, Files: files}, nil + } + return nil, nil +} + +func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) { + runs, err := server.store.ListRuns(request.Context(), string(domain.RunStatusCompleted)) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + items := make([]domain.PackageRecord, 0) + latestByWorkflowPackage := make(map[string]domain.PackageRecord) + for _, run := range runs { + manifest := artifactByName(run, "packages-manifest.json") + manifestPayload := artifactStructuredData(manifest) + if manifest == nil || manifestPayload == nil { + continue + } + for _, raw := range listValue(manifestPayload["items"]) { + item := mapValue(raw) + if item == nil { + continue + } + packageName := stringValue(item["packageName"]) + rawFiles := dedupePackageArtifacts(run, packageName) + files := make([]domain.ExecutionArtifact, 0, len(rawFiles)) + for _, artifact := range rawFiles { + files = append(files, packageArtifactForResponse(run.ID, packageName, artifact)) + } + record := packageRecordFromManifest(run, item, files, rawFiles) + key := run.WorkflowID + ":" + record.PackageName + current, exists := latestByWorkflowPackage[key] + if !exists || record.CreatedAt > current.CreatedAt { + latestByWorkflowPackage[key] = record + } + } + } + for _, item := range latestByWorkflowPackage { + items = append(items, item) + } + sort.Slice(items, func(left int, right int) bool { + if items[left].CreatedAt == items[right].CreatedAt { + return items[left].PackageName < items[right].PackageName + } + return items[left].CreatedAt > items[right].CreatedAt + }) + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handlePackageDetail(writer http.ResponseWriter, request *http.Request) { + item, err := server.packageDetail(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "packageName")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"}) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func samplingRulesFromManifest(manifest map[string]any) []domain.SamplingRuleSummary { + rules := make([]domain.SamplingRuleSummary, 0) + for _, raw := range listValue(manifest["rules"]) { + item := mapValue(raw) + if item == nil { + continue + } + rules = append(rules, domain.SamplingRuleSummary{ + ID: stringValue(item["id"]), + Field: stringValue(item["field"]), + Operator: stringValue(item["operator"]), + Value: item["value"], + SamplePercent: floatValue(item["samplePercent"]), + MatchedCount: intValue(item["matchedCount"]), + SampledCount: intValue(item["sampledCount"]), + }) + } + return rules +} + +func buildSamplingItems(run domain.RunRecord, manifest map[string]any) []domain.SamplingItemRecord { + items := make([]domain.SamplingItemRecord, 0) + previewByPackage := make(map[string]*domain.ExecutionArtifact) + for _, raw := range listValue(manifest["items"]) { + item := mapValue(raw) + packageName := stringValue(item["packageName"]) + if _, ok := previewByPackage[packageName]; !ok { + previewByPackage[packageName] = choosePackagePreview(packageArtifacts(run, packageName)) + } + preview := previewByPackage[packageName] + items = append(items, domain.SamplingItemRecord{ + ID: domain.NewID("sampling-item"), + PackageName: packageName, + ClipID: stringValue(item["clipId"]), + SourceVideo: stringValue(item["sourceVideo"]), + SampleRuleID: stringValue(item["sampleRuleId"]), + StoragePath: stringValue(item["storagePath"]), + Quality: mapValue(item["quality"]), + Annotation: mapValue(item["annotation"]), + PreviewArtifact: preview, + Status: domain.SamplingItemStatusPending, + }) + } + return items +} + +func (server *server) syncSamplingTasks(ctx context.Context) error { + existing, err := server.store.ListSamplingTasks(ctx, "") + if err != nil { + return err + } + existingByRun := make(map[string]domain.SamplingTaskRecord, len(existing)) + for _, task := range existing { + existingByRun[task.RunID] = task + } + runs, err := server.store.ListRuns(ctx, string(domain.RunStatusCompleted)) + if err != nil { + return err + } + for _, run := range runs { + if _, exists := existingByRun[run.ID]; exists { + continue + } + manifest := artifactByName(run, "audit-samples.json") + manifestPayload := artifactStructuredData(manifest) + if manifest == nil || manifestPayload == nil || len(listValue(manifestPayload["items"])) == 0 { + continue + } + now := nowISO() + items := buildSamplingItems(run, manifestPayload) + task := domain.SamplingTaskRecord{ + ID: domain.NewID("sampling"), + RunID: run.ID, + WorkflowID: run.WorkflowID, + WorkflowName: run.WorkflowName, + ProjectID: run.ProjectID, + ProjectName: run.ProjectName, + Name: fmt.Sprintf("%s 抽检任务", run.WorkflowName), + Description: fmt.Sprintf("来源于运行 %s 的真实抽检采样结果", run.ID), + Status: domain.SamplingTaskStatusPending, + Rules: samplingRulesFromManifest(manifestPayload), + Items: items, + CreatedAt: now, + UpdatedAt: now, + } + for index := range task.Items { + task.Items[index].TaskID = task.ID + } + if _, err := server.store.CreateSamplingTask(ctx, task); err != nil { + return err + } + } + return nil +} + +func (server *server) handleSamplingTasksList(writer http.ResponseWriter, request *http.Request) { + if err := server.syncSamplingTasks(request.Context()); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + items, err := server.store.ListSamplingTasks(request.Context(), request.URL.Query().Get("status")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleSamplingTaskGet(writer http.ResponseWriter, request *http.Request) { + if err := server.syncSamplingTasks(request.Context()); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + item, err := server.store.GetSamplingTask(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling task not found"}) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func validateSamplingReviewPayload(payload samplingReviewRequest) []validationIssue { + issues := make([]validationIssue, 0, 4) + if strings.TrimSpace(payload.VideoQuality) == "" { + issues = append(issues, validationIssue{Field: "videoQuality", Message: "videoQuality is required"}) + } + if strings.TrimSpace(payload.AnnotationAccuracy) == "" { + issues = append(issues, validationIssue{Field: "annotationAccuracy", Message: "annotationAccuracy is required"}) + } + if strings.TrimSpace(payload.AnnotationCompleteness) == "" { + issues = append(issues, validationIssue{Field: "annotationCompleteness", Message: "annotationCompleteness is required"}) + } + if strings.TrimSpace(payload.OverallGrade) == "" { + issues = append(issues, validationIssue{Field: "overallGrade", Message: "overallGrade is required"}) + } + return issues +} + +func (server *server) handleSamplingItemReview(writer http.ResponseWriter, request *http.Request) { + if err := server.syncSamplingTasks(request.Context()); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + var payload samplingReviewRequest + if !decodeJSON(writer, request, &payload) { + return + } + if issues := validateSamplingReviewPayload(payload); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + tasks, err := server.store.ListSamplingTasks(request.Context(), "") + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + itemID := chi.URLParam(request, "id") + for _, task := range tasks { + updated := false + for index := range task.Items { + if task.Items[index].ID != itemID { + continue + } + task.Items[index].Status = domain.SamplingItemStatusCompleted + task.Items[index].Review = &domain.SamplingReview{ + VideoQuality: strings.TrimSpace(payload.VideoQuality), + AnnotationAccuracy: strings.TrimSpace(payload.AnnotationAccuracy), + AnnotationCompleteness: strings.TrimSpace(payload.AnnotationCompleteness), + OverallGrade: strings.TrimSpace(payload.OverallGrade), + Notes: strings.TrimSpace(payload.Notes), + ReviewedAt: nowISO(), + } + updated = true + break + } + if !updated { + continue + } + task.Status = domain.SamplingTaskStatusCompleted + for _, item := range task.Items { + if item.Status != domain.SamplingItemStatusCompleted { + task.Status = domain.SamplingTaskStatusPending + break + } + } + task.UpdatedAt = nowISO() + if err := server.store.UpsertSamplingTask(request.Context(), task); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, task) + return + } + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling item not found"}) +} diff --git a/internal/api/data_platform_test.go b/internal/api/data_platform_test.go new file mode 100644 index 00000000..6d85fa87 --- /dev/null +++ b/internal/api/data_platform_test.go @@ -0,0 +1,1736 @@ +package api + +import ( + "archive/tar" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" + + "go.mongodb.org/mongo-driver/bson" +) + +func tarDataURL(t *testing.T, files map[string][]byte) string { + t.Helper() + var buffer bytes.Buffer + writer := tar.NewWriter(&buffer) + for name, payload := range files { + header := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(payload)), + } + if err := writer.WriteHeader(header); err != nil { + t.Fatalf("write tar header: %v", err) + } + if _, err := writer.Write(payload); err != nil { + t.Fatalf("write tar payload: %v", err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("close tar writer: %v", err) + } + return "data:application/x-tar;base64," + base64.StdEncoding.EncodeToString(buffer.Bytes()) +} + +func externalizedJSONArtifact(t *testing.T, runID string, nodeID string, name string, payload map[string]any) domain.ExecutionArtifact { + t.Helper() + root := t.TempDir() + payloadPath := filepath.Join(root, name) + payloadBytes, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + if err := os.WriteFile(payloadPath, payloadBytes, 0o644); err != nil { + t.Fatalf("write payload: %v", err) + } + manifestPath := filepath.Join(root, strings.TrimSuffix(name, filepath.Ext(name))+"-manifest.json") + manifestBytes, err := json.Marshal(map[string]any{ + "kind": "dataset", + "datasetType": payload["datasetType"], + "payloadPath": payloadPath, + "summary": map[string]any{"name": name}, + }) + if err != nil { + t.Fatalf("marshal manifest: %v", err) + } + if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } + return domain.ExecutionArtifact{ + ID: runID + "-" + strings.TrimSuffix(name, filepath.Ext(name)), + RunID: runID, + NodeID: nodeID, + Name: name, + Type: domain.PortTypeJSON, + Path: payloadPath, + Data: map[string]any{ + "datasetType": payload["datasetType"], + "dataRef": map[string]any{ + "kind": "dataset", + "datasetType": payload["datasetType"], + "manifestPath": manifestPath, + "payloadPath": payloadPath, + }, + }, + } +} + +func TestScanConfigBOSSecretsAreReturnedAndPreserved(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} { + if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ + Name: name, + Type: "builtin", + Category: "io", + Description: name, + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}}, + }); err != nil { + t.Fatalf("create operator %s: %v", name, err) + } + } + handler := NewHandler(dataStore, nil) + + createPayload := `{ + "name":"BOS 扫描", + "description":"扫描真实 BOS 目录", + "dataSource":"BOS", + "bucket":"datastorage", + "prefix":"algo/X-platform-test/", + "endpoint":"https://bj.bcebos.com", + "accessKeyId":"ak-example-1234", + "secretAccessKey":"sk-example-5678", + "interval":86400, + "enabled":true + }` + + createRecorder := httptest.NewRecorder() + handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload))) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create scan config status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + var created domain.WorkflowRecord + if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created scan config: %v", err) + } + if created.ScanConfig == nil { + t.Fatalf("expected scan config") + } + if created.ScanConfig.AccessKeyID != "ak-example-1234" || created.ScanConfig.SecretAccessKey != "sk-example-5678" { + t.Fatalf("scan config secrets should be returned in workflow responses: %+v", created.ScanConfig) + } + if !created.ScanConfig.HasAccessKeyID || !created.ScanConfig.HasSecretAccessKey { + t.Fatalf("expected secret presence flags in API response: %+v", created.ScanConfig) + } + if created.ScanConfig.AccessKeyPreview == "" || created.ScanConfig.SecretAccessKeyPreview == "" { + t.Fatalf("expected secret previews in API response: %+v", created.ScanConfig) + } + + stored, err := dataStore.GetWorkflow(ctx, created.ID) + if err != nil { + t.Fatalf("get stored workflow: %v", err) + } + if stored == nil || stored.ScanConfig == nil { + t.Fatalf("expected stored scan config") + } + if stored.ScanConfig.AccessKeyID != "ak-example-1234" || stored.ScanConfig.SecretAccessKey != "sk-example-5678" { + t.Fatalf("expected secrets preserved in store: %+v", stored.ScanConfig) + } + + updatePayload := `{ + "name":"BOS 扫描更新", + "description":"保留原始密钥", + "dataSource":"BOS", + "bucket":"datastorage", + "prefix":"algo/X-platform-test/", + "endpoint":"https://bj.bcebos.com", + "interval":3600, + "enabled":false + }` + updateRecorder := httptest.NewRecorder() + handler.ServeHTTP(updateRecorder, httptest.NewRequest(http.MethodPut, "/api/scan-configs/"+created.ID, strings.NewReader(updatePayload))) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("update scan config status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + + updated, err := dataStore.GetWorkflow(ctx, created.ID) + if err != nil { + t.Fatalf("get updated workflow: %v", err) + } + if updated == nil || updated.ScanConfig == nil { + t.Fatalf("expected updated scan config") + } + if updated.ScanConfig.AccessKeyID != "ak-example-1234" || updated.ScanConfig.SecretAccessKey != "sk-example-5678" { + t.Fatalf("expected secrets preserved across updates: %+v", updated.ScanConfig) + } +} + +func TestPackagesAndSamplingCanLoadExternalizedManifestPayloads(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "外置成品项目"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "成品工作流", + Type: domain.WorkflowTypeBatch, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + packagesPayload := map[string]any{ + "datasetType": "x-platform-packages", + "packageCount": 1, + "items": []any{ + map[string]any{ + "packageName": "DJI_0001_D_clip001_package", + "clipId": "DJI_0001_D_clip001", + "sourceVideo": "DJI_0001_D.mp4", + "fileCount": 1, + "storagePath": "/tmp/k8s-job/package", + }, + }, + } + samplingPayload := map[string]any{ + "datasetType": "x-platform-audit-samples", + "rules": []any{ + map[string]any{ + "id": "confidence-check", + "field": "annotation.minConfidence", + "operator": "lt", + "value": 0.7, + "samplePercent": 10, + "matchedCount": 1, + "sampledCount": 1, + }, + }, + "items": []any{ + map[string]any{ + "packageName": "DJI_0001_D_clip001_package", + "clipId": "DJI_0001_D_clip001", + "sourceVideo": "DJI_0001_D.mp4", + "sampleRuleId": "confidence-check", + "storagePath": "/tmp/k8s-job/package", + }, + }, + } + + run := domain.RunRecord{ + ID: "run-externalized-package", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + externalizedJSONArtifact(t, "run-externalized-package", "package", "packages-manifest.json", packagesPayload), + externalizedJSONArtifact(t, "run-externalized-package", "sample", "audit-samples.json", samplingPayload), + }, + CreatedAt: "2026-04-10T10:05:00Z", + UpdatedAt: "2026-04-10T10:05:00Z", + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + handler := NewHandler(dataStore, nil) + + packagesRecorder := httptest.NewRecorder() + handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil)) + if packagesRecorder.Code != http.StatusOK { + t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String()) + } + var packagesBody struct { + Items []domain.PackageRecord `json:"items"` + } + if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil { + t.Fatalf("decode packages: %v", err) + } + if len(packagesBody.Items) != 1 { + t.Fatalf("expected one package from externalized manifest, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String()) + } + + samplingRecorder := httptest.NewRecorder() + handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil)) + if samplingRecorder.Code != http.StatusOK { + t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String()) + } + var samplingBody struct { + Items []domain.SamplingTaskRecord `json:"items"` + } + if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil { + t.Fatalf("decode sampling tasks: %v", err) + } + if len(samplingBody.Items) != 1 || len(samplingBody.Items[0].Items) != 1 { + t.Fatalf("expected one sampling task with one item from externalized manifest, got %#v", samplingBody.Items) + } +} + +func TestScanConfigCreateDraftDoesNotEnsureSystemProject(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} { + if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ + Name: name, + Type: "builtin", + Category: "io", + Description: name, + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}}, + }); err != nil { + t.Fatalf("create operator %s: %v", name, err) + } + } + handler := NewHandler(dataStore, nil) + + createPayload := `{ + "name":"BOS 草稿扫描", + "description":"创建后再配 bucket/prefix", + "dataSource":"BOS", + "interval":0, + "enabled":false + }` + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload))) + if recorder.Code != http.StatusCreated { + t.Fatalf("create draft scan config status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var workflow domain.WorkflowRecord + if err := json.Unmarshal(recorder.Body.Bytes(), &workflow); err != nil { + t.Fatalf("decode draft workflow: %v", err) + } + if workflow.ProjectID != "" { + t.Fatalf("expected projectless scan workflow, got projectId=%q", workflow.ProjectID) + } + if workflow.Kind != domain.WorkflowKindScan { + t.Fatalf("expected scan workflow kind, got %s", workflow.Kind) + } + + projects, err := dataStore.ListProjects(ctx) + if err != nil { + t.Fatalf("list projects: %v", err) + } + if len(projects) != 0 { + t.Fatalf("expected no auto-created system project, got %d", len(projects)) + } +} + +func TestDataPlatformHandlersSupportMongoStyleArtifactData(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "数据资产项目", + Description: "用于数据资产 API 测试", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + scanWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "目录扫描", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + }) + if err != nil { + t.Fatalf("create scan workflow: %v", err) + } + + pipelineWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "成品流水线", + Type: domain.WorkflowTypeBatch, + }) + if err != nil { + t.Fatalf("create pipeline workflow: %v", err) + } + + scanRun := domain.RunRecord{ + ID: "run-scan-1", + WorkflowID: scanWorkflow.ID, + WorkflowName: scanWorkflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + { + ID: "run-scan-1-source", + RunID: "run-scan-1", + NodeID: "scan-source", + Name: "source-manifest.json", + Type: "json", + Data: map[string]any{ + "dataSource": "BOS", + "sourcePath": "bos://datastorage/algo/X-platform-test", + "listedCount": 4, + "newCount": 4, + }, + }, + { + ID: "run-scan-1-index", + RunID: "run-scan-1", + NodeID: "scan-index", + Name: "dataset-index.json", + Type: "json", + Data: map[string]any{ + "archiveSources": bson.M{ + "archive-1": bson.M{ + "name": "xplatform-dataset.tar", + "previewURL": tarDataURL(t, map[string][]byte{ + "BJ_001/DJI_001/DJI_0001_D.mp4": []byte("video-preview-bytes"), + }), + }, + }, + "collectionCount": 1, + "collections": bson.A{ + bson.M{ + "id": "BJ_001", + "path": "/tmp/run-scan-1/index/_materialized_inputs/BJ_001", + "relativePath": "BJ_001", + "meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"}, + "hasIntrinsics": true, + "hasVideoMeta": true, + "requiredFiles": bson.A{"meta.json", "video_meta.json"}, + "missingRequiredFiles": bson.A{}, + "videoCount": 1, + "fileCount": 4, + "totalBytes": 1234, + "valid": true, + }, + }, + "items": bson.A{ + bson.M{ + "collectionId": "BJ_001", + "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", + "filename": "DJI_0001_D.mp4", + "codec": "hevc", + "resolution": "3840x2160", + "frameRate": 29.97, + "duration": 41.8, + "sizeBytes": 2048, + "meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"}, + "source": bson.M{ + "archiveKey": "archive-1", + "archiveMemberPath": "BJ_001/DJI_001/DJI_0001_D.mp4", + "mimeType": "video/mp4", + "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4", + "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", + }, + }, + }, + }, + }, + }, + CreatedAt: "2026-04-10T10:00:00Z", + UpdatedAt: "2026-04-10T10:00:00Z", + } + if err := dataStore.UpsertRun(ctx, scanRun); err != nil { + t.Fatalf("upsert scan run: %v", err) + } + + pipelineRun := domain.RunRecord{ + ID: "run-pipeline-1", + WorkflowID: pipelineWorkflow.ID, + WorkflowName: pipelineWorkflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + { + ID: "run-pipeline-1-packages", + RunID: "run-pipeline-1", + NodeID: "package", + Name: "packages-manifest.json", + Type: "json", + Data: map[string]any{ + "packageCount": 1, + "items": bson.A{ + bson.M{ + "packageName": "DJI_0001_D_clip001_package", + "clipId": "DJI_0001_D_clip001", + "sourceVideo": "DJI_0001_D.mp4", + "clipDuration": 300, + "fileCount": 4, + "pipelineTags": bson.A{"quality_v1", "annotation_v1"}, + "storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package", + "quality": bson.M{"clarity": 0.92, "shake": 0.08}, + "annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}}, + }, + }, + }, + }, + { + ID: "run-pipeline-1-audit", + RunID: "run-pipeline-1", + NodeID: "sample", + Name: "audit-samples.json", + Type: "json", + Data: map[string]any{ + "rules": bson.A{ + bson.M{ + "id": "confidence-check", + "field": "annotation.minConfidence", + "operator": "lt", + "value": 0.7, + "samplePercent": 10, + "matchedCount": 1, + "sampledCount": 1, + }, + }, + "items": bson.A{ + bson.M{ + "packageName": "DJI_0001_D_clip001_package", + "clipId": "DJI_0001_D_clip001", + "sourceVideo": "DJI_0001_D.mp4", + "sampleRuleId": "confidence-check", + "storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package", + "quality": bson.M{"clarity": 0.92, "shake": 0.08}, + "annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}}, + }, + }, + }, + }, + { + ID: "run-pipeline-1-index", + RunID: "run-pipeline-1", + NodeID: "setup-index", + Name: "dataset-index.json", + Type: "json", + Data: map[string]any{ + "archiveSources": bson.M{ + "archive-test": bson.M{"name": "synthetic-dataset.tgz", "previewURL": "data:application/x-tar;base64,AAAA"}, + }, + "collectionCount": 1, + "collections": bson.A{ + bson.M{ + "id": "synthetic-small", + "name": "synthetic-small", + "path": "/tmp/droboticflow/run-pipeline-1/index/_materialized_inputs/synthetic-small", + "relativePath": "synthetic-small", + "fileCount": 1, + "totalBytes": 21, + "valid": true, + }, + }, + }, + }, + { + ID: "run-pipeline-1-video", + RunID: "run-pipeline-1", + NodeID: "package", + Name: "DJI_0001_D_clip001.mp4", + Type: "video", + Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + Data: map[string]any{ + "preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"}, + }, + }, + { + ID: "run-pipeline-1-quality", + RunID: "run-pipeline-1", + NodeID: "package", + Name: "DJI_0001_D_clip001_quality.json", + Type: "json", + Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001_quality.json", + Data: map[string]any{"clarity": 0.92}, + }, + }, + CreatedAt: "2026-04-10T10:05:00Z", + UpdatedAt: "2026-04-10T10:05:00Z", + } + if err := dataStore.UpsertRun(ctx, pipelineRun); err != nil { + t.Fatalf("upsert pipeline run: %v", err) + } + + handler := NewHandler(dataStore, nil) + + packagesRecorder := httptest.NewRecorder() + handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil)) + if packagesRecorder.Code != http.StatusOK { + t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String()) + } + var packagesBody struct { + Items []domain.PackageRecord `json:"items"` + } + if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil { + t.Fatalf("decode packages: %v", err) + } + if len(packagesBody.Items) != 1 { + t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String()) + } + if packagesBody.Items[0].PackageName != "DJI_0001_D_clip001_package" { + t.Fatalf("unexpected package name: %s", packagesBody.Items[0].PackageName) + } + if packagesBody.Items[0].Storage.Provider != "k8s-job" { + t.Fatalf("expected k8s-job package storage, got %+v", packagesBody.Items[0].Storage) + } + if packagesBody.Items[0].Storage.Prefix != "/tmp/run-pipeline-1/package/packages" { + t.Fatalf("unexpected package storage prefix: %+v", packagesBody.Items[0].Storage) + } + + packageDetailRecorder := httptest.NewRecorder() + handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package", nil)) + if packageDetailRecorder.Code != http.StatusOK { + t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String()) + } + var packageDetail domain.PackageDetail + if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil { + t.Fatalf("decode package detail: %v", err) + } + if packageDetail.Package.Storage.Path != "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package" { + t.Fatalf("unexpected package storage path: %+v", packageDetail.Package.Storage) + } + if len(packageDetail.Files) == 0 { + t.Fatalf("expected package files") + } + videoPreview := mapValue(packageDetail.Files[0].Data["preview"]) + if !strings.Contains(stringValue(videoPreview["url"]), "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package/content?relativePath=") { + t.Fatalf("expected package video preview url to use content api, got %+v", packageDetail.Files[0].Data) + } + + samplingRecorder := httptest.NewRecorder() + handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil)) + if samplingRecorder.Code != http.StatusOK { + t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String()) + } + var samplingBody struct { + Items []domain.SamplingTaskRecord `json:"items"` + } + if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil { + t.Fatalf("decode sampling tasks: %v", err) + } + if len(samplingBody.Items) != 1 { + t.Fatalf("expected one sampling task, got %d body=%s", len(samplingBody.Items), samplingRecorder.Body.String()) + } + if len(samplingBody.Items[0].Items) != 1 { + t.Fatalf("expected one sampling item, got %d", len(samplingBody.Items[0].Items)) + } + + collectionsRecorder := httptest.NewRecorder() + handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil)) + if collectionsRecorder.Code != http.StatusOK { + t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String()) + } + var collectionsBody struct { + Items []domain.SourceCollectionRecord `json:"items"` + } + if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil { + t.Fatalf("decode source collections: %v", err) + } + if len(collectionsBody.Items) != 1 { + t.Fatalf("expected one source collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String()) + } + if collectionsBody.Items[0].CollectionID != "BJ_001" { + t.Fatalf("unexpected collection id: %s", collectionsBody.Items[0].CollectionID) + } + if collectionsBody.Items[0].CollectionName != "BJ_001" { + t.Fatalf("unexpected collection name fallback: %s", collectionsBody.Items[0].CollectionName) + } + if collectionsBody.Items[0].RunID != "run-scan-1" { + t.Fatalf("expected source collections to ignore synthetic pipeline dataset indexes, got run %s", collectionsBody.Items[0].RunID) + } + if collectionsBody.Items[0].Storage.Provider != "bos" { + t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage) + } + if collectionsBody.Items[0].Storage.Bucket != "datastorage" || collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" { + t.Fatalf("unexpected source storage details: %+v", collectionsBody.Items[0].Storage) + } + + detailRecorder := httptest.NewRecorder() + handler.ServeHTTP(detailRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001", nil)) + if detailRecorder.Code != http.StatusOK { + t.Fatalf("source detail status: %d body=%s", detailRecorder.Code, detailRecorder.Body.String()) + } + var detail domain.SourceCollectionDetail + if err := json.Unmarshal(detailRecorder.Body.Bytes(), &detail); err != nil { + t.Fatalf("decode source detail: %v", err) + } + if len(detail.Entries) != 3 { + t.Fatalf("expected root entries synthesized from dataset index, got %d body=%s", len(detail.Entries), detailRecorder.Body.String()) + } + var videoDirectory *domain.SourceCollectionEntry + for index := range detail.Entries { + if detail.Entries[index].Kind == "directory" && detail.Entries[index].Path == "DJI_001" { + videoDirectory = &detail.Entries[index] + break + } + } + if videoDirectory == nil { + t.Fatalf("expected DJI_001 directory in synthesized source entries: %+v", detail.Entries) + } + + entryRecorder := httptest.NewRecorder() + handler.ServeHTTP(entryRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/entries?prefix=DJI_001", nil)) + if entryRecorder.Code != http.StatusOK { + t.Fatalf("source entries status: %d body=%s", entryRecorder.Code, entryRecorder.Body.String()) + } + var nestedDetail domain.SourceCollectionDetail + if err := json.Unmarshal(entryRecorder.Body.Bytes(), &nestedDetail); err != nil { + t.Fatalf("decode nested source entries: %v", err) + } + if len(nestedDetail.Entries) != 1 { + t.Fatalf("expected one nested video entry, got %d body=%s", len(nestedDetail.Entries), entryRecorder.Body.String()) + } + videoEntry := nestedDetail.Entries[0] + if videoEntry.Kind != "video" { + t.Fatalf("expected nested entry to be video, got %s", videoEntry.Kind) + } + if videoEntry.Artifact == nil { + t.Fatalf("expected nested video entry artifact") + } + preview := mapValue(videoEntry.Artifact.Data["preview"]) + if !strings.Contains(stringValue(preview["url"]), "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=") { + t.Fatalf("expected nested video preview url from archive sources: %+v", videoEntry.Artifact.Data) + } + + contentRecorder := httptest.NewRecorder() + contentRequest := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", nil) + handler.ServeHTTP(contentRecorder, contentRequest) + if contentRecorder.Code != http.StatusOK { + t.Fatalf("source content status: %d body=%s", contentRecorder.Code, contentRecorder.Body.String()) + } + if contentRecorder.Header().Get("Content-Type") != "video/mp4" { + t.Fatalf("unexpected source content type: %s", contentRecorder.Header().Get("Content-Type")) + } + if string(contentRecorder.Body.Bytes()) != "video-preview-bytes" { + t.Fatalf("unexpected source content payload: %q", contentRecorder.Body.Bytes()) + } +} + +func TestSourceCollectionsUseCatalogFieldsForIncrementalScanRuns(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "BOS 增量扫描", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + collection := bson.M{ + "id": "BJ_001", + "name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43", + "path": "bos://datastorage/algo/X-platform-test/BJ_001", + "relativePath": "BJ_001", + "meta": bson.M{"project_id": "BJ_001"}, + "hasVideoMeta": true, + "videoCount": 1, + "fileCount": 4, + "totalBytes": int64(16000036740), + "valid": true, + "requiredFiles": bson.A{"meta.json", "video_meta.json"}, + } + catalogEntries := bson.A{ + bson.M{ + "collectionId": "BJ_001", + "relativePath": "BJ_001/meta.json", + "sizeBytes": 256, + "mimeType": "application/json", + "data": bson.M{"project_id": "BJ_001"}, + }, + bson.M{ + "collectionId": "BJ_001", + "relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4", + "sizeBytes": int64(16004000000), + "mimeType": "video/mp4", + "source": bson.M{ + "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4", + "previewURL": "https://example.com/DJI_20260327132729_0003_D.MP4", + }, + }, + bson.M{ + "collectionId": "BJ_001", + "relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.json", + "sizeBytes": 4226, + "mimeType": "application/json", + }, + bson.M{ + "collectionId": "BJ_001", + "relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D_imu.json", + "sizeBytes": int64(36080000), + "mimeType": "application/json", + }, + } + run := domain.RunRecord{ + ID: "run-incremental-empty", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + { + ID: "run-incremental-empty-source", + RunID: "run-incremental-empty", + NodeID: "scan-source", + Name: "source-manifest.json", + Type: "json", + Data: map[string]any{ + "dataSource": "BOS", + "sourcePath": "bos://datastorage/algo/X-platform-test", + "scanMode": "INCREMENTAL", + "listedCount": 4, + "newCount": 0, + "catalogEntries": catalogEntries, + }, + }, + { + ID: "run-incremental-empty-index", + RunID: "run-incremental-empty", + NodeID: "scan-index", + Name: "dataset-index.json", + Type: "json", + Data: map[string]any{ + "dataSource": "BOS", + "sourcePath": "bos://datastorage/algo/X-platform-test", + "scanMode": "INCREMENTAL", + "listedCount": 4, + "newCount": 0, + "collectionCount": 0, + "collections": bson.A{}, + "items": bson.A{}, + "entries": bson.A{}, + "catalogCollectionCount": 1, + "catalogVideoCount": 1, + "catalogCollections": bson.A{collection}, + "catalogItems": bson.A{}, + "catalogEntries": catalogEntries, + }, + }, + }, + CreatedAt: "2026-04-10T12:00:00Z", + UpdatedAt: "2026-04-10T12:00:00Z", + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + handler := NewHandler(dataStore, nil) + collectionsRecorder := httptest.NewRecorder() + handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil)) + if collectionsRecorder.Code != http.StatusOK { + t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String()) + } + var collectionsBody struct { + Items []domain.SourceCollectionRecord `json:"items"` + } + if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil { + t.Fatalf("decode source collections: %v", err) + } + if len(collectionsBody.Items) != 1 { + t.Fatalf("expected one catalog collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String()) + } + if collectionsBody.Items[0].FileCount != 4 || collectionsBody.Items[0].TotalBytes != 16000036740 { + t.Fatalf("expected catalog file stats, got %+v", collectionsBody.Items[0]) + } + + entriesRecorder := httptest.NewRecorder() + handler.ServeHTTP(entriesRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-incremental-empty/BJ_001/entries?prefix=DJI_001", nil)) + if entriesRecorder.Code != http.StatusOK { + t.Fatalf("entries status: %d body=%s", entriesRecorder.Code, entriesRecorder.Body.String()) + } + var detail domain.SourceCollectionDetail + if err := json.Unmarshal(entriesRecorder.Body.Bytes(), &detail); err != nil { + t.Fatalf("decode source detail: %v", err) + } + if len(detail.Entries) != 3 { + t.Fatalf("expected all catalog files under DJI_001, got %d body=%s", len(detail.Entries), entriesRecorder.Body.String()) + } + var video *domain.SourceCollectionEntry + for index := range detail.Entries { + if detail.Entries[index].Name == "DJI_20260327132729_0003_D.MP4" { + video = &detail.Entries[index] + break + } + } + if video == nil || video.SizeBytes != 16004000000 { + t.Fatalf("expected real mp4 size from catalog entries, got %+v", video) + } + if collectionsBody.Items[0].Storage.Provider != "bos" { + t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage) + } + if collectionsBody.Items[0].Storage.Bucket != "datastorage" { + t.Fatalf("unexpected source storage bucket: %+v", collectionsBody.Items[0].Storage) + } + if collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" { + t.Fatalf("unexpected source storage prefix: %+v", collectionsBody.Items[0].Storage) + } +} + +func TestSourceCollectionsCanLoadExternalizedDatasetIndexPayload(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "BOS 增量扫描", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + indexPayload := map[string]any{ + "datasetType": "x-platform-index", + "dataSource": "BOS", + "sourcePath": "bos://datastorage/algo/X-platform-test", + "collectionCount": 0, + "collections": []any{}, + "items": []any{}, + "entries": []any{}, + "catalogCollectionCount": 1, + "catalogCollections": []any{ + map[string]any{ + "id": "BJ_001", + "name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43", + "relativePath": "BJ_001", + "videoCount": 1, + "valid": true, + }, + }, + "catalogItems": []any{}, + "catalogEntries": []any{}, + } + + run := domain.RunRecord{ + ID: "run-externalized-index", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + externalizedJSONArtifact(t, "run-externalized-index", "scan-index", "dataset-index.json", indexPayload), + }, + CreatedAt: "2026-04-10T12:00:00Z", + UpdatedAt: "2026-04-10T12:00:00Z", + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + handler := NewHandler(dataStore, nil) + collectionsRecorder := httptest.NewRecorder() + handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil)) + if collectionsRecorder.Code != http.StatusOK { + t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String()) + } + var collectionsBody struct { + Items []domain.SourceCollectionRecord `json:"items"` + } + if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil { + t.Fatalf("decode source collections: %v", err) + } + if len(collectionsBody.Items) != 1 { + t.Fatalf("expected externalized payload to produce one collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String()) + } +} + +func TestPackageContentCanServeBrowserPreviewVideo(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品预览项目"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "成品预览工作流", + Type: domain.WorkflowTypeBatch, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + videoPath := filepath.Join(t.TempDir(), "DJI_0001_D_clip001.mp4") + if err := os.WriteFile(videoPath, []byte("package-video-bytes"), 0o644); err != nil { + t.Fatalf("write package video: %v", err) + } + + run := domain.RunRecord{ + ID: "run-package-preview", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + { + ID: "run-package-preview-manifest", + RunID: "run-package-preview", + NodeID: "package", + Name: "packages-manifest.json", + Type: "json", + Data: map[string]any{ + "items": bson.A{ + bson.M{ + "packageName": "DJI_0001_D_clip001_package", + "clipId": "DJI_0001_D_clip001", + "sourceVideo": "DJI_0001_D.mp4", + "fileCount": 1, + "storagePath": filepath.Dir(videoPath), + }, + }, + }, + }, + { + ID: "run-package-preview-video", + RunID: "run-package-preview", + NodeID: "package", + Name: "DJI_0001_D_clip001.mp4", + Type: "video", + Path: filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4"), + Metadata: &domain.ArtifactMetadata{ + MimeType: "video/mp4", + }, + Data: map[string]any{ + "preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"}, + }, + }, + { + ID: "run-package-preview-upload-video", + RunID: "run-package-preview", + NodeID: "upload", + Name: "DJI_0001_D_clip001.mp4", + Type: "video", + Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + Metadata: &domain.ArtifactMetadata{ + MimeType: "video/mp4", + }, + Data: map[string]any{ + "source": map[string]any{ + "path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + }, + }, + }, + }, + } + run.Artifacts[1].Path = filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4") + if err := os.MkdirAll(filepath.Dir(run.Artifacts[1].Path), 0o755); err != nil { + t.Fatalf("mkdir package dir: %v", err) + } + if err := os.WriteFile(run.Artifacts[1].Path, []byte("package-video-bytes"), 0o644); err != nil { + t.Fatalf("write package artifact: %v", err) + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + previewPath := filepath.Join(t.TempDir(), "package-browser-preview.mp4") + if err := os.WriteFile(previewPath, []byte("package-browser-preview"), 0o644); err != nil { + t.Fatalf("write preview file: %v", err) + } + + originalPreviewResolver := sourceBrowserVideoPreviewPath + sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { + if request.RedirectURL != run.Artifacts[1].Path { + t.Fatalf("unexpected package preview input: %s", request.RedirectURL) + } + if request.FileName != "DJI_0001_D_clip001.mp4" { + t.Fatalf("unexpected file name: %s", request.FileName) + } + return previewPath, nil + } + t.Cleanup(func() { + sourceBrowserVideoPreviewPath = originalPreviewResolver + }) + + handler := NewHandler(dataStore, nil) + + packagesRecorder := httptest.NewRecorder() + handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil)) + if packagesRecorder.Code != http.StatusOK { + t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String()) + } + var packagesBody struct { + Items []domain.PackageRecord `json:"items"` + } + if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil { + t.Fatalf("decode packages: %v", err) + } + if len(packagesBody.Items) != 1 { + t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String()) + } + if packagesBody.Items[0].Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" { + t.Fatalf("expected package storage to prefer BOS upload path, got %+v", packagesBody.Items[0].Storage) + } + + packageDetailRecorder := httptest.NewRecorder() + handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-package-preview/DJI_0001_D_clip001_package", nil)) + if packageDetailRecorder.Code != http.StatusOK { + t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String()) + } + var packageDetail domain.PackageDetail + if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil { + t.Fatalf("decode package detail: %v", err) + } + if packageDetail.Package.Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" { + t.Fatalf("expected package detail storage to prefer BOS upload path, got %+v", packageDetail.Package.Storage) + } + if len(packageDetail.Files) != 1 { + t.Fatalf("expected duplicate package files to collapse to one, got %d", len(packageDetail.Files)) + } + if packageDetail.Files[0].Name != "DJI_0001_D_clip001.mp4" { + t.Fatalf("unexpected package file after dedupe: %+v", packageDetail.Files[0]) + } + + recorder := httptest.NewRecorder() + request := httptest.NewRequest( + http.MethodGet, + "/api/packages/run-package-preview/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser", + nil, + ) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String()) + } + if recorder.Header().Get("Content-Type") != "video/mp4" { + t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type")) + } + if string(recorder.Body.Bytes()) != "package-browser-preview" { + t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes()) + } +} + +func TestPackageContentFallsBackToUploadedBOSVideoWhenLocalPackageFileUnavailable(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品上传回退项目"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "上传 BOS 连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + AccessKeyID: "ak-upload", + SecretAccessKey: "sk-upload", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "成品上传回退工作流", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "upload", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "结果上传", + ConnectionID: connection.ID, + Bucket: "datastorage", + Prefix: "factory/output/test-package-preview", + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "BOS"}, + {Key: "SOURCE_PATH", Value: "bos://datastorage/factory/output/test-package-preview"}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run := domain.RunRecord{ + ID: "run-package-upload-fallback", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + { + ID: "run-package-upload-fallback-manifest", + RunID: "run-package-upload-fallback", + NodeID: "package", + Name: "packages-manifest.json", + Type: "json", + Data: map[string]any{ + "items": bson.A{ + bson.M{ + "packageName": "DJI_0001_D_clip001_package", + "clipId": "DJI_0001_D_clip001", + "sourceVideo": "DJI_0001_D.mp4", + "fileCount": 1, + "storagePath": "/tmp/droboticflow/missing/package", + }, + }, + }, + }, + { + ID: "run-package-upload-fallback-temp-video", + RunID: "run-package-upload-fallback", + NodeID: "package", + Name: "DJI_0001_D_clip001.mp4", + Type: "video", + Path: "/tmp/droboticflow/missing/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + Metadata: &domain.ArtifactMetadata{ + MimeType: "video/mp4", + }, + Data: map[string]any{ + "preview": map[string]any{ + "kind": "video", + "url": "/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4", + }, + }, + }, + { + ID: "run-package-upload-fallback-remote-video", + RunID: "run-package-upload-fallback", + NodeID: "upload", + Name: "DJI_0001_D_clip001.mp4", + Type: "video", + Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + Metadata: &domain.ArtifactMetadata{ + MimeType: "video/mp4", + }, + Data: map[string]any{ + "relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + "source": map[string]any{ + "path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + "previewURL": "https://expired.example.com/DJI_0001_D_clip001.mp4", + "mimeType": "video/mp4", + }, + }, + }, + }, + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + previewPath := filepath.Join(t.TempDir(), "package-upload-browser-preview.mp4") + if err := os.WriteFile(previewPath, []byte("package-upload-browser-preview"), 0o644); err != nil { + t.Fatalf("write preview file: %v", err) + } + + originalSignedPreviewResolver := sourceCollectionSignedPreviewURL + sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { + if scanConfig == nil || scanConfig.AccessKeyID != "ak-upload" || scanConfig.SecretAccessKey != "sk-upload" { + t.Fatalf("expected upload connection secrets, got %+v", scanConfig) + } + if sourcePath != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4" { + t.Fatalf("unexpected source path: %s", sourcePath) + } + return "https://signed.example.com/DJI_0001_D_clip001.mp4", nil + } + t.Cleanup(func() { + sourceCollectionSignedPreviewURL = originalSignedPreviewResolver + }) + + originalPreviewResolver := sourceBrowserVideoPreviewPath + sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { + if request.RedirectURL != "https://signed.example.com/DJI_0001_D_clip001.mp4" { + t.Fatalf("unexpected package preview redirect input: %s", request.RedirectURL) + } + if request.FileName != "DJI_0001_D_clip001.mp4" { + t.Fatalf("unexpected file name: %s", request.FileName) + } + return previewPath, nil + } + t.Cleanup(func() { + sourceBrowserVideoPreviewPath = originalPreviewResolver + }) + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + request := httptest.NewRequest( + http.MethodGet, + "/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser", + nil, + ) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String()) + } + if recorder.Header().Get("Content-Type") != "video/mp4" { + t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type")) + } + if string(recorder.Body.Bytes()) != "package-upload-browser-preview" { + t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes()) + } +} + +func TestPackagesListKeepsLatestPackageVersionPerWorkflow(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "包去重项目"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "包去重工作流", + Type: domain.WorkflowTypeBatch, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + makeRun := func(id string, createdAt string, storagePath string) domain.RunRecord { + return domain.RunRecord{ + ID: id, + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + CreatedAt: createdAt, + Artifacts: []domain.ExecutionArtifact{ + { + ID: id + "-manifest", + RunID: id, + NodeID: "package", + Name: "packages-manifest.json", + Type: "json", + Data: map[string]any{ + "items": []any{ + map[string]any{ + "packageName": "DJI_0001_D_clip001_package", + "clipId": "DJI_0001_D_clip001", + "sourceVideo": "DJI_0001_D.mp4", + "fileCount": 1, + "storagePath": "/tmp/k8s-job/package", + }, + }, + }, + }, + { + ID: id + "-upload-video", + RunID: id, + NodeID: "upload", + Name: "DJI_0001_D_clip001.mp4", + Type: "video", + Path: storagePath + "/DJI_0001_D_clip001.mp4", + Metadata: &domain.ArtifactMetadata{ + MimeType: "video/mp4", + }, + Data: map[string]any{ + "relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + "source": map[string]any{ + "path": storagePath + "/DJI_0001_D_clip001.mp4", + "mimeType": "video/mp4", + "relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", + }, + }, + }, + }, + } + } + + if err := dataStore.UpsertRun(ctx, makeRun( + "run-package-old", + "2026-04-12T10:00:00Z", + "bos://datastorage/factory/output/old/packages/DJI_0001_D_clip001_package", + )); err != nil { + t.Fatalf("upsert old run: %v", err) + } + if err := dataStore.UpsertRun(ctx, makeRun( + "run-package-new", + "2026-04-12T11:00:00Z", + "bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package", + )); err != nil { + t.Fatalf("upsert new run: %v", err) + } + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("packages status: %d body=%s", recorder.Code, recorder.Body.String()) + } + var body struct { + Items []domain.PackageRecord `json:"items"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("decode packages: %v", err) + } + if len(body.Items) != 1 { + t.Fatalf("expected latest package only, got %d body=%s", len(body.Items), recorder.Body.String()) + } + if body.Items[0].RunID != "run-package-new" { + t.Fatalf("expected latest run package, got %+v", body.Items[0]) + } + if body.Items[0].Storage.Path != "bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package" { + t.Fatalf("expected latest BOS storage path, got %+v", body.Items[0].Storage) + } +} + +func TestSourceCollectionContentCanServeBrowserPreviewVideo(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "浏览器预览项目"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "BOS 增量扫描", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + ScanConfig: &domain.ScanConfig{ + DataSource: "BOS", + Endpoint: "https://bj.bcebos.com", + AccessKeyID: "ak", + SecretAccessKey: "sk", + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run := domain.RunRecord{ + ID: "run-browser-preview", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + { + ID: "run-browser-preview-index", + RunID: "run-browser-preview", + NodeID: "scan-index", + Name: "dataset-index.json", + Type: "json", + Data: map[string]any{ + "catalogCollections": bson.A{ + bson.M{ + "id": "BJ_001", + "name": "BJ_001", + "path": "bos://datastorage/algo/X-platform-test/BJ_001", + "relativePath": "BJ_001", + "videoCount": 1, + "fileCount": 1, + "totalBytes": int64(2048), + "valid": true, + }, + }, + "catalogEntries": bson.A{ + bson.M{ + "collectionId": "BJ_001", + "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", + "sizeBytes": int64(2048), + "mimeType": "video/mp4", + "source": bson.M{ + "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4", + "previewURL": "https://example.com/DJI_0001_D.mp4", + }, + }, + }, + }, + }, + }, + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + previewPath := filepath.Join(t.TempDir(), "preview.mp4") + if err := os.WriteFile(previewPath, []byte("browser-preview-video"), 0o644); err != nil { + t.Fatalf("write preview file: %v", err) + } + + originalPreviewResolver := sourceBrowserVideoPreviewPath + sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { + if request.RedirectURL != "https://signed.example.com/DJI_0001_D.mp4" { + t.Fatalf("unexpected redirect url: %s", request.RedirectURL) + } + if request.FileName != "DJI_0001_D.mp4" { + t.Fatalf("unexpected file name: %s", request.FileName) + } + return previewPath, nil + } + t.Cleanup(func() { + sourceBrowserVideoPreviewPath = originalPreviewResolver + }) + originalSignedPreviewResolver := sourceCollectionSignedPreviewURL + sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { + if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" { + t.Fatalf("unexpected scan config: %+v", scanConfig) + } + if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" { + t.Fatalf("unexpected source path: %s", sourcePath) + } + return "https://signed.example.com/DJI_0001_D.mp4", nil + } + t.Cleanup(func() { + sourceCollectionSignedPreviewURL = originalSignedPreviewResolver + }) + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + request := httptest.NewRequest( + http.MethodGet, + "/api/source-data/collections/run-browser-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4&transcode=browser", + nil, + ) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String()) + } + if recorder.Header().Get("Content-Type") != "video/mp4" { + t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type")) + } + if string(recorder.Body.Bytes()) != "browser-preview-video" { + t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes()) + } +} + +func TestSourceCollectionContentRefreshesBOSRedirectURL(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 刷新项目"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "BOS 增量扫描", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + ScanConfig: &domain.ScanConfig{ + DataSource: "BOS", + Endpoint: "https://bj.bcebos.com", + AccessKeyID: "ak", + SecretAccessKey: "sk", + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run := domain.RunRecord{ + ID: "run-refresh-bos-preview", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + { + ID: "run-refresh-bos-preview-index", + RunID: "run-refresh-bos-preview", + NodeID: "scan-index", + Name: "dataset-index.json", + Type: "json", + Data: map[string]any{ + "catalogCollections": bson.A{ + bson.M{ + "id": "BJ_001", + "name": "BJ_001", + "path": "bos://datastorage/algo/X-platform-test/BJ_001", + "relativePath": "BJ_001", + "videoCount": 1, + "fileCount": 1, + "totalBytes": int64(2048), + "valid": true, + }, + }, + "catalogEntries": bson.A{ + bson.M{ + "collectionId": "BJ_001", + "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", + "sizeBytes": int64(2048), + "mimeType": "video/mp4", + "source": bson.M{ + "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4", + "previewURL": "https://expired.example.com/DJI_0001_D.mp4", + }, + }, + }, + }, + }, + }, + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + originalSignedPreviewResolver := sourceCollectionSignedPreviewURL + sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { + if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" { + t.Fatalf("unexpected scan config: %+v", scanConfig) + } + if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" { + t.Fatalf("unexpected source path: %s", sourcePath) + } + return "https://signed.example.com/DJI_0001_D.mp4", nil + } + t.Cleanup(func() { + sourceCollectionSignedPreviewURL = originalSignedPreviewResolver + }) + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + request := httptest.NewRequest( + http.MethodGet, + "/api/source-data/collections/run-refresh-bos-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", + nil, + ) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusTemporaryRedirect { + t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String()) + } + if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" { + t.Fatalf("unexpected redirect location: %s", location) + } +} + +func TestSourceCollectionContentRefreshesBOSRedirectURLFromStorageConnection(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 连接预览项目"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 主连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + AccessKeyID: "ak-from-connection", + SecretAccessKey: "sk-from-connection", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "BOS 增量扫描", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + ScanConfig: &domain.ScanConfig{ + DataSource: "BOS", + ConnectionID: connection.ID, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run := domain.RunRecord{ + ID: "run-refresh-bos-preview-connection", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerManual, + Status: domain.RunStatusCompleted, + Artifacts: []domain.ExecutionArtifact{ + { + ID: "run-refresh-bos-preview-connection-index", + RunID: "run-refresh-bos-preview-connection", + NodeID: "scan-index", + Name: "dataset-index.json", + Type: "json", + Data: map[string]any{ + "catalogCollections": bson.A{ + bson.M{ + "id": "BJ_001", + "name": "BJ_001", + "path": "bos://datastorage/algo/X-platform-test/BJ_001", + "relativePath": "BJ_001", + "videoCount": 1, + "fileCount": 1, + "totalBytes": int64(2048), + "valid": true, + }, + }, + "catalogEntries": bson.A{ + bson.M{ + "collectionId": "BJ_001", + "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", + "sizeBytes": int64(2048), + "mimeType": "video/mp4", + "source": bson.M{ + "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4", + "previewURL": "https://expired.example.com/DJI_0001_D.mp4", + }, + }, + }, + }, + }, + }, + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + originalSignedPreviewResolver := sourceCollectionSignedPreviewURL + sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { + if scanConfig == nil || scanConfig.AccessKeyID != "ak-from-connection" || scanConfig.SecretAccessKey != "sk-from-connection" { + t.Fatalf("expected resolved connection secrets, got %+v", scanConfig) + } + if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" { + t.Fatalf("unexpected source path: %s", sourcePath) + } + return "https://signed.example.com/DJI_0001_D.mp4", nil + } + t.Cleanup(func() { + sourceCollectionSignedPreviewURL = originalSignedPreviewResolver + }) + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + request := httptest.NewRequest( + http.MethodGet, + "/api/source-data/collections/run-refresh-bos-preview-connection/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", + nil, + ) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusTemporaryRedirect { + t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String()) + } + if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" { + t.Fatalf("unexpected redirect location: %s", location) + } +} diff --git a/internal/api/database_browser.go b/internal/api/database_browser.go new file mode 100644 index 00000000..d29dd3dd --- /dev/null +++ b/internal/api/database_browser.go @@ -0,0 +1,576 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" + + "github.com/go-chi/chi/v5" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type databaseBrowserTarget string + +const ( + databaseBrowserTargetPlatform databaseBrowserTarget = "platform" + databaseBrowserTargetBusiness databaseBrowserTarget = "business" +) + +type databaseBrowserStore interface { + GetDatabaseOverview(context.Context) (domain.DatabaseOverview, error) + QueryDatabaseCollection(context.Context, domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) + CreateDatabaseCollection(context.Context, string) (domain.DatabaseCollectionSummary, error) + RenameDatabaseCollection(context.Context, string, string) (domain.DatabaseCollectionSummary, error) + DeleteDatabaseCollection(context.Context, string) error + ListDatabaseIndexes(context.Context, string) ([]domain.DatabaseIndexSummary, error) + CreateDatabaseIndex(context.Context, string, domain.DatabaseIndexCreateInput) (domain.DatabaseIndexSummary, error) + DeleteDatabaseIndex(context.Context, string, string) error + InsertDatabaseDocument(context.Context, string, map[string]any) (map[string]any, error) + ReplaceDatabaseDocument(context.Context, string, string, map[string]any) (map[string]any, error) + PatchDatabaseDocument(context.Context, string, string, map[string]any) (map[string]any, error) + UpsertDatabaseDocument(context.Context, string, map[string]any, map[string]any, map[string]any) (map[string]any, bool, error) + DeleteDatabaseDocument(context.Context, string, string) error +} + +var mongoConnectionStoreResolver = func(ctx context.Context, connection *domain.MongoConnectionRecord) (databaseBrowserStore, func(), error) { + if connection == nil { + return nil, nil, nil + } + client, err := mongo.Connect(ctx, options.Client().ApplyURI(strings.TrimSpace(connection.URI))) + if err != nil { + return nil, nil, err + } + cleanup := func() { + disconnectCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = client.Disconnect(disconnectCtx) + } + if err := client.Ping(ctx, nil); err != nil { + cleanup() + return nil, nil, err + } + return store.NewMongoStore(client.Database(strings.TrimSpace(connection.Database))), cleanup, nil +} + +func parseDatabaseQueryFilter(raw string) (map[string]any, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return map[string]any{}, nil + } + filter := map[string]any{} + if err := json.Unmarshal([]byte(trimmed), &filter); err != nil { + return nil, err + } + return filter, nil +} + +func parsePositiveIntQuery(value string, fallback int, max int) int { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return fallback + } + parsed, err := strconv.Atoi(trimmed) + if err != nil || parsed < 0 { + return fallback + } + if max > 0 && parsed > max { + return max + } + return parsed +} + +func normalizeDatabaseBrowserTarget(raw string) databaseBrowserTarget { + switch strings.ToLower(strings.TrimSpace(raw)) { + case string(databaseBrowserTargetPlatform): + return databaseBrowserTargetPlatform + case string(databaseBrowserTargetBusiness): + return databaseBrowserTargetBusiness + default: + return databaseBrowserTargetBusiness + } +} + +func mongoConnectionConfigured(connection *domain.MongoConnectionRecord) bool { + if connection == nil { + return false + } + return strings.TrimSpace(connection.URI) != "" && strings.TrimSpace(connection.Database) != "" +} + +func (server *server) databaseBrowserOverview(ctx context.Context, target databaseBrowserTarget, connectionID string) (domain.DatabaseOverview, error) { + item, err := server.store.GetDatabaseOverview(ctx) + if target == databaseBrowserTargetPlatform { + if err != nil { + return domain.DatabaseOverview{}, err + } + item.Target = string(databaseBrowserTargetPlatform) + item.Configured = true + return item, nil + } + + connection, err := server.getMongoConnection(ctx, connectionID) + if err != nil && strings.TrimSpace(connectionID) != "" { + return domain.DatabaseOverview{}, err + } + overview := domain.DatabaseOverview{ + Target: string(databaseBrowserTargetBusiness), + ConnectionID: strings.TrimSpace(connectionID), + ConnectionName: "", + Provider: "mongodb", + Database: "", + ReadOnly: true, + Configured: mongoConnectionConfigured(connection), + Collections: []domain.DatabaseCollectionSummary{}, + } + if connection == nil || !overview.Configured { + return overview, nil + } + overview.ConnectionID = connection.ID + overview.ConnectionName = connection.Name + overview.Database = strings.TrimSpace(connection.Database) + browserStore, cleanup, err := mongoConnectionStoreResolver(ctx, connection) + if err != nil { + return domain.DatabaseOverview{}, err + } + defer cleanup() + item, err = browserStore.GetDatabaseOverview(ctx) + if err != nil { + return domain.DatabaseOverview{}, err + } + item.Target = string(databaseBrowserTargetBusiness) + item.ConnectionID = connection.ID + item.ConnectionName = connection.Name + item.Configured = true + item.ReadOnly = false + return item, nil +} + +func (server *server) databaseBrowserQuery(ctx context.Context, target databaseBrowserTarget, connectionID string, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) { + if target == databaseBrowserTargetPlatform { + item, err := server.store.QueryDatabaseCollection(ctx, input) + if err != nil { + return domain.DatabaseQueryResult{}, err + } + item.Target = string(databaseBrowserTargetPlatform) + item.Configured = true + return item, nil + } + + connection, err := server.getMongoConnection(ctx, connectionID) + if err != nil && strings.TrimSpace(connectionID) != "" { + return domain.DatabaseQueryResult{}, err + } + if connection == nil || !mongoConnectionConfigured(connection) { + return domain.DatabaseQueryResult{ + Target: string(databaseBrowserTargetBusiness), + ConnectionID: strings.TrimSpace(connectionID), + Provider: "mongodb", + Database: "", + Collection: input.Collection, + Limit: input.Limit, + Skip: input.Skip, + Configured: false, + Documents: []map[string]any{}, + }, nil + } + browserStore, cleanup, err := mongoConnectionStoreResolver(ctx, connection) + if err != nil { + return domain.DatabaseQueryResult{}, err + } + defer cleanup() + item, err := browserStore.QueryDatabaseCollection(ctx, input) + if err != nil { + return domain.DatabaseQueryResult{}, err + } + item.Target = string(databaseBrowserTargetBusiness) + item.ConnectionID = connection.ID + item.ConnectionName = connection.Name + item.Configured = true + return item, nil +} + +func (server *server) resolveBusinessDatabaseStore(ctx context.Context, connectionID string) (databaseBrowserStore, *domain.MongoConnectionRecord, func(), error) { + connection, err := server.getMongoConnection(ctx, connectionID) + if err != nil && strings.TrimSpace(connectionID) != "" { + return nil, nil, nil, err + } + if connection == nil || !mongoConnectionConfigured(connection) { + return nil, connection, nil, fmt.Errorf("mongo connection is not configured") + } + browserStore, cleanup, err := mongoConnectionStoreResolver(ctx, connection) + if err != nil { + return nil, connection, nil, err + } + return browserStore, connection, cleanup, nil +} + +func decodeDatabaseCollectionCreateInput(request *http.Request) (domain.DatabaseCollectionCreateInput, error) { + input := domain.DatabaseCollectionCreateInput{} + if err := json.NewDecoder(request.Body).Decode(&input); err != nil { + return domain.DatabaseCollectionCreateInput{}, err + } + return input, nil +} + +func decodeDatabaseCollectionRenameInput(request *http.Request) (domain.DatabaseCollectionRenameInput, error) { + input := domain.DatabaseCollectionRenameInput{} + if err := json.NewDecoder(request.Body).Decode(&input); err != nil { + return domain.DatabaseCollectionRenameInput{}, err + } + return input, nil +} + +func decodeDatabaseDocumentMutationInput(request *http.Request) (domain.DatabaseDocumentMutationInput, error) { + input := domain.DatabaseDocumentMutationInput{} + if err := json.NewDecoder(request.Body).Decode(&input); err != nil { + return domain.DatabaseDocumentMutationInput{}, err + } + if input.Document == nil { + input.Document = map[string]any{} + } + return input, nil +} + +func decodeMongoConnectionTestInput(request *http.Request) (domain.MongoConnectionTestInput, error) { + input := domain.MongoConnectionTestInput{} + if err := json.NewDecoder(request.Body).Decode(&input); err != nil { + return domain.MongoConnectionTestInput{}, err + } + return input, nil +} + +func databaseBrowserNotFound(err error) bool { + return strings.Contains(strings.ToLower(err.Error()), "not found") +} + +func renderDatabaseBrowserWriteError(writer http.ResponseWriter, err error) { + if databaseBrowserNotFound(err) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()}) + return + } + renderError(writer, http.StatusInternalServerError, err) +} + +func (server *server) handleMongoConnectionsTest(writer http.ResponseWriter, request *http.Request) { + input, err := decodeMongoConnectionTestInput(request) + if err != nil { + renderValidationError(writer, validationIssue{Field: "body", Message: "request body must be valid JSON"}) + return + } + + var connection *domain.MongoConnectionRecord + connectionID := strings.TrimSpace(input.ID) + if connectionID != "" { + connection, err = server.getMongoConnection(request.Context(), connectionID) + if err != nil { + renderValidationError(writer, validationIssue{Field: "id", Message: err.Error()}) + return + } + } else { + uri := strings.TrimSpace(input.URI) + database := strings.TrimSpace(input.Database) + if uri == "" { + renderValidationError(writer, validationIssue{Field: "uri", Message: "uri is required"}) + return + } + if database == "" { + renderValidationError(writer, validationIssue{Field: "database", Message: "database is required"}) + return + } + connection = &domain.MongoConnectionRecord{ + ID: "", + Name: "probe", + URI: uri, + Database: database, + } + } + + browserStore, cleanup, err := mongoConnectionStoreResolver(request.Context(), connection) + if err != nil { + renderError(writer, http.StatusBadGateway, err) + return + } + defer cleanup() + item, err := browserStore.GetDatabaseOverview(request.Context()) + if err != nil { + renderError(writer, http.StatusBadGateway, err) + return + } + item.Target = string(databaseBrowserTargetBusiness) + item.Configured = true + item.ReadOnly = false + if connection != nil { + item.ConnectionID = connection.ID + item.ConnectionName = connection.Name + item.Database = strings.TrimSpace(connection.Database) + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleDatabaseOverview(writer http.ResponseWriter, request *http.Request) { + item, err := server.databaseBrowserOverview( + request.Context(), + normalizeDatabaseBrowserTarget(request.URL.Query().Get("target")), + request.URL.Query().Get("connectionId"), + ) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleDatabaseCollectionCreate(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + input, err := decodeDatabaseCollectionCreateInput(request) + if err != nil { + renderValidationError(writer, validationIssue{Field: "body", Message: "request body must be valid JSON"}) + return + } + name := strings.TrimSpace(input.Name) + if name == "" { + renderValidationError(writer, validationIssue{Field: "name", Message: "collection name is required"}) + return + } + summary, err := browserStore.CreateDatabaseCollection(request.Context(), name) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusCreated, domain.DatabaseCollectionMutationResult{ + Collection: summary.Name, + Summary: summary, + }) +} + +func (server *server) handleDatabaseCollectionRename(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + input, err := decodeDatabaseCollectionRenameInput(request) + if err != nil { + renderValidationError(writer, validationIssue{Field: "body", Message: "request body must be valid JSON"}) + return + } + newName := strings.TrimSpace(input.NewName) + if newName == "" { + renderValidationError(writer, validationIssue{Field: "newName", Message: "new collection name is required"}) + return + } + summary, err := browserStore.RenameDatabaseCollection(request.Context(), chi.URLParam(request, "name"), newName) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.DatabaseCollectionMutationResult{ + Collection: summary.Name, + Summary: summary, + }) +} + +func (server *server) handleDatabaseCollectionDelete(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + name := chi.URLParam(request, "name") + if err := browserStore.DeleteDatabaseCollection(request.Context(), name); err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.DatabaseCollectionDeleteResult{ + Collection: name, + Deleted: true, + }) +} + +func (server *server) handleDatabaseCollectionIndexes(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + indexes, err := browserStore.ListDatabaseIndexes(request.Context(), chi.URLParam(request, "name")) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.DatabaseIndexesResult{ + Collection: chi.URLParam(request, "name"), + Indexes: indexes, + }) +} + +func (server *server) handleDatabaseCollectionIndexCreate(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + input := domain.DatabaseIndexCreateInput{} + if !decodeJSON(writer, request, &input) { + return + } + if len(input.Keys) == 0 { + renderValidationError(writer, validationIssue{Field: "keys", Message: "index keys are required"}) + return + } + indexSummary, err := browserStore.CreateDatabaseIndex(request.Context(), chi.URLParam(request, "name"), input) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusCreated, domain.DatabaseIndexMutationResult{ + Collection: chi.URLParam(request, "name"), + Index: indexSummary, + }) +} + +func (server *server) handleDatabaseCollectionIndexDelete(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + indexName := chi.URLParam(request, "indexName") + if err := browserStore.DeleteDatabaseIndex(request.Context(), chi.URLParam(request, "name"), indexName); err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.DatabaseIndexDeleteResult{ + Collection: chi.URLParam(request, "name"), + IndexName: indexName, + Deleted: true, + }) +} + +func (server *server) handleDatabaseCollectionDocuments(writer http.ResponseWriter, request *http.Request) { + target := normalizeDatabaseBrowserTarget(request.URL.Query().Get("target")) + filter, err := parseDatabaseQueryFilter(request.URL.Query().Get("filter")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "filter", Message: "filter must be valid JSON object"}) + return + } + input := domain.DatabaseQueryInput{ + Collection: chi.URLParam(request, "name"), + Filter: filter, + Limit: parsePositiveIntQuery(request.URL.Query().Get("limit"), 50, 200), + Skip: parsePositiveIntQuery(request.URL.Query().Get("skip"), 0, 5000), + } + item, err := server.databaseBrowserQuery(request.Context(), target, request.URL.Query().Get("connectionId"), input) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "not found") { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()}) + return + } + renderError(writer, http.StatusInternalServerError, err) + return + } + if target == databaseBrowserTargetBusiness && !item.Configured { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: "mongo connection is not configured"}) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleDatabaseDocumentCreate(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + input, err := decodeDatabaseDocumentMutationInput(request) + if err != nil { + renderValidationError(writer, validationIssue{Field: "body", Message: "request body must be valid JSON"}) + return + } + if len(input.Document) == 0 { + renderValidationError(writer, validationIssue{Field: "document", Message: "document is required"}) + return + } + document, err := browserStore.InsertDatabaseDocument(request.Context(), chi.URLParam(request, "name"), input.Document) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusCreated, domain.DatabaseDocumentMutationResult{ + Collection: chi.URLParam(request, "name"), + Document: document, + }) +} + +func (server *server) handleDatabaseDocumentReplace(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + input, err := decodeDatabaseDocumentMutationInput(request) + if err != nil { + renderValidationError(writer, validationIssue{Field: "body", Message: "request body must be valid JSON"}) + return + } + if len(input.Document) == 0 { + renderValidationError(writer, validationIssue{Field: "document", Message: "document is required"}) + return + } + document, err := browserStore.ReplaceDatabaseDocument( + request.Context(), + chi.URLParam(request, "name"), + chi.URLParam(request, "documentId"), + input.Document, + ) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.DatabaseDocumentMutationResult{ + Collection: chi.URLParam(request, "name"), + Document: document, + }) +} + +func (server *server) handleDatabaseDocumentDelete(writer http.ResponseWriter, request *http.Request) { + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), request.URL.Query().Get("connectionId")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + if err := browserStore.DeleteDatabaseDocument( + request.Context(), + chi.URLParam(request, "name"), + chi.URLParam(request, "documentId"), + ); err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.DatabaseDocumentDeleteResult{ + Collection: chi.URLParam(request, "name"), + DocumentID: chi.URLParam(request, "documentId"), + Deleted: true, + }) +} diff --git a/internal/api/database_external_write.go b/internal/api/database_external_write.go new file mode 100644 index 00000000..aef33e41 --- /dev/null +++ b/internal/api/database_external_write.go @@ -0,0 +1,631 @@ +package api + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "fmt" + "net/http" + "sort" + "strings" + + "droboticflow/internal/domain" + + "github.com/go-chi/chi/v5" +) + +type externalBusinessWriteTokenCreateRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Enabled *bool `json:"enabled"` + Scopes []domain.ExternalBusinessWriteScope `json:"scopes"` +} + +type externalBusinessWriteTokenUpdateRequest struct { + Name *string `json:"name"` + Description *string `json:"description"` + Enabled *bool `json:"enabled"` + Scopes *[]domain.ExternalBusinessWriteScope `json:"scopes"` +} + +type externalBusinessWriteCreateRequest struct { + ConnectionID string `json:"connectionId"` + Collection string `json:"collection"` + Document map[string]any `json:"document"` +} + +type externalBusinessWritePatchRequest struct { + ConnectionID string `json:"connectionId"` + Collection string `json:"collection"` + Patch map[string]any `json:"patch"` +} + +type externalBusinessWriteUpsertRequest struct { + ConnectionID string `json:"connectionId"` + Collection string `json:"collection"` + Match map[string]any `json:"match"` + Patch map[string]any `json:"patch"` + SetOnInsert map[string]any `json:"setOnInsert,omitempty"` +} + +type externalBusinessWriteTokenMutationResult struct { + Token domain.ExternalBusinessWriteTokenRecord `json:"token"` + APIKey string `json:"apiKey,omitempty"` +} + +type externalBusinessWriteTokenRevealResult struct { + TokenID string `json:"tokenId"` + APIKey string `json:"apiKey"` +} + +type externalBusinessWriteTokensResult struct { + Items []domain.ExternalBusinessWriteTokenRecord `json:"items"` +} + +type externalBusinessWriteDocumentResult struct { + ConnectionID string `json:"connectionId"` + Collection string `json:"collection"` + Operation string `json:"operation"` + Document map[string]any `json:"document"` +} + +type externalBusinessWriteTokenContextKey struct{} + +func externalBusinessWriteAPIKeyHash(value string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(value))) + return hex.EncodeToString(sum[:]) +} + +func generateExternalBusinessWriteAPIKey() (string, string, string, error) { + randomBytes := make([]byte, 24) + if _, err := rand.Read(randomBytes); err != nil { + return "", "", "", err + } + apiKey := "dbw_" + hex.EncodeToString(randomBytes) + return apiKey, externalBusinessWriteAPIKeyHash(apiKey), maskedSecretPreview(apiKey), nil +} + +func sanitizeExternalBusinessWriteToken(item domain.ExternalBusinessWriteTokenRecord) domain.ExternalBusinessWriteTokenRecord { + sanitized := item + sanitized.TokenValue = "" + sanitized.TokenHash = "" + if sanitized.Scopes == nil { + sanitized.Scopes = []domain.ExternalBusinessWriteScope{} + } + return sanitized +} + +func sanitizeExternalBusinessWriteTokens(items []domain.ExternalBusinessWriteTokenRecord) []domain.ExternalBusinessWriteTokenRecord { + sanitized := make([]domain.ExternalBusinessWriteTokenRecord, 0, len(items)) + for _, item := range items { + sanitized = append(sanitized, sanitizeExternalBusinessWriteToken(item)) + } + sort.Slice(sanitized, func(left int, right int) bool { + return sanitized[left].CreatedAt > sanitized[right].CreatedAt + }) + return sanitized +} + +func normalizeExternalBusinessWriteScopes(items []domain.ExternalBusinessWriteScope) []domain.ExternalBusinessWriteScope { + if len(items) == 0 { + return []domain.ExternalBusinessWriteScope{} + } + normalized := make([]domain.ExternalBusinessWriteScope, 0, len(items)) + seenConnections := map[string]int{} + for _, item := range items { + connectionID := strings.TrimSpace(item.ConnectionID) + if connectionID == "" { + continue + } + collections := make([]string, 0, len(item.Collections)) + seenCollections := map[string]struct{}{} + for _, collection := range item.Collections { + trimmed := strings.TrimSpace(collection) + if trimmed == "" { + continue + } + if _, exists := seenCollections[trimmed]; exists { + continue + } + seenCollections[trimmed] = struct{}{} + collections = append(collections, trimmed) + } + sort.Strings(collections) + if len(collections) == 0 { + continue + } + if index, exists := seenConnections[connectionID]; exists { + existingCollections := normalized[index].Collections + for _, collection := range collections { + if slicesContains(existingCollections, collection) { + continue + } + existingCollections = append(existingCollections, collection) + } + sort.Strings(existingCollections) + normalized[index].Collections = existingCollections + continue + } + seenConnections[connectionID] = len(normalized) + normalized = append(normalized, domain.ExternalBusinessWriteScope{ + ConnectionID: connectionID, + Collections: collections, + }) + } + return normalized +} + +func slicesContains(items []string, target string) bool { + for _, item := range items { + if item == target { + return true + } + } + return false +} + +func (server *server) validateExternalBusinessWriteScopes(ctx context.Context, scopes []domain.ExternalBusinessWriteScope) []validationIssue { + issues := make([]validationIssue, 0) + normalized := normalizeExternalBusinessWriteScopes(scopes) + if len(normalized) == 0 { + return []validationIssue{{Field: "scopes", Message: "at least one connectionId and collection whitelist is required"}} + } + for index, scope := range normalized { + if _, err := server.getMongoConnection(ctx, scope.ConnectionID); err != nil { + issues = append(issues, validationIssue{ + Field: fmt.Sprintf("scopes[%d].connectionId", index), + Message: "connectionId must reference an existing mongo connection", + }) + } + if len(scope.Collections) == 0 { + issues = append(issues, validationIssue{ + Field: fmt.Sprintf("scopes[%d].collections", index), + Message: "at least one collection is required", + }) + } + } + return issues +} + +func validateExternalBusinessWriteObject(field string, value map[string]any, requireNonEmpty bool) []validationIssue { + if value == nil { + if requireNonEmpty { + return []validationIssue{{Field: field, Message: field + " is required"}} + } + return nil + } + if requireNonEmpty && len(value) == 0 { + return []validationIssue{{Field: field, Message: field + " is required"}} + } + issues := make([]validationIssue, 0) + for key := range value { + if strings.HasPrefix(strings.TrimSpace(key), "$") { + issues = append(issues, validationIssue{ + Field: field, + Message: field + " cannot contain Mongo operator keys", + }) + break + } + } + return issues +} + +func externalBusinessWriteTokenEnabled(value *bool) bool { + if value == nil { + return true + } + return *value +} + +func externalBusinessWriteSettings(settings *domain.SettingsRecord) *domain.ExternalBusinessWriteSettings { + if settings == nil { + return &domain.ExternalBusinessWriteSettings{Tokens: []domain.ExternalBusinessWriteTokenRecord{}} + } + if settings.ExternalBusinessWrite.Tokens == nil { + settings.ExternalBusinessWrite.Tokens = []domain.ExternalBusinessWriteTokenRecord{} + } + return &settings.ExternalBusinessWrite +} + +func findExternalBusinessWriteTokenByID(settings *domain.SettingsRecord, tokenID string) (int, *domain.ExternalBusinessWriteTokenRecord) { + config := externalBusinessWriteSettings(settings) + for index := range config.Tokens { + if config.Tokens[index].ID == tokenID { + return index, &config.Tokens[index] + } + } + return -1, nil +} + +func findExternalBusinessWriteTokenByAPIKey(settings *domain.SettingsRecord, apiKey string) *domain.ExternalBusinessWriteTokenRecord { + config := externalBusinessWriteSettings(settings) + hash := externalBusinessWriteAPIKeyHash(apiKey) + for index := range config.Tokens { + item := &config.Tokens[index] + if !item.Enabled || strings.TrimSpace(item.TokenHash) == "" { + continue + } + if subtle.ConstantTimeCompare([]byte(item.TokenHash), []byte(hash)) == 1 { + return item + } + } + return nil +} + +func currentExternalBusinessWriteToken(request *http.Request) *domain.ExternalBusinessWriteTokenRecord { + item, _ := request.Context().Value(externalBusinessWriteTokenContextKey{}).(*domain.ExternalBusinessWriteTokenRecord) + return item +} + +func externalBusinessWriteAllowed(token *domain.ExternalBusinessWriteTokenRecord, connectionID string, collection string) bool { + if token == nil { + return false + } + targetConnectionID := strings.TrimSpace(connectionID) + targetCollection := strings.TrimSpace(collection) + for _, scope := range token.Scopes { + if strings.TrimSpace(scope.ConnectionID) != targetConnectionID { + continue + } + for _, allowedCollection := range scope.Collections { + if strings.TrimSpace(allowedCollection) == targetCollection { + return true + } + } + } + return false +} + +func (server *server) touchExternalBusinessWriteToken(ctx context.Context, tokenID string) { + settings, err := server.store.GetSettings(ctx) + if err != nil { + return + } + index, item := findExternalBusinessWriteTokenByID(settings, tokenID) + if item == nil { + return + } + now := nowISO() + settings.ExternalBusinessWrite.Tokens[index].LastUsedAt = now + settings.ExternalBusinessWrite.Tokens[index].UpdatedAt = now + _ = server.store.UpsertSettings(ctx, *settings) +} + +func (server *server) externalBusinessWriteAuth(next http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + apiKey := strings.TrimSpace(request.Header.Get("X-API-Key")) + if apiKey == "" { + renderJSON(writer, http.StatusUnauthorized, map[string]string{"error": "X-API-Key header is required"}) + return + } + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + token := findExternalBusinessWriteTokenByAPIKey(settings, apiKey) + if token == nil { + renderJSON(writer, http.StatusUnauthorized, map[string]string{"error": "invalid API key"}) + return + } + tokenCopy := *token + ctx := context.WithValue(request.Context(), externalBusinessWriteTokenContextKey{}, &tokenCopy) + next.ServeHTTP(writer, request.WithContext(ctx)) + }) +} + +func (server *server) handleExternalBusinessWriteTokensList(writer http.ResponseWriter, request *http.Request) { + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, externalBusinessWriteTokensResult{ + Items: sanitizeExternalBusinessWriteTokens(settings.ExternalBusinessWrite.Tokens), + }) +} + +func (server *server) handleExternalBusinessWriteTokensCreate(writer http.ResponseWriter, request *http.Request) { + var payload externalBusinessWriteTokenCreateRequest + if !decodeJSON(writer, request, &payload) { + return + } + normalizedScopes := normalizeExternalBusinessWriteScopes(payload.Scopes) + issues := make([]validationIssue, 0) + if strings.TrimSpace(payload.Name) == "" { + issues = append(issues, validationIssue{Field: "name", Message: "name is required"}) + } + issues = append(issues, server.validateExternalBusinessWriteScopes(request.Context(), normalizedScopes)...) + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + + apiKey, tokenHash, tokenPreview, err := generateExternalBusinessWriteAPIKey() + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + now := nowISO() + item := domain.ExternalBusinessWriteTokenRecord{ + ID: domain.NewID("exttoken"), + Name: strings.TrimSpace(payload.Name), + Description: strings.TrimSpace(payload.Description), + Enabled: externalBusinessWriteTokenEnabled(payload.Enabled), + TokenValue: apiKey, + TokenHash: tokenHash, + TokenPreview: tokenPreview, + Scopes: normalizedScopes, + CreatedAt: now, + UpdatedAt: now, + } + settings.ExternalBusinessWrite.Tokens = append(settings.ExternalBusinessWrite.Tokens, item) + if err := server.store.UpsertSettings(request.Context(), *settings); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, externalBusinessWriteTokenMutationResult{ + Token: sanitizeExternalBusinessWriteToken(item), + APIKey: apiKey, + }) +} + +func (server *server) handleExternalBusinessWriteTokensUpdate(writer http.ResponseWriter, request *http.Request) { + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + index, item := findExternalBusinessWriteTokenByID(settings, chi.URLParam(request, "id")) + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "External write token not found"}) + return + } + var payload externalBusinessWriteTokenUpdateRequest + if !decodeJSON(writer, request, &payload) { + return + } + updated := *item + if payload.Name != nil { + updated.Name = strings.TrimSpace(*payload.Name) + } + if payload.Description != nil { + updated.Description = strings.TrimSpace(*payload.Description) + } + if payload.Enabled != nil { + updated.Enabled = *payload.Enabled + } + if payload.Scopes != nil { + updated.Scopes = normalizeExternalBusinessWriteScopes(*payload.Scopes) + } + issues := make([]validationIssue, 0) + if strings.TrimSpace(updated.Name) == "" { + issues = append(issues, validationIssue{Field: "name", Message: "name is required"}) + } + issues = append(issues, server.validateExternalBusinessWriteScopes(request.Context(), updated.Scopes)...) + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + updated.UpdatedAt = nowISO() + settings.ExternalBusinessWrite.Tokens[index] = updated + if err := server.store.UpsertSettings(request.Context(), *settings); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, externalBusinessWriteTokenMutationResult{ + Token: sanitizeExternalBusinessWriteToken(updated), + }) +} + +func (server *server) handleExternalBusinessWriteTokensReveal(writer http.ResponseWriter, request *http.Request) { + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + _, item := findExternalBusinessWriteTokenByID(settings, chi.URLParam(request, "id")) + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "External write token not found"}) + return + } + apiKey := strings.TrimSpace(item.TokenValue) + if apiKey == "" { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "该 Token 创建于仅支持预览的旧版本,请轮换 Key 后再查看完整 Token"}) + return + } + renderJSON(writer, http.StatusOK, externalBusinessWriteTokenRevealResult{ + TokenID: item.ID, + APIKey: apiKey, + }) +} + +func (server *server) handleExternalBusinessWriteTokensRotate(writer http.ResponseWriter, request *http.Request) { + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + index, item := findExternalBusinessWriteTokenByID(settings, chi.URLParam(request, "id")) + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "External write token not found"}) + return + } + apiKey, tokenHash, tokenPreview, err := generateExternalBusinessWriteAPIKey() + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + updated := *item + updated.TokenValue = apiKey + updated.TokenHash = tokenHash + updated.TokenPreview = tokenPreview + updated.UpdatedAt = nowISO() + settings.ExternalBusinessWrite.Tokens[index] = updated + if err := server.store.UpsertSettings(request.Context(), *settings); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, externalBusinessWriteTokenMutationResult{ + Token: sanitizeExternalBusinessWriteToken(updated), + APIKey: apiKey, + }) +} + +func (server *server) handleExternalBusinessWriteTokensDelete(writer http.ResponseWriter, request *http.Request) { + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + index, item := findExternalBusinessWriteTokenByID(settings, chi.URLParam(request, "id")) + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "External write token not found"}) + return + } + settings.ExternalBusinessWrite.Tokens = append(settings.ExternalBusinessWrite.Tokens[:index], settings.ExternalBusinessWrite.Tokens[index+1:]...) + if err := server.store.UpsertSettings(request.Context(), *settings); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func validateExternalBusinessWriteTarget(connectionID string, collection string) []validationIssue { + issues := make([]validationIssue, 0, 2) + if strings.TrimSpace(connectionID) == "" { + issues = append(issues, validationIssue{Field: "connectionId", Message: "connectionId is required"}) + } + if strings.TrimSpace(collection) == "" { + issues = append(issues, validationIssue{Field: "collection", Message: "collection is required"}) + } + return issues +} + +func (server *server) renderExternalBusinessWriteForbidden(writer http.ResponseWriter) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "token is not allowed to write to the requested connection or collection"}) +} + +func (server *server) handleExternalBusinessWriteCreate(writer http.ResponseWriter, request *http.Request) { + var payload externalBusinessWriteCreateRequest + if !decodeJSON(writer, request, &payload) { + return + } + issues := validateExternalBusinessWriteTarget(payload.ConnectionID, payload.Collection) + issues = append(issues, validateExternalBusinessWriteObject("document", payload.Document, true)...) + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if !externalBusinessWriteAllowed(currentExternalBusinessWriteToken(request), payload.ConnectionID, payload.Collection) { + server.renderExternalBusinessWriteForbidden(writer) + return + } + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), payload.ConnectionID) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + document, err := browserStore.InsertDatabaseDocument(request.Context(), payload.Collection, payload.Document) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + if token := currentExternalBusinessWriteToken(request); token != nil { + server.touchExternalBusinessWriteToken(request.Context(), token.ID) + } + renderJSON(writer, http.StatusCreated, externalBusinessWriteDocumentResult{ + ConnectionID: strings.TrimSpace(payload.ConnectionID), + Collection: strings.TrimSpace(payload.Collection), + Operation: "created", + Document: document, + }) +} + +func (server *server) handleExternalBusinessWritePatch(writer http.ResponseWriter, request *http.Request) { + var payload externalBusinessWritePatchRequest + if !decodeJSON(writer, request, &payload) { + return + } + issues := validateExternalBusinessWriteTarget(payload.ConnectionID, payload.Collection) + issues = append(issues, validateExternalBusinessWriteObject("patch", payload.Patch, true)...) + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if !externalBusinessWriteAllowed(currentExternalBusinessWriteToken(request), payload.ConnectionID, payload.Collection) { + server.renderExternalBusinessWriteForbidden(writer) + return + } + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), payload.ConnectionID) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + document, err := browserStore.PatchDatabaseDocument(request.Context(), payload.Collection, chi.URLParam(request, "documentId"), payload.Patch) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + if token := currentExternalBusinessWriteToken(request); token != nil { + server.touchExternalBusinessWriteToken(request.Context(), token.ID) + } + renderJSON(writer, http.StatusOK, externalBusinessWriteDocumentResult{ + ConnectionID: strings.TrimSpace(payload.ConnectionID), + Collection: strings.TrimSpace(payload.Collection), + Operation: "updated", + Document: document, + }) +} + +func (server *server) handleExternalBusinessWriteUpsert(writer http.ResponseWriter, request *http.Request) { + var payload externalBusinessWriteUpsertRequest + if !decodeJSON(writer, request, &payload) { + return + } + issues := validateExternalBusinessWriteTarget(payload.ConnectionID, payload.Collection) + issues = append(issues, validateExternalBusinessWriteObject("match", payload.Match, true)...) + issues = append(issues, validateExternalBusinessWriteObject("patch", payload.Patch, true)...) + issues = append(issues, validateExternalBusinessWriteObject("setOnInsert", payload.SetOnInsert, false)...) + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if !externalBusinessWriteAllowed(currentExternalBusinessWriteToken(request), payload.ConnectionID, payload.Collection) { + server.renderExternalBusinessWriteForbidden(writer) + return + } + browserStore, _, cleanup, err := server.resolveBusinessDatabaseStore(request.Context(), payload.ConnectionID) + if err != nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: err.Error()}) + return + } + defer cleanup() + document, created, err := browserStore.UpsertDatabaseDocument(request.Context(), payload.Collection, payload.Match, payload.Patch, payload.SetOnInsert) + if err != nil { + renderDatabaseBrowserWriteError(writer, err) + return + } + if token := currentExternalBusinessWriteToken(request); token != nil { + server.touchExternalBusinessWriteToken(request.Context(), token.ID) + } + operation := "updated" + if created { + operation = "created" + } + renderJSON(writer, http.StatusOK, externalBusinessWriteDocumentResult{ + ConnectionID: strings.TrimSpace(payload.ConnectionID), + Collection: strings.TrimSpace(payload.Collection), + Operation: operation, + Document: document, + }) +} diff --git a/internal/api/database_external_write_test.go b/internal/api/database_external_write_test.go new file mode 100644 index 00000000..aa4479de --- /dev/null +++ b/internal/api/database_external_write_test.go @@ -0,0 +1,416 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +type externalWriteTokenCreateResponse struct { + Token map[string]any `json:"token"` + APIKey string `json:"apiKey"` +} + +type externalWriteDocumentResponse struct { + ConnectionID string `json:"connectionId"` + Collection string `json:"collection"` + Operation string `json:"operation"` + Document map[string]any `json:"document"` +} + +func setupExternalBusinessWriteTest(t *testing.T) (*store.MemoryStore, *store.MemoryStore, http.Handler, *domain.MongoConnectionRecord) { + t.Helper() + + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{ + Name: "第三方业务库", + URI: "mongodb://partner-write.example:27017", + Database: "partner-business", + }) + if err != nil { + t.Fatalf("create mongo connection: %v", err) + } + + businessStore := store.NewMemoryStore() + originalResolver := mongoConnectionStoreResolver + mongoConnectionStoreResolver = func(_ context.Context, item *domain.MongoConnectionRecord) (databaseBrowserStore, func(), error) { + if item == nil || item.ID != connection.ID { + t.Fatalf("unexpected mongo connection: %#v", item) + } + return businessStore, func() {}, nil + } + t.Cleanup(func() { + mongoConnectionStoreResolver = originalResolver + }) + + return dataStore, businessStore, NewHandler(dataStore, nil), connection +} + +func createExternalBusinessWriteToken( + t *testing.T, + handler http.Handler, + connectionID string, + collections []string, +) (string, string) { + t.Helper() + + payload := map[string]any{ + "name": "第三方写入方", + "description": "测试 token", + "enabled": true, + "scopes": []map[string]any{ + { + "connectionId": connectionID, + "collections": collections, + }, + }, + } + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal token create payload: %v", err) + } + + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/external/business-db/tokens", bytes.NewReader(body)) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusCreated { + t.Fatalf("create external write token status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var response externalWriteTokenCreateResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode token create response: %v", err) + } + if response.APIKey == "" { + t.Fatalf("expected one-time api key in response: %s", recorder.Body.String()) + } + tokenID, _ := response.Token["id"].(string) + if tokenID == "" { + t.Fatalf("expected token id in response: %s", recorder.Body.String()) + } + return tokenID, response.APIKey +} + +func TestExternalBusinessWriteTokenCRUD(t *testing.T) { + ctx := context.Background() + dataStore, businessStore, handler, connection := setupExternalBusinessWriteTest(t) + + if _, err := businessStore.CreateDatabaseCollection(ctx, "orders"); err != nil { + t.Fatalf("create business collection: %v", err) + } + + tokenID, apiKey := createExternalBusinessWriteToken(t, handler, connection.ID, []string{"orders"}) + + settings, err := dataStore.GetSettings(ctx) + if err != nil { + t.Fatalf("get settings: %v", err) + } + serialized, err := json.Marshal(settings) + if err != nil { + t.Fatalf("marshal settings: %v", err) + } + settingsDocument := map[string]any{} + if err := json.Unmarshal(serialized, &settingsDocument); err != nil { + t.Fatalf("decode settings document: %v", err) + } + externalConfig, ok := settingsDocument["externalBusinessWrite"].(map[string]any) + if !ok { + t.Fatalf("expected externalBusinessWrite in settings, got %#v", settingsDocument) + } + tokens, ok := externalConfig["tokens"].([]any) + if !ok || len(tokens) != 1 { + t.Fatalf("expected one stored token, got %#v", externalConfig["tokens"]) + } + storedToken, ok := tokens[0].(map[string]any) + if !ok { + t.Fatalf("expected token document, got %#v", tokens[0]) + } + if storedToken["tokenHash"] == "" { + t.Fatalf("expected stored token hash, got %#v", storedToken) + } + if storedToken["tokenValue"] != apiKey { + t.Fatalf("expected stored plaintext token for explicit reveal, got %#v", storedToken["tokenValue"]) + } + if storedToken["apiKey"] != nil { + t.Fatalf("did not expect plaintext apiKey in stored settings: %#v", storedToken) + } + + listRecorder := httptest.NewRecorder() + handler.ServeHTTP(listRecorder, httptest.NewRequest(http.MethodGet, "/api/external/business-db/tokens", nil)) + if listRecorder.Code != http.StatusOK { + t.Fatalf("list external write tokens status: %d body=%s", listRecorder.Code, listRecorder.Body.String()) + } + if bytes.Contains(listRecorder.Body.Bytes(), []byte(apiKey)) { + t.Fatalf("list response must not expose plaintext api keys: %s", listRecorder.Body.String()) + } + if bytes.Contains(listRecorder.Body.Bytes(), []byte("tokenValue")) { + t.Fatalf("list response must not expose tokenValue field: %s", listRecorder.Body.String()) + } + + revealRecorder := httptest.NewRecorder() + handler.ServeHTTP(revealRecorder, httptest.NewRequest(http.MethodGet, "/api/external/business-db/tokens/"+tokenID+"/reveal", nil)) + if revealRecorder.Code != http.StatusOK { + t.Fatalf("reveal external write token status: %d body=%s", revealRecorder.Code, revealRecorder.Body.String()) + } + if !bytes.Contains(revealRecorder.Body.Bytes(), []byte(apiKey)) { + t.Fatalf("expected reveal response to contain plaintext api key, got %s", revealRecorder.Body.String()) + } + + rotateRecorder := httptest.NewRecorder() + handler.ServeHTTP( + rotateRecorder, + httptest.NewRequest(http.MethodPost, "/api/external/business-db/tokens/"+tokenID+"/rotate", nil), + ) + if rotateRecorder.Code != http.StatusOK { + t.Fatalf("rotate external write token status: %d body=%s", rotateRecorder.Code, rotateRecorder.Body.String()) + } + var rotateResponse externalWriteTokenCreateResponse + if err := json.Unmarshal(rotateRecorder.Body.Bytes(), &rotateResponse); err != nil { + t.Fatalf("decode rotate response: %v", err) + } + if rotateResponse.APIKey == "" || rotateResponse.APIKey == apiKey { + t.Fatalf("expected a new api key on rotation: old=%q new=%q", apiKey, rotateResponse.APIKey) + } + + revealRotatedRecorder := httptest.NewRecorder() + handler.ServeHTTP(revealRotatedRecorder, httptest.NewRequest(http.MethodGet, "/api/external/business-db/tokens/"+tokenID+"/reveal", nil)) + if revealRotatedRecorder.Code != http.StatusOK { + t.Fatalf("reveal rotated external write token status: %d body=%s", revealRotatedRecorder.Code, revealRotatedRecorder.Body.String()) + } + if !bytes.Contains(revealRotatedRecorder.Body.Bytes(), []byte(rotateResponse.APIKey)) { + t.Fatalf("expected reveal response to contain rotated api key, got %s", revealRotatedRecorder.Body.String()) + } + + createPayload := fmt.Sprintf(`{"connectionId":"%s","collection":"orders","document":{"orderNo":"SO-100","status":"new"}}`, connection.ID) + + oldKeyRecorder := httptest.NewRecorder() + oldKeyRequest := httptest.NewRequest(http.MethodPost, "/api/external/business-db/documents", bytes.NewBufferString(createPayload)) + oldKeyRequest.Header.Set("X-API-Key", apiKey) + handler.ServeHTTP(oldKeyRecorder, oldKeyRequest) + if oldKeyRecorder.Code != http.StatusUnauthorized { + t.Fatalf("expected old api key to be invalid after rotation, got status=%d body=%s", oldKeyRecorder.Code, oldKeyRecorder.Body.String()) + } + + newKeyRecorder := httptest.NewRecorder() + newKeyRequest := httptest.NewRequest(http.MethodPost, "/api/external/business-db/documents", bytes.NewBufferString(createPayload)) + newKeyRequest.Header.Set("X-API-Key", rotateResponse.APIKey) + handler.ServeHTTP(newKeyRecorder, newKeyRequest) + if newKeyRecorder.Code != http.StatusCreated { + t.Fatalf("expected rotated api key to work, got status=%d body=%s", newKeyRecorder.Code, newKeyRecorder.Body.String()) + } +} + +func TestExternalBusinessWriteAuthAndCollectionScope(t *testing.T) { + ctx := context.Background() + dataStore, businessStore, handler, connection := setupExternalBusinessWriteTest(t) + + if _, err := businessStore.CreateDatabaseCollection(ctx, "orders"); err != nil { + t.Fatalf("create orders collection: %v", err) + } + if _, err := businessStore.CreateDatabaseCollection(ctx, "customers"); err != nil { + t.Fatalf("create customers collection: %v", err) + } + tokenID, apiKey := createExternalBusinessWriteToken(t, handler, connection.ID, []string{"orders"}) + + createPayload := fmt.Sprintf(`{"connectionId":"%s","collection":"orders","document":{"orderNo":"SO-200","status":"new"}}`, connection.ID) + + missingKeyRecorder := httptest.NewRecorder() + handler.ServeHTTP(missingKeyRecorder, httptest.NewRequest(http.MethodPost, "/api/external/business-db/documents", bytes.NewBufferString(createPayload))) + if missingKeyRecorder.Code != http.StatusUnauthorized { + t.Fatalf("expected missing key to be rejected, got status=%d body=%s", missingKeyRecorder.Code, missingKeyRecorder.Body.String()) + } + + invalidKeyRecorder := httptest.NewRecorder() + invalidKeyRequest := httptest.NewRequest(http.MethodPost, "/api/external/business-db/documents", bytes.NewBufferString(createPayload)) + invalidKeyRequest.Header.Set("X-API-Key", "invalid-token") + handler.ServeHTTP(invalidKeyRecorder, invalidKeyRequest) + if invalidKeyRecorder.Code != http.StatusUnauthorized { + t.Fatalf("expected invalid key to be rejected, got status=%d body=%s", invalidKeyRecorder.Code, invalidKeyRecorder.Body.String()) + } + + otherConnection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{ + Name: "未授权业务库", + URI: "mongodb://other-business.example:27017", + Database: "other-business", + }) + if err != nil { + t.Fatalf("create second mongo connection: %v", err) + } + + unauthorizedConnectionRecorder := httptest.NewRecorder() + unauthorizedConnectionRequest := httptest.NewRequest( + http.MethodPost, + "/api/external/business-db/documents", + bytes.NewBufferString(fmt.Sprintf(`{"connectionId":"%s","collection":"orders","document":{"orderNo":"SO-201"}}`, otherConnection.ID)), + ) + unauthorizedConnectionRequest.Header.Set("X-API-Key", apiKey) + handler.ServeHTTP(unauthorizedConnectionRecorder, unauthorizedConnectionRequest) + if unauthorizedConnectionRecorder.Code != http.StatusForbidden { + t.Fatalf("expected unauthorized connection to be rejected, got status=%d body=%s", unauthorizedConnectionRecorder.Code, unauthorizedConnectionRecorder.Body.String()) + } + + unauthorizedCollectionRecorder := httptest.NewRecorder() + unauthorizedCollectionRequest := httptest.NewRequest( + http.MethodPost, + "/api/external/business-db/documents", + bytes.NewBufferString(fmt.Sprintf(`{"connectionId":"%s","collection":"customers","document":{"customerNo":"C-1"}}`, connection.ID)), + ) + unauthorizedCollectionRequest.Header.Set("X-API-Key", apiKey) + handler.ServeHTTP(unauthorizedCollectionRecorder, unauthorizedCollectionRequest) + if unauthorizedCollectionRecorder.Code != http.StatusForbidden { + t.Fatalf("expected unauthorized collection to be rejected, got status=%d body=%s", unauthorizedCollectionRecorder.Code, unauthorizedCollectionRecorder.Body.String()) + } + + disableRecorder := httptest.NewRecorder() + disableRequest := httptest.NewRequest( + http.MethodPut, + "/api/external/business-db/tokens/"+tokenID, + bytes.NewBufferString(`{"enabled":false}`), + ) + handler.ServeHTTP(disableRecorder, disableRequest) + if disableRecorder.Code != http.StatusOK { + t.Fatalf("disable token status: %d body=%s", disableRecorder.Code, disableRecorder.Body.String()) + } + + disabledRecorder := httptest.NewRecorder() + disabledRequest := httptest.NewRequest(http.MethodPost, "/api/external/business-db/documents", bytes.NewBufferString(createPayload)) + disabledRequest.Header.Set("X-API-Key", apiKey) + handler.ServeHTTP(disabledRecorder, disabledRequest) + if disabledRecorder.Code != http.StatusUnauthorized { + t.Fatalf("expected disabled token to be rejected, got status=%d body=%s", disabledRecorder.Code, disabledRecorder.Body.String()) + } +} + +func TestExternalBusinessWriteCreateAndPatchByID(t *testing.T) { + ctx := context.Background() + _, businessStore, handler, connection := setupExternalBusinessWriteTest(t) + + if _, err := businessStore.CreateDatabaseCollection(ctx, "orders"); err != nil { + t.Fatalf("create business collection: %v", err) + } + _, apiKey := createExternalBusinessWriteToken(t, handler, connection.ID, []string{"orders"}) + + createRecorder := httptest.NewRecorder() + createRequest := httptest.NewRequest( + http.MethodPost, + "/api/external/business-db/documents", + bytes.NewBufferString(fmt.Sprintf(`{"connectionId":"%s","collection":"orders","document":{"orderNo":"SO-300","status":"new","amount":10}}`, connection.ID)), + ) + createRequest.Header.Set("X-API-Key", apiKey) + handler.ServeHTTP(createRecorder, createRequest) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create external document status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var createResponse externalWriteDocumentResponse + if err := json.Unmarshal(createRecorder.Body.Bytes(), &createResponse); err != nil { + t.Fatalf("decode create response: %v", err) + } + documentID, _ := createResponse.Document["_id"].(string) + if documentID == "" { + t.Fatalf("expected created document id, got %#v", createResponse.Document) + } + if createResponse.Operation != "created" { + t.Fatalf("expected create operation, got %#v", createResponse) + } + + patchRecorder := httptest.NewRecorder() + patchRequest := httptest.NewRequest( + http.MethodPatch, + "/api/external/business-db/documents/"+documentID, + bytes.NewBufferString(fmt.Sprintf(`{"connectionId":"%s","collection":"orders","patch":{"status":"done","amount":18}}`, connection.ID)), + ) + patchRequest.Header.Set("X-API-Key", apiKey) + handler.ServeHTTP(patchRecorder, patchRequest) + if patchRecorder.Code != http.StatusOK { + t.Fatalf("patch external document status: %d body=%s", patchRecorder.Code, patchRecorder.Body.String()) + } + + var patchResponse externalWriteDocumentResponse + if err := json.Unmarshal(patchRecorder.Body.Bytes(), &patchResponse); err != nil { + t.Fatalf("decode patch response: %v", err) + } + if patchResponse.Operation != "updated" { + t.Fatalf("expected update operation, got %#v", patchResponse) + } + if patchResponse.Document["_id"] != documentID { + t.Fatalf("expected _id to stay stable, got %#v", patchResponse.Document["_id"]) + } + if patchResponse.Document["status"] != "done" { + t.Fatalf("expected patched status, got %#v", patchResponse.Document) + } +} + +func TestExternalBusinessWriteUpsertByBusinessKey(t *testing.T) { + ctx := context.Background() + _, businessStore, handler, connection := setupExternalBusinessWriteTest(t) + + if _, err := businessStore.CreateDatabaseCollection(ctx, "orders"); err != nil { + t.Fatalf("create business collection: %v", err) + } + _, apiKey := createExternalBusinessWriteToken(t, handler, connection.ID, []string{"orders"}) + + createRecorder := httptest.NewRecorder() + createRequest := httptest.NewRequest( + http.MethodPost, + "/api/external/business-db/upserts", + bytes.NewBufferString(fmt.Sprintf(`{"connectionId":"%s","collection":"orders","match":{"externalId":"EXT-1"},"patch":{"status":"new","amount":20},"setOnInsert":{"source":"partner-a"}}`, connection.ID)), + ) + createRequest.Header.Set("X-API-Key", apiKey) + handler.ServeHTTP(createRecorder, createRequest) + if createRecorder.Code != http.StatusOK { + t.Fatalf("create upsert status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var createResponse externalWriteDocumentResponse + if err := json.Unmarshal(createRecorder.Body.Bytes(), &createResponse); err != nil { + t.Fatalf("decode create upsert response: %v", err) + } + if createResponse.Operation != "created" { + t.Fatalf("expected created operation from upsert create path, got %#v", createResponse) + } + documentID, _ := createResponse.Document["_id"].(string) + if documentID == "" { + t.Fatalf("expected upserted document id, got %#v", createResponse.Document) + } + if createResponse.Document["source"] != "partner-a" { + t.Fatalf("expected setOnInsert field, got %#v", createResponse.Document) + } + + updateRecorder := httptest.NewRecorder() + updateRequest := httptest.NewRequest( + http.MethodPost, + "/api/external/business-db/upserts", + bytes.NewBufferString(fmt.Sprintf(`{"connectionId":"%s","collection":"orders","match":{"externalId":"EXT-1"},"patch":{"status":"done","amount":25},"setOnInsert":{"source":"partner-b"}}`, connection.ID)), + ) + updateRequest.Header.Set("X-API-Key", apiKey) + handler.ServeHTTP(updateRecorder, updateRequest) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("update upsert status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + + var updateResponse externalWriteDocumentResponse + if err := json.Unmarshal(updateRecorder.Body.Bytes(), &updateResponse); err != nil { + t.Fatalf("decode update upsert response: %v", err) + } + if updateResponse.Operation != "updated" { + t.Fatalf("expected updated operation from upsert update path, got %#v", updateResponse) + } + if updateResponse.Document["_id"] != documentID { + t.Fatalf("expected same document to be updated, got %#v", updateResponse.Document) + } + if updateResponse.Document["status"] != "done" || updateResponse.Document["amount"] != float64(25) { + t.Fatalf("expected updated fields in upsert response, got %#v", updateResponse.Document) + } + if updateResponse.Document["source"] != "partner-a" { + t.Fatalf("expected setOnInsert fields to be preserved on update, got %#v", updateResponse.Document) + } +} diff --git a/internal/api/object_storage_browser.go b/internal/api/object_storage_browser.go new file mode 100644 index 00000000..ae7b85ee --- /dev/null +++ b/internal/api/object_storage_browser.go @@ -0,0 +1,829 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "strings" + + "droboticflow/internal/domain" + + "github.com/baidubce/bce-sdk-go/bce" + "github.com/baidubce/bce-sdk-go/services/bos" + bosapi "github.com/baidubce/bce-sdk-go/services/bos/api" +) + +type objectStorageBrowser interface { + ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error) + ListObjects(context.Context, string, string) ([]domain.ObjectStorageObjectSummary, error) + CreateBucket(context.Context, string) (domain.ObjectStorageBucketSummary, error) + DeleteBucket(context.Context, string) error + PutObject(context.Context, string, string, []byte, string) (domain.ObjectStorageObjectSummary, error) + RenameObject(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error) + DeleteObject(context.Context, string, string) error + DeletePrefix(context.Context, string, string) error + GetObjectSummary(context.Context, string, string) (domain.ObjectStorageObjectSummary, error) + ReadObject(context.Context, string, string) ([]byte, string, error) + ReadObjectPreview(context.Context, string, string, int64) ([]byte, string, error) + BuildPreviewURL(context.Context, string, string) (string, error) +} + +type bosObjectStorageBrowser struct { + connection *domain.StorageConnectionRecord + client *bos.Client +} + +const ( + objectStorageInlinePreviewMaxBytes int64 = 256 * 1024 + objectStoragePartialPreviewBytes int64 = 64 * 1024 + objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated" + objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes" + objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes" +) + +var storageConnectionBrowserResolver = func(ctx context.Context, connection *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + if connection == nil { + return nil, nil, nil + } + client, err := bos.NewClient( + strings.TrimSpace(connection.AccessKeyID), + strings.TrimSpace(connection.SecretAccessKey), + storageEndpointForProvider(connection.Provider, connection.Endpoint), + ) + if err != nil { + return nil, nil, err + } + return &bosObjectStorageBrowser{connection: connection, client: client}, func() {}, nil +} + +func objectStorageKindForObject(key string, contentType string) string { + contentType = strings.ToLower(strings.TrimSpace(contentType)) + if strings.HasPrefix(contentType, "image/") { + return "image" + } + if strings.HasPrefix(contentType, "video/") { + return "video" + } + if strings.HasPrefix(contentType, "audio/") { + return "audio" + } + if strings.HasPrefix(contentType, "application/json") { + return "json" + } + switch strings.ToLower(path.Ext(key)) { + case ".json": + return "json" + case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg": + return "image" + case ".mp4", ".mov", ".m4v", ".webm", ".avi", ".mkv": + return "video" + case ".mp3", ".wav", ".aac", ".ogg", ".flac", ".m4a": + return "audio" + } + return "file" +} + +func objectStorageSummaryFromListObject(bucket string, item bosapi.ObjectSummaryType) domain.ObjectStorageObjectSummary { + key := strings.TrimSpace(item.Key) + contentType := detectObjectStorageContentType(key, nil, "") + return domain.ObjectStorageObjectSummary{ + Bucket: bucket, + Key: key, + Name: path.Base(strings.TrimRight(key, "/")), + Kind: objectStorageKindForObject(key, contentType), + SizeBytes: int64(item.Size), + ContentType: contentType, + LastModified: strings.TrimSpace(item.LastModified), + ETag: strings.Trim(item.ETag, "\""), + StorageClass: strings.TrimSpace(item.StorageClass), + } +} + +func objectStorageSummaryFromPrefix(bucket string, prefix string) domain.ObjectStorageObjectSummary { + trimmed := strings.TrimSpace(prefix) + return domain.ObjectStorageObjectSummary{ + Bucket: bucket, + Key: strings.TrimRight(trimmed, "/"), + Name: path.Base(strings.TrimRight(trimmed, "/")), + Kind: "directory", + } +} + +func objectStorageSummaryFromMeta(bucket string, key string, meta *bosapi.GetObjectMetaResult) domain.ObjectStorageObjectSummary { + contentType := "" + sizeBytes := int64(0) + lastModified := "" + etag := "" + storageClass := "" + metadata := map[string]any{} + if meta != nil { + contentType = strings.TrimSpace(meta.ContentType) + sizeBytes = meta.ContentLength + lastModified = strings.TrimSpace(meta.LastModified) + etag = strings.Trim(meta.ETag, "\"") + storageClass = strings.TrimSpace(meta.StorageClass) + for metaKey, metaValue := range meta.UserMeta { + metadata[metaKey] = metaValue + } + } + if len(metadata) == 0 { + metadata = nil + } + return domain.ObjectStorageObjectSummary{ + Bucket: bucket, + Key: key, + Name: path.Base(strings.TrimRight(key, "/")), + Kind: objectStorageKindForObject(key, contentType), + SizeBytes: sizeBytes, + ContentType: contentType, + LastModified: lastModified, + ETag: etag, + StorageClass: storageClass, + Metadata: metadata, + } +} + +func (browser *bosObjectStorageBrowser) ListBuckets(ctx context.Context) ([]domain.ObjectStorageBucketSummary, error) { + result, err := browser.client.ListBucketsWithContext(ctx) + if err != nil { + return nil, err + } + items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets)) + for _, bucket := range result.Buckets { + name := strings.TrimSpace(bucket.Name) + if name == "" { + continue + } + items = append(items, domain.ObjectStorageBucketSummary{Name: name, CreatedAt: strings.TrimSpace(bucket.CreationDate)}) + } + return items, nil +} + +func (browser *bosObjectStorageBrowser) ListObjects(ctx context.Context, bucket string, prefix string) ([]domain.ObjectStorageObjectSummary, error) { + normalizedBucket := strings.TrimSpace(bucket) + requestPrefix := storageObjectRequestPrefix(prefix) + items := make([]domain.ObjectStorageObjectSummary, 0) + seen := make(map[string]struct{}) + marker := "" + for { + result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ + Prefix: requestPrefix, + Delimiter: "/", + MaxKeys: 1000, + Marker: marker, + }) + if err != nil { + return nil, err + } + for _, item := range result.CommonPrefixes { + prefixValue := strings.Trim(strings.TrimSpace(item.Prefix), "/") + if prefixValue == "" || prefixValue == strings.Trim(strings.TrimSpace(prefix), "/") { + continue + } + if _, exists := seen["dir|"+prefixValue]; exists { + continue + } + seen["dir|"+prefixValue] = struct{}{} + items = append(items, objectStorageSummaryFromPrefix(normalizedBucket, prefixValue)) + } + for _, item := range result.Contents { + key := strings.Trim(strings.TrimSpace(item.Key), "/") + if key == "" || key == strings.Trim(strings.TrimSpace(prefix), "/") { + continue + } + if _, exists := seen["file|"+key]; exists { + continue + } + seen["file|"+key] = struct{}{} + items = append(items, objectStorageSummaryFromListObject(normalizedBucket, item)) + } + if result == nil || !result.IsTruncated { + break + } + nextMarker := strings.TrimSpace(result.NextMarker) + if nextMarker == "" || nextMarker == marker { + break + } + marker = nextMarker + } + return items, nil +} + +func (browser *bosObjectStorageBrowser) CreateBucket(_ context.Context, bucket string) (domain.ObjectStorageBucketSummary, error) { + if _, err := browser.client.PutBucket(strings.TrimSpace(bucket)); err != nil { + return domain.ObjectStorageBucketSummary{}, err + } + return domain.ObjectStorageBucketSummary{Name: strings.TrimSpace(bucket)}, nil +} + +func (browser *bosObjectStorageBrowser) DeleteBucket(_ context.Context, bucket string) error { + return browser.client.DeleteBucket(strings.TrimSpace(bucket)) +} + +func detectObjectStorageContentType(key string, payload []byte, explicit string) string { + if trimmed := strings.TrimSpace(explicit); trimmed != "" { + return trimmed + } + if guessed := mime.TypeByExtension(strings.ToLower(path.Ext(key))); guessed != "" { + return guessed + } + if len(payload) > 0 { + return http.DetectContentType(payload) + } + return "application/octet-stream" +} + +func supportsObjectStorageInlinePreview(summary domain.ObjectStorageObjectSummary) bool { + contentType := strings.ToLower(strings.TrimSpace(summary.ContentType)) + if summary.Kind == "json" { + return true + } + if strings.HasPrefix(contentType, "text/") { + return true + } + switch strings.ToLower(path.Ext(summary.Key)) { + case ".txt", ".log", ".md", ".csv", ".tsv", ".yaml", ".yml", ".xml", ".jsonl": + return true + } + return false +} + +func shouldReadObjectStorageInline(summary domain.ObjectStorageObjectSummary) bool { + if !supportsObjectStorageInlinePreview(summary) { + return false + } + if summary.SizeBytes <= 0 { + return true + } + return summary.SizeBytes <= objectStorageInlinePreviewMaxBytes +} + +func (browser *bosObjectStorageBrowser) PutObject(ctx context.Context, bucket string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) { + objectKey := strings.Trim(strings.TrimSpace(key), "/") + args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, payload, contentType)} + if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + return browser.GetObjectSummary(ctx, bucket, objectKey) +} + +func (browser *bosObjectStorageBrowser) RenameObject(ctx context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) { + objectKey := exactObjectStorageKey(key) + destinationKey := exactObjectStorageKey(newKey) + if _, err := browser.client.CopyObjectWithContext(ctx, strings.TrimSpace(bucket), destinationKey, strings.TrimSpace(bucket), objectKey, nil); err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + if err := browser.client.DeleteObject(strings.TrimSpace(bucket), objectKey); err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + return browser.GetObjectSummary(ctx, bucket, destinationKey) +} + +func (browser *bosObjectStorageBrowser) DeleteObject(_ context.Context, bucket string, key string) error { + return browser.client.DeleteObject(strings.TrimSpace(bucket), exactObjectStorageKey(key)) +} + +func (browser *bosObjectStorageBrowser) DeletePrefix(ctx context.Context, bucket string, prefix string) error { + normalizedBucket := strings.TrimSpace(bucket) + normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/") + if normalizedBucket == "" || normalizedPrefix == "" { + return fmt.Errorf("bucket and prefix are required") + } + requestPrefix := normalizedPrefix + "/" + marker := "" + for { + result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ + Prefix: requestPrefix, + MaxKeys: 1000, + Marker: marker, + }) + if err != nil { + return err + } + for _, item := range result.Contents { + key := exactObjectStorageKey(item.Key) + if key == "" { + continue + } + if err := browser.client.DeleteObject(normalizedBucket, key); err != nil { + return err + } + } + if result == nil || !result.IsTruncated { + break + } + nextMarker := strings.TrimSpace(result.NextMarker) + if nextMarker == "" || nextMarker == marker { + break + } + marker = nextMarker + } + for _, markerKey := range []string{normalizedPrefix + "/", normalizedPrefix} { + if err := browser.client.DeleteObject(normalizedBucket, markerKey); err != nil && !isBOSNoSuchKey(err) { + return err + } + } + return nil +} + +func (browser *bosObjectStorageBrowser) GetObjectSummary(ctx context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) { + objectKey := exactObjectStorageKey(key) + meta, err := browser.client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), objectKey) + if err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + return objectStorageSummaryFromMeta(strings.TrimSpace(bucket), objectKey, meta), nil +} + +func (browser *bosObjectStorageBrowser) ReadObject(ctx context.Context, bucket string, key string) ([]byte, string, error) { + objectKey := exactObjectStorageKey(key) + result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil) + if err != nil { + return nil, "", err + } + defer result.Body.Close() + payload, err := io.ReadAll(result.Body) + if err != nil { + return nil, "", err + } + return payload, strings.TrimSpace(result.ContentType), nil +} + +func (browser *bosObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucket string, key string, maxBytes int64) ([]byte, string, error) { + if maxBytes <= 0 { + return browser.ReadObject(ctx, bucket, key) + } + objectKey := exactObjectStorageKey(key) + result, err := browser.client.GetObjectWithContext( + ctx, + strings.TrimSpace(bucket), + objectKey, + nil, + 0, + maxBytes-1, + ) + if err != nil { + return nil, "", err + } + defer result.Body.Close() + payload, err := io.ReadAll(io.LimitReader(result.Body, maxBytes)) + if err != nil { + return nil, "", err + } + return payload, strings.TrimSpace(result.ContentType), nil +} + +func (browser *bosObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucket string, key string) (string, error) { + previewURL := strings.TrimSpace(browser.client.BasicGeneratePresignedUrl(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), 3600)) + if previewURL == "" { + return "", fmt.Errorf("failed to generate BOS preview url") + } + return previewURL, nil +} + +func decodeObjectStorageBucketCreateInput(request *http.Request) (domain.ObjectStorageBucketCreateInput, error) { + input := domain.ObjectStorageBucketCreateInput{} + if err := json.NewDecoder(request.Body).Decode(&input); err != nil { + return domain.ObjectStorageBucketCreateInput{}, err + } + return input, nil +} + +func decodeObjectStorageObjectPutInput(request *http.Request) (domain.ObjectStorageObjectPutInput, error) { + input := domain.ObjectStorageObjectPutInput{} + if err := json.NewDecoder(request.Body).Decode(&input); err != nil { + return domain.ObjectStorageObjectPutInput{}, err + } + return input, nil +} + +func decodeObjectStorageObjectRenameInput(request *http.Request) (domain.ObjectStorageObjectRenameInput, error) { + input := domain.ObjectStorageObjectRenameInput{} + if err := json.NewDecoder(request.Body).Decode(&input); err != nil { + return domain.ObjectStorageObjectRenameInput{}, err + } + return input, nil +} + +func (server *server) resolveObjectStorageBrowser(ctx context.Context, connectionID string) (objectStorageBrowser, *domain.StorageConnectionRecord, func(), error) { + connection, err := server.getStorageConnection(ctx, connectionID) + if err != nil && strings.TrimSpace(connectionID) != "" { + return nil, nil, nil, err + } + if connection == nil { + return nil, nil, nil, nil + } + browser, cleanup, err := storageConnectionBrowserResolver(ctx, connection) + if err != nil { + return nil, nil, nil, err + } + return browser, connection, cleanup, nil +} + +func resolveObjectStorageBucket(connection *domain.StorageConnectionRecord, requested string) (string, error) { + requested = strings.Trim(strings.TrimSpace(requested), "/") + if connection == nil { + return requested, nil + } + boundBucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/") + if boundBucket == "" { + return requested, nil + } + if requested != "" && requested != boundBucket { + return "", fmt.Errorf("connection is bound to bucket %s", boundBucket) + } + return boundBucket, nil +} + +func renderObjectStorageWriteError(writer http.ResponseWriter, err error) { + if err == nil { + return + } + if os.IsNotExist(err) || isBOSNoSuchKey(err) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()}) + return + } + renderError(writer, http.StatusInternalServerError, err) +} + +func exactObjectStorageKey(key string) string { + return strings.TrimSpace(key) +} + +func isBOSNoSuchKey(err error) bool { + if err == nil { + return false + } + var serviceErr *bce.BceServiceError + if errors.As(err, &serviceErr) { + return strings.EqualFold(strings.TrimSpace(serviceErr.Code), "NoSuchKey") + } + return strings.Contains(strings.ToLower(err.Error()), "nosuchkey") +} + +func (server *server) handleObjectStorageOverview(writer http.ResponseWriter, request *http.Request) { + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if cleanup != nil { + defer cleanup() + } + if browser == nil || connection == nil { + renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{ + ConnectionID: connectionID, + Provider: "BOS", + Configured: false, + Buckets: []domain.ObjectStorageBucketSummary{}, + }) + return + } + bucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/") + items := []domain.ObjectStorageBucketSummary{} + if bucket != "" { + items = append(items, domain.ObjectStorageBucketSummary{Name: bucket}) + } + renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{ + ConnectionID: connection.ID, + ConnectionName: connection.Name, + Provider: string(connection.Provider), + Endpoint: storageEndpointForProvider(connection.Provider, connection.Endpoint), + Bucket: bucket, + Configured: true, + Buckets: items, + }) +} + +func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, request *http.Request) { + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if cleanup != nil { + defer cleanup() + } + bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) + return + } + prefix := strings.Trim(strings.TrimSpace(request.URL.Query().Get("prefix")), "/") + if browser == nil || connection == nil { + renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{ + ConnectionID: connectionID, + Provider: "BOS", + Configured: false, + Bucket: bucket, + Prefix: prefix, + Items: []domain.ObjectStorageObjectSummary{}, + }) + return + } + items, err := browser.ListObjects(request.Context(), bucket, prefix) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{ + ConnectionID: connection.ID, + ConnectionName: connection.Name, + Provider: string(connection.Provider), + Endpoint: storageEndpointForProvider(connection.Provider, connection.Endpoint), + Configured: true, + Bucket: bucket, + Prefix: prefix, + Items: items, + }) +} + +func (server *server) handleObjectStorageBucketCreate(writer http.ResponseWriter, request *http.Request) { + renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"}) +} + +func (server *server) handleObjectStorageBucketDelete(writer http.ResponseWriter, request *http.Request) { + renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"}) +} + +func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter, request *http.Request) { + input, err := decodeObjectStorageObjectPutInput(request) + if err != nil { + renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"}) + return + } + if strings.TrimSpace(input.Key) == "" { + renderValidationError(writer, + validationIssue{Field: "key", Message: "key is required"}, + ) + return + } + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if cleanup != nil { + defer cleanup() + } + if browser == nil || connection == nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) + return + } + bucket, err := resolveObjectStorageBucket(connection, input.Bucket) + if err != nil || bucket == "" { + renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}) + return + } + item, err := browser.PutObject(request.Context(), bucket, input.Key, []byte(input.Content), input.ContentType) + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item}) +} + +func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter, request *http.Request) { + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if cleanup != nil { + defer cleanup() + } + if browser == nil || connection == nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) + return + } + bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) + return + } + key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/") + if bucket == "" || key == "" { + renderValidationError(writer, + validationIssue{Field: "bucket", Message: "bucket is required"}, + validationIssue{Field: "key", Message: "key is required"}, + ) + return + } + if err := request.ParseMultipartForm(32 << 20); err != nil { + renderValidationError(writer, validationIssue{Field: "file", Message: "invalid multipart payload"}) + return + } + file, header, err := request.FormFile("file") + if err != nil { + renderValidationError(writer, validationIssue{Field: "file", Message: "file is required"}) + return + } + defer file.Close() + payload, err := io.ReadAll(file) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + contentType := strings.TrimSpace(request.URL.Query().Get("contentType")) + if contentType == "" && header != nil { + contentType = strings.TrimSpace(header.Header.Get("Content-Type")) + } + item, err := browser.PutObject(request.Context(), bucket, key, payload, contentType) + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item}) +} + +func (server *server) handleObjectStorageObjectRename(writer http.ResponseWriter, request *http.Request) { + input, err := decodeObjectStorageObjectRenameInput(request) + if err != nil { + renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"}) + return + } + if strings.TrimSpace(input.Key) == "" || strings.TrimSpace(input.NewKey) == "" { + renderValidationError(writer, + validationIssue{Field: "key", Message: "key is required"}, + validationIssue{Field: "newKey", Message: "newKey is required"}, + ) + return + } + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if cleanup != nil { + defer cleanup() + } + if browser == nil || connection == nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) + return + } + bucket, err := resolveObjectStorageBucket(connection, input.Bucket) + if err != nil || bucket == "" { + renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}) + return + } + item, err := browser.RenameObject(request.Context(), bucket, input.Key, input.NewKey) + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item}) +} + +func (server *server) handleObjectStorageObjectDelete(writer http.ResponseWriter, request *http.Request) { + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if cleanup != nil { + defer cleanup() + } + if browser == nil || connection == nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) + return + } + bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) + return + } + key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/") + recursive := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("recursive")), "true") + if bucket == "" || key == "" { + renderValidationError(writer, + validationIssue{Field: "bucket", Message: "bucket is required"}, + validationIssue{Field: "key", Message: "key is required"}, + ) + return + } + if recursive { + err = browser.DeletePrefix(request.Context(), bucket, key) + } else { + err = browser.DeleteObject(request.Context(), bucket, key) + } + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectDeleteResult{Bucket: bucket, Key: key, Deleted: true}) +} + +func (server *server) handleObjectStorageContent(writer http.ResponseWriter, request *http.Request) { + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if cleanup != nil { + defer cleanup() + } + if browser == nil || connection == nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) + return + } + bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) + return + } + key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/") + if bucket == "" || key == "" { + renderValidationError(writer, + validationIssue{Field: "bucket", Message: "bucket is required"}, + validationIssue{Field: "key", Message: "key is required"}, + ) + return + } + summary, err := browser.GetObjectSummary(request.Context(), bucket, key) + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + redirectURL := "" + if previewURL, previewErr := browser.BuildPreviewURL(request.Context(), bucket, key); previewErr == nil { + redirectURL = previewURL + } + previewOnly := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("preview")), "true") + transcodeBrowser := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && summary.Kind == "video" + if transcodeBrowser { + previewRequest := sourceBrowserPreviewRequest{ + FileName: path.Base(key), + RedirectURL: redirectURL, + } + if previewRequest.RedirectURL == "" { + payload, _, err := browser.ReadObject(request.Context(), bucket, key) + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + previewRequest.Payload = payload + } + if previewPath, previewErr := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); previewErr == nil && previewPath != "" { + if err := writeSourceContentFile(writer, request, strings.TrimSuffix(path.Base(key), path.Ext(key))+"-browser.mp4", "video/mp4", previewPath); err != nil { + renderError(writer, http.StatusInternalServerError, err) + } + return + } + writer.Header().Set("Content-Type", "video/mp4") + writer.Header().Set("Cache-Control", "no-store") + writer.WriteHeader(http.StatusOK) + if flusher, ok := writer.(http.Flusher); ok { + flusher.Flush() + } + if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil { + fmt.Printf("[api] object storage browser preview stream failed: %v\n", err) + } + return + } + if previewOnly && supportsObjectStorageInlinePreview(summary) && summary.SizeBytes > objectStorageInlinePreviewMaxBytes { + payload, contentType, err := browser.ReadObjectPreview(request.Context(), bucket, key, objectStoragePartialPreviewBytes) + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + writer.Header().Set(objectStoragePreviewTruncatedHeader, "true") + writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes)) + writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload))) + writeSourceContent(writer, request, path.Base(key), contentType, payload) + return + } + if redirectURL != "" && !shouldReadObjectStorageInline(summary) { + http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) + return + } + payload, contentType, err := browser.ReadObject(request.Context(), bucket, key) + if err != nil { + if redirectURL != "" { + http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) + return + } + renderObjectStorageWriteError(writer, err) + return + } + if previewOnly { + writer.Header().Set(objectStoragePreviewTruncatedHeader, "false") + if summary.SizeBytes > 0 { + writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes)) + } + writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload))) + } + writeSourceContent(writer, request, path.Base(key), contentType, payload) +} diff --git a/internal/api/object_storage_browser_test.go b/internal/api/object_storage_browser_test.go new file mode 100644 index 00000000..fb3b3d04 --- /dev/null +++ b/internal/api/object_storage_browser_test.go @@ -0,0 +1,37 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/baidubce/bce-sdk-go/bce" +) + +func TestExactObjectStorageKeyPreservesTrailingSlash(t *testing.T) { + if got := exactObjectStorageKey(" algo/nested/ "); got != "algo/nested/" { + t.Fatalf("exactObjectStorageKey preserved key = %q, want %q", got, "algo/nested/") + } +} + +func TestIsBOSNoSuchKeyRecognizesServiceError(t *testing.T) { + err := bce.NewBceServiceError("NoSuchKey", "The specified key does not exist.", "req-1", http.StatusNotFound) + if !isBOSNoSuchKey(err) { + t.Fatalf("expected NoSuchKey to be recognized") + } +} + +func TestRenderObjectStorageWriteErrorTreatsBOSNoSuchKeyAsNotFound(t *testing.T) { + recorder := httptest.NewRecorder() + err := bce.NewBceServiceError("NoSuchKey", "The specified key does not exist.", "req-2", http.StatusNotFound) + + renderObjectStorageWriteError(recorder, err) + + if recorder.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusNotFound) + } + if !strings.Contains(recorder.Body.String(), "NoSuchKey") { + t.Fatalf("expected response to include NoSuchKey, got %s", recorder.Body.String()) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index c4021319..888999c3 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -4,15 +4,23 @@ import ( "context" "encoding/json" "errors" + "io" "net/http" "net/mail" + "regexp" + "sort" + "strconv" "strings" "time" "droboticflow/internal/ai" + "droboticflow/internal/config" "droboticflow/internal/domain" + "droboticflow/internal/k8s" "droboticflow/internal/store" + "github.com/baidubce/bce-sdk-go/services/bos" + bosapi "github.com/baidubce/bce-sdk-go/services/bos/api" "github.com/go-chi/chi/v5" ) @@ -22,10 +30,29 @@ type validationIssue struct { } type server struct { - store store.Store - ai ai.Service + store store.Store + ai ai.Service + operatorRunnerImage string } +type runJobTerminator interface { + TerminateRun(ctx context.Context, namespace string, runID string) error + TerminateJob(ctx context.Context, namespace string, jobName string) error +} + +var storageConnectionBucketsResolver = listStorageBucketsFromConnection +var storageConnectionObjectsResolver = listStorageObjectsFromConnection +var runJobTerminatorFactory = func(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string) (runJobTerminator, error) { + return k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath) +} +var workerAPIBaseURLResolver = func() string { + return "http://droboticflow-worker:3002" +} + +const maxStoragePathFavoritesPerScope = 12 + +var workflowScheduleTimeOfDayPattern = regexp.MustCompile(`^(?:[01]\d|2[0-3]):[0-5]\d$`) + func maskedSecretPreview(value string) string { compact := strings.Join(strings.Fields(strings.TrimSpace(value)), "") if compact == "" { @@ -48,14 +75,614 @@ func maskedSecretPreview(value string) string { } } +func sanitizeScanConfig(item *domain.ScanConfig) *domain.ScanConfig { + if item == nil { + return nil + } + copy := *item + copy.HasAccessKeyID = strings.TrimSpace(copy.AccessKeyID) != "" + copy.HasSecretAccessKey = strings.TrimSpace(copy.SecretAccessKey) != "" + copy.AccessKeyPreview = maskedSecretPreview(copy.AccessKeyID) + copy.SecretAccessKeyPreview = maskedSecretPreview(copy.SecretAccessKey) + return © +} + +func sanitizeStorageConnection(item *domain.StorageConnectionRecord) *domain.StorageConnectionRecord { + if item == nil { + return nil + } + copy := *item + copy.HasAccessKeyID = strings.TrimSpace(copy.AccessKeyID) != "" + copy.HasSecretAccessKey = strings.TrimSpace(copy.SecretAccessKey) != "" + copy.AccessKeyPreview = maskedSecretPreview(copy.AccessKeyID) + copy.SecretAccessKeyPreview = maskedSecretPreview(copy.SecretAccessKey) + copy.AccessKeyID = "" + copy.SecretAccessKey = "" + return © +} + +func sanitizeStorageConnections(items []domain.StorageConnectionRecord) []domain.StorageConnectionRecord { + if len(items) == 0 { + return items + } + sanitized := make([]domain.StorageConnectionRecord, 0, len(items)) + for _, item := range items { + copy := item + sanitized = append(sanitized, *sanitizeStorageConnection(©)) + } + return sanitized +} + +func sanitizeMongoConnection(item *domain.MongoConnectionRecord) *domain.MongoConnectionRecord { + if item == nil { + return nil + } + copy := *item + copy.HasURI = strings.TrimSpace(copy.URI) != "" + copy.URIPreview = maskedSecretPreview(copy.URI) + copy.URI = "" + return © +} + +func sanitizeMongoConnections(items []domain.MongoConnectionRecord) []domain.MongoConnectionRecord { + if len(items) == 0 { + return items + } + sanitized := make([]domain.MongoConnectionRecord, 0, len(items)) + for _, item := range items { + copy := item + sanitized = append(sanitized, *sanitizeMongoConnection(©)) + } + return sanitized +} + +func normalizeStoragePathFavoritePrefix(value string) string { + return strings.Trim(strings.TrimSpace(value), "/") +} + +func normalizeStoragePathFavoriteRecord(item domain.StoragePathFavoriteRecord) (domain.StoragePathFavoriteRecord, bool) { + scopeKey := strings.TrimSpace(item.ScopeKey) + bucket := strings.TrimSpace(item.Bucket) + if scopeKey == "" || bucket == "" { + return domain.StoragePathFavoriteRecord{}, false + } + prefix := normalizeStoragePathFavoritePrefix(item.Prefix) + label := strings.TrimSpace(item.Label) + if label == "" { + label = bucket + if prefix != "" { + label += "/" + prefix + } + } + return domain.StoragePathFavoriteRecord{ + ScopeKey: scopeKey, + Bucket: bucket, + Prefix: prefix, + Label: label, + }, true +} + +func sanitizeStoragePathFavorites(items []domain.StoragePathFavoriteRecord) []domain.StoragePathFavoriteRecord { + if len(items) == 0 { + return []domain.StoragePathFavoriteRecord{} + } + sanitized := make([]domain.StoragePathFavoriteRecord, 0, len(items)) + countsByScope := map[string]int{} + seen := map[string]struct{}{} + for _, item := range items { + normalized, ok := normalizeStoragePathFavoriteRecord(item) + if !ok { + continue + } + pathKey := normalized.Bucket + if normalized.Prefix != "" { + pathKey += "/" + normalized.Prefix + } + compositeKey := normalized.ScopeKey + "|" + pathKey + if _, exists := seen[compositeKey]; exists { + continue + } + if countsByScope[normalized.ScopeKey] >= maxStoragePathFavoritesPerScope { + continue + } + seen[compositeKey] = struct{}{} + countsByScope[normalized.ScopeKey] += 1 + sanitized = append(sanitized, normalized) + } + return sanitized +} + +func listStorageBucketsFromConnection(ctx context.Context, connection *domain.StorageConnectionRecord) ([]string, error) { + if connection == nil { + return []string{}, nil + } + client, err := bos.NewClient( + strings.TrimSpace(connection.AccessKeyID), + strings.TrimSpace(connection.SecretAccessKey), + storageEndpointForProvider(connection.Provider, connection.Endpoint), + ) + if err != nil { + return nil, err + } + result, err := client.ListBucketsWithContext(ctx) + if err != nil { + return nil, err + } + items := make([]string, 0, len(result.Buckets)) + for _, bucket := range result.Buckets { + name := strings.TrimSpace(bucket.Name) + if name != "" { + items = append(items, name) + } + } + sort.Strings(items) + return items, nil +} + +func storageObjectRequestPrefix(prefix string) string { + normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/") + if normalizedPrefix == "" { + return "" + } + return normalizedPrefix + "/" +} + +func appendStorageObjectItems(items map[string]struct{}, result *bosapi.ListObjectsResult, normalizedPrefix string, requestPrefix string) { + if result == nil { + return + } + for _, item := range result.CommonPrefixes { + prefixValue := strings.TrimSpace(item.Prefix) + if prefixValue == "" || prefixValue == requestPrefix { + continue + } + items[prefixValue] = struct{}{} + } + for _, item := range result.Contents { + key := strings.TrimSpace(item.Key) + if key == "" || key == normalizedPrefix || key == requestPrefix { + continue + } + items[key] = struct{}{} + } +} + +func listStorageObjectsFromConnection(ctx context.Context, connection *domain.StorageConnectionRecord, bucket string, prefix string) ([]string, error) { + if connection == nil { + return []string{}, nil + } + normalizedBucket := strings.TrimSpace(bucket) + if normalizedBucket == "" { + return []string{}, nil + } + client, err := bos.NewClient( + strings.TrimSpace(connection.AccessKeyID), + strings.TrimSpace(connection.SecretAccessKey), + storageEndpointForProvider(connection.Provider, connection.Endpoint), + ) + if err != nil { + return nil, err + } + normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/") + requestPrefix := storageObjectRequestPrefix(prefix) + collected := make(map[string]struct{}) + marker := "" + for { + result, err := client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ + Prefix: requestPrefix, + Delimiter: "/", + MaxKeys: 1000, + Marker: marker, + }) + if err != nil { + return nil, err + } + appendStorageObjectItems(collected, result, normalizedPrefix, requestPrefix) + if result == nil || !result.IsTruncated { + break + } + nextMarker := strings.TrimSpace(result.NextMarker) + if nextMarker == "" || nextMarker == marker { + break + } + marker = nextMarker + } + items := make([]string, 0, len(collected)) + for item := range collected { + items = append(items, item) + } + sort.Strings(items) + return items, nil +} + +func cloneWorkflowNodes(items []domain.WorkflowNode) []domain.WorkflowNode { + if len(items) == 0 { + return items + } + cloned := make([]domain.WorkflowNode, len(items)) + copy(cloned, items) + return cloned +} + +func sanitizeWorkflow(item *domain.WorkflowRecord) *domain.WorkflowRecord { + if item == nil { + return nil + } + copy := *item + copy.Nodes = cloneWorkflowNodes(item.Nodes) + copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig) + return © +} + +func sanitizeWorkflows(items []domain.WorkflowRecord) []domain.WorkflowRecord { + if len(items) == 0 { + return items + } + sanitized := make([]domain.WorkflowRecord, 0, len(items)) + for _, item := range items { + copy := item + copy.Nodes = cloneWorkflowNodes(item.Nodes) + copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig) + sanitized = append(sanitized, copy) + } + return sanitized +} + +type workflowNodeSourceStateResponse struct { + NodeID string `json:"nodeId"` + Supported bool `json:"supported"` + Stateful bool `json:"stateful"` + Reason string `json:"reason,omitempty"` + ScanMode string `json:"scanMode,omitempty"` + IncrementalUnit string `json:"incrementalUnit,omitempty"` + DirectoryDepth int `json:"directoryDepth,omitempty"` + DataSource string `json:"dataSource,omitempty"` + SourcePath string `json:"sourcePath,omitempty"` + Bucket string `json:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty"` + Pattern string `json:"pattern,omitempty"` + CurrentFingerprint string `json:"currentFingerprint,omitempty"` + Current *domain.SourceStateRecord `json:"current,omitempty"` + Versions []domain.SourceStateRecord `json:"versions,omitempty"` +} + +func workflowNodeByID(workflow *domain.WorkflowRecord, nodeID string) *domain.WorkflowNode { + if workflow == nil { + return nil + } + for index := range workflow.Nodes { + if workflow.Nodes[index].ID == nodeID { + return &workflow.Nodes[index] + } + } + return nil +} + +func runSummaryFromRecord(run domain.RunRecord) domain.RunSummary { + return domain.RunSummary{ + ID: run.ID, + WorkflowID: run.WorkflowID, + WorkflowName: run.WorkflowName, + ProjectID: run.ProjectID, + ProjectName: run.ProjectName, + Trigger: run.Trigger, + Attempt: run.Attempt, + ParentRunID: run.ParentRunID, + RestartFromNodeID: run.RestartFromNodeID, + AvailableAt: run.AvailableAt, + Status: run.Status, + StopMode: run.StopMode, + StopRequestedAt: run.StopRequestedAt, + StartedAt: run.StartedAt, + EndedAt: run.EndedAt, + DurationMS: run.DurationMS, + CurrentNodeID: run.CurrentNodeID, + Error: run.Error, + CreatedAt: run.CreatedAt, + UpdatedAt: run.UpdatedAt, + } +} + +func rerunSubtreeNodeSet(edges []domain.WorkflowEdge, startNodeID string) map[string]struct{} { + visited := map[string]struct{}{} + queue := []string{startNodeID} + for len(queue) > 0 { + nodeID := queue[0] + queue = queue[1:] + if _, ok := visited[nodeID]; ok { + continue + } + visited[nodeID] = struct{}{} + for _, edge := range domain.GetOutgoingEdges(edges, nodeID) { + if strings.TrimSpace(edge.Target) != "" { + queue = append(queue, edge.Target) + } + } + } + return visited +} + +func rerunBaselineNodeSet(edges []domain.WorkflowEdge, subtree map[string]struct{}) map[string]struct{} { + baseline := map[string]struct{}{} + seen := map[string]struct{}{} + queue := make([]string, 0, len(subtree)) + for nodeID := range subtree { + queue = append(queue, nodeID) + } + for len(queue) > 0 { + nodeID := queue[0] + queue = queue[1:] + if _, ok := seen[nodeID]; ok { + continue + } + seen[nodeID] = struct{}{} + for _, edge := range domain.GetIncomingEdges(edges, nodeID) { + sourceID := strings.TrimSpace(edge.Source) + if sourceID == "" { + continue + } + if _, ok := subtree[sourceID]; ok { + continue + } + if _, ok := baseline[sourceID]; ok { + continue + } + baseline[sourceID] = struct{}{} + queue = append(queue, sourceID) + } + } + return baseline +} + +func cloneLogEntries(items []domain.LogEntryRecord) []domain.LogEntryRecord { + if len(items) == 0 { + return []domain.LogEntryRecord{} + } + cloned := make([]domain.LogEntryRecord, len(items)) + copy(cloned, items) + return cloned +} + +func cloneExecutionArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact { + if len(items) == 0 { + return []domain.ExecutionArtifact{} + } + cloned := make([]domain.ExecutionArtifact, len(items)) + copy(cloned, items) + return cloned +} + +func cloneDataRefs(items []domain.DataRefRecord) []domain.DataRefRecord { + if len(items) == 0 { + return []domain.DataRefRecord{} + } + cloned := make([]domain.DataRefRecord, len(items)) + copy(cloned, items) + return cloned +} + +func cloneResourceSamples(items []domain.ResourceSample) []domain.ResourceSample { + if len(items) == 0 { + return []domain.ResourceSample{} + } + cloned := make([]domain.ResourceSample, len(items)) + copy(cloned, items) + return cloned +} + +func cloneNodeExecutionRecord(item domain.NodeExecutionRecord) domain.NodeExecutionRecord { + cloned := item + cloned.Logs = cloneLogEntries(item.Logs) + cloned.Artifacts = cloneExecutionArtifacts(item.Artifacts) + cloned.DataRefs = cloneDataRefs(item.DataRefs) + cloned.ResourceSeries = cloneResourceSamples(item.ResourceSeries) + return cloned +} + +func rebuildRunAggregates(run *domain.RunRecord) { + if run == nil { + return + } + run.Logs = []domain.LogEntryRecord{} + run.ResourceSeries = []domain.ResourceSample{} + run.Artifacts = []domain.ExecutionArtifact{} + for _, execution := range run.NodeExecutions { + run.Logs = append(run.Logs, cloneLogEntries(execution.Logs)...) + run.ResourceSeries = append(run.ResourceSeries, cloneResourceSamples(execution.ResourceSeries)...) + run.Artifacts = append(run.Artifacts, cloneExecutionArtifacts(execution.Artifacts)...) + } +} + +func workflowScanSourceNode(node *domain.WorkflowNode) bool { + if node == nil { + return false + } + label := strings.TrimSpace(node.Data.Label) + return node.ID == "scan-source" || label == "BOS 增量扫描" +} + +func sortSourceStatesDescending(items []domain.SourceStateRecord) { + sort.SliceStable(items, func(i int, j int) bool { + return items[i].UpdatedAt > items[j].UpdatedAt + }) +} + +func sourceStateFingerprintFromScanConfig(scanConfig *domain.ScanConfig) string { + if scanConfig == nil { + return "" + } + return domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{ + ScanMode: chooseWorkflowScanMode(scanConfig.ScanMode), + IncrementalUnit: chooseWorkflowIncrementalUnit(scanConfig.IncrementalUnit), + DirectoryDepth: chooseWorkflowDirectoryDepth(scanConfig.DirectoryDepth), + DataSource: chooseDataSource(scanConfig.DataSource), + SourcePath: scanConfig.SourcePath, + Bucket: scanConfig.Bucket, + Prefix: scanConfig.Prefix, + Pattern: scanConfig.Pattern, + Endpoint: scanConfig.Endpoint, + }) +} + +func (server *server) buildWorkflowNodeSourceStateResponse(ctx context.Context, workflow *domain.WorkflowRecord, nodeID string) (*workflowNodeSourceStateResponse, int, error) { + if workflow == nil { + return nil, http.StatusNotFound, nil + } + node := workflowNodeByID(workflow, nodeID) + if node == nil { + return nil, http.StatusNotFound, nil + } + response := &workflowNodeSourceStateResponse{ + NodeID: nodeID, + Supported: workflowScanSourceNode(node), + } + if !response.Supported { + response.Reason = "当前节点没有可持久化的扫描状态" + return response, http.StatusOK, nil + } + resolvedScanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig) + if err != nil { + return nil, http.StatusInternalServerError, err + } + if resolvedScanConfig == nil { + response.Reason = "当前工作流尚未配置扫描源" + return response, http.StatusOK, nil + } + response.ScanMode = chooseWorkflowScanMode(resolvedScanConfig.ScanMode) + response.IncrementalUnit = chooseWorkflowIncrementalUnit(resolvedScanConfig.IncrementalUnit) + response.DirectoryDepth = chooseWorkflowDirectoryDepth(resolvedScanConfig.DirectoryDepth) + response.DataSource = chooseDataSource(resolvedScanConfig.DataSource) + response.SourcePath = strings.TrimSpace(resolvedScanConfig.SourcePath) + response.Bucket = strings.TrimSpace(resolvedScanConfig.Bucket) + response.Prefix = strings.TrimSpace(resolvedScanConfig.Prefix) + response.Pattern = strings.TrimSpace(resolvedScanConfig.Pattern) + response.Stateful = response.ScanMode == "INCREMENTAL" + response.CurrentFingerprint = sourceStateFingerprintFromScanConfig(resolvedScanConfig) + versions, err := server.store.ListSourceStates(ctx, workflow.ID, nodeID) + if err != nil { + return nil, http.StatusInternalServerError, err + } + sortSourceStatesDescending(versions) + response.Versions = versions + for index := range versions { + if versions[index].SourceFingerprint == response.CurrentFingerprint { + state := versions[index] + response.Current = &state + break + } + } + if !response.Stateful { + response.Reason = "当前为 FULL 模式,不会读取或提交增量状态" + } else if response.Current == nil { + response.Reason = "当前配置下还没有已提交的增量状态" + } + return response, http.StatusOK, nil +} + +func sanitizeTemplate(item *domain.TemplateRecord) *domain.TemplateRecord { + if item == nil { + return nil + } + copy := *item + copy.Nodes = cloneWorkflowNodes(item.Nodes) + copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig) + return © +} + +func sanitizeTemplates(items []domain.TemplateRecord) []domain.TemplateRecord { + if len(items) == 0 { + return items + } + sanitized := make([]domain.TemplateRecord, 0, len(items)) + for _, item := range items { + copy := item + copy.Nodes = cloneWorkflowNodes(item.Nodes) + copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig) + sanitized = append(sanitized, copy) + } + return sanitized +} + +func sanitizeOperator(item *domain.OperatorRecord) *domain.OperatorRecord { + if item == nil { + return nil + } + copy := *item + return © +} + +func sanitizeOperators(items []domain.OperatorRecord) []domain.OperatorRecord { + if len(items) == 0 { + return items + } + sanitized := make([]domain.OperatorRecord, 0, len(items)) + for _, item := range items { + copy := item + sanitized = append(sanitized, copy) + } + return sanitized +} + +func sanitizeUser(item *domain.UserRecord) *domain.UserRecord { + if item == nil { + return nil + } + copy := *item + copy.StoragePathFavorites = nil + return © +} + +func sanitizeUsers(items []domain.UserRecord) []domain.UserRecord { + if len(items) == 0 { + return items + } + sanitized := make([]domain.UserRecord, 0, len(items)) + for _, item := range items { + copy := item + copy.StoragePathFavorites = nil + sanitized = append(sanitized, copy) + } + return sanitized +} + +func sanitizeProject(item *domain.ProjectRecord) *domain.ProjectRecord { + if item == nil { + return nil + } + copy := *item + copy.StoragePathFavorites = nil + return © +} + +func sanitizeProjects(items []domain.ProjectRecord) []domain.ProjectRecord { + if len(items) == 0 { + return items + } + sanitized := make([]domain.ProjectRecord, 0, len(items)) + for _, item := range items { + copy := item + copy.StoragePathFavorites = nil + sanitized = append(sanitized, copy) + } + return sanitized +} + func NewHandler(dataStore store.Store, aiService ai.Service) http.Handler { if aiService == nil { aiService = ai.NewDisabledService("", "", "AI 服务未配置") } - instance := &server{store: dataStore, ai: aiService} + instance := &server{ + store: dataStore, + ai: aiService, + operatorRunnerImage: config.Load().OperatorRunnerImage, + } router := chi.NewRouter() router.Get("/api/health", instance.handleHealth) + router.Get("/worker-api/health", instance.handleWorkerAPIHealth) + router.Get("/worker-api/stats", instance.handleWorkerAPIStats) + router.Post("/worker-api/process-once", instance.handleWorkerAPIProcessOnce) + router.Post("/worker-api/runs/{id}/retry", instance.handleWorkerAPIRetry) router.Get("/api/ai/status", instance.handleAIStatus) router.Post("/api/ai/operators/generate", instance.handleAIOperatorGenerate) router.Post("/api/ai/workflows/generate", instance.handleAIWorkflowGenerate) @@ -66,14 +693,39 @@ func NewHandler(dataStore store.Store, aiService ai.Service) http.Handler { router.Get("/api/projects/{id}", instance.handleProjectsGet) router.Put("/api/projects/{id}", instance.handleProjectsUpdate) router.Delete("/api/projects/{id}", instance.handleProjectsDelete) + router.Get("/api/projects/{id}/storage-path-favorites", instance.handleProjectStoragePathFavoritesGet) + router.Put("/api/projects/{id}/storage-path-favorites", instance.handleProjectStoragePathFavoritesUpdate) router.Get("/api/projects/{id}/workflows", instance.handleProjectWorkflowsList) router.Post("/api/projects/{id}/workflows", instance.handleProjectWorkflowsCreate) router.Get("/api/workflows/{id}", instance.handleWorkflowsGet) router.Put("/api/workflows/{id}", instance.handleWorkflowsUpdate) router.Delete("/api/workflows/{id}", instance.handleWorkflowsDelete) + router.Get("/api/workflows/{id}/runs", instance.handleWorkflowRunsList) + router.Get("/api/workflows/{id}/nodes/{nodeId}/source-state", instance.handleWorkflowNodeSourceStateGet) + router.Delete("/api/workflows/{id}/nodes/{nodeId}/source-state", instance.handleWorkflowNodeSourceStateDelete) router.Post("/api/workflows/{id}/run", instance.handleWorkflowRun) + router.Get("/api/scan-configs", instance.handleScanConfigsList) + router.Post("/api/scan-configs", instance.handleScanConfigsCreate) + router.Get("/api/scan-configs/{id}", instance.handleScanConfigsGet) + router.Put("/api/scan-configs/{id}", instance.handleScanConfigsUpdate) + router.Delete("/api/scan-configs/{id}", instance.handleScanConfigsDelete) + router.Post("/api/scan-configs/{id}/run", instance.handleScanConfigsRun) + + router.Get("/api/source-data/collections", instance.handleSourceCollectionsList) + router.Get("/api/source-data/collections/{runId}/{collectionId}", instance.handleSourceCollectionDetail) + router.Get("/api/source-data/collections/{runId}/{collectionId}/entries", instance.handleSourceCollectionEntries) + router.Get("/api/source-data/collections/{runId}/{collectionId}/content", instance.handleSourceCollectionContent) + + router.Get("/api/packages", instance.handlePackagesList) + router.Get("/api/packages/{runId}/{packageName}", instance.handlePackageDetail) + router.Get("/api/packages/{runId}/{packageName}/content", instance.handlePackageContent) + + router.Get("/api/sampling-tasks", instance.handleSamplingTasksList) + router.Get("/api/sampling-tasks/{id}", instance.handleSamplingTaskGet) + router.Post("/api/sampling-items/{id}/review", instance.handleSamplingItemReview) + router.Get("/api/templates", instance.handleTemplatesList) router.Post("/api/templates", instance.handleTemplatesCreate) router.Get("/api/templates/{id}", instance.handleTemplatesGet) @@ -86,8 +738,23 @@ func NewHandler(dataStore store.Store, aiService ai.Service) http.Handler { router.Put("/api/operators/{id}", instance.handleOperatorsUpdate) router.Delete("/api/operators/{id}", instance.handleOperatorsDelete) + router.Get("/api/database", instance.handleDatabaseOverview) + router.Post("/api/database/collections", instance.handleDatabaseCollectionCreate) + router.Patch("/api/database/collections/{name}", instance.handleDatabaseCollectionRename) + router.Delete("/api/database/collections/{name}", instance.handleDatabaseCollectionDelete) + router.Get("/api/database/collections/{name}/indexes", instance.handleDatabaseCollectionIndexes) + router.Post("/api/database/collections/{name}/indexes", instance.handleDatabaseCollectionIndexCreate) + router.Delete("/api/database/collections/{name}/indexes/{indexName}", instance.handleDatabaseCollectionIndexDelete) + router.Get("/api/database/collections/{name}/documents", instance.handleDatabaseCollectionDocuments) + router.Post("/api/database/collections/{name}/documents", instance.handleDatabaseDocumentCreate) + router.Put("/api/database/collections/{name}/documents/{documentId}", instance.handleDatabaseDocumentReplace) + router.Delete("/api/database/collections/{name}/documents/{documentId}", instance.handleDatabaseDocumentDelete) + router.Get("/api/runs", instance.handleRunsList) router.Get("/api/runs/{id}", instance.handleRunsGet) + router.Post("/api/runs/{id}/rerun-subtree", instance.handleRunsRerunSubtree) + router.Post("/api/runs/{id}/stop", instance.handleRunsStop) + router.Post("/api/runs/{id}/force-stop", instance.handleRunsForceStop) router.Get("/api/runs/{id}/logs", instance.handleRunsLogs) router.Get("/api/runs/{id}/resources", instance.handleRunsResources) router.Get("/api/runs/{id}/artifacts", instance.handleRunsArtifacts) @@ -100,12 +767,50 @@ func NewHandler(dataStore store.Store, aiService ai.Service) http.Handler { router.Post("/api/users", instance.handleUsersCreate) router.Put("/api/users/{id}", instance.handleUsersUpdate) router.Delete("/api/users/{id}", instance.handleUsersDelete) + router.Get("/api/users/{id}/storage-path-favorites", instance.handleUserStoragePathFavoritesGet) + router.Put("/api/users/{id}/storage-path-favorites", instance.handleUserStoragePathFavoritesUpdate) + + router.Get("/api/storage/connections", instance.handleStorageConnectionsList) + router.Post("/api/storage/connections", instance.handleStorageConnectionsCreate) + router.Get("/api/storage/connections/{id}", instance.handleStorageConnectionsGet) + router.Put("/api/storage/connections/{id}", instance.handleStorageConnectionsUpdate) + router.Delete("/api/storage/connections/{id}", instance.handleStorageConnectionsDelete) + + router.Get("/api/mongo/connections", instance.handleMongoConnectionsList) + router.Post("/api/mongo/connections", instance.handleMongoConnectionsCreate) + router.Post("/api/mongo/connections/test", instance.handleMongoConnectionsTest) + router.Get("/api/mongo/connections/{id}", instance.handleMongoConnectionsGet) + router.Put("/api/mongo/connections/{id}", instance.handleMongoConnectionsUpdate) + router.Delete("/api/mongo/connections/{id}", instance.handleMongoConnectionsDelete) + + router.Get("/api/external/business-db/tokens", instance.handleExternalBusinessWriteTokensList) + router.Post("/api/external/business-db/tokens", instance.handleExternalBusinessWriteTokensCreate) + router.Put("/api/external/business-db/tokens/{id}", instance.handleExternalBusinessWriteTokensUpdate) + router.Get("/api/external/business-db/tokens/{id}/reveal", instance.handleExternalBusinessWriteTokensReveal) + router.Post("/api/external/business-db/tokens/{id}/rotate", instance.handleExternalBusinessWriteTokensRotate) + router.Delete("/api/external/business-db/tokens/{id}", instance.handleExternalBusinessWriteTokensDelete) + + router.Route("/api/external/business-db", func(router chi.Router) { + router.Use(instance.externalBusinessWriteAuth) + router.Post("/documents", instance.handleExternalBusinessWriteCreate) + router.Patch("/documents/{documentId}", instance.handleExternalBusinessWritePatch) + router.Post("/upserts", instance.handleExternalBusinessWriteUpsert) + }) router.Get("/api/settings", instance.handleSettingsGet) router.Put("/api/settings", instance.handleSettingsUpdate) router.Get("/api/storage/buckets", instance.handleStorageBuckets) router.Get("/api/storage/objects", instance.handleStorageObjects) + router.Get("/api/object-storage", instance.handleObjectStorageOverview) + router.Get("/api/object-storage/objects", instance.handleObjectStorageObjects) + router.Post("/api/object-storage/buckets", instance.handleObjectStorageBucketCreate) + router.Delete("/api/object-storage/buckets/{name}", instance.handleObjectStorageBucketDelete) + router.Post("/api/object-storage/objects", instance.handleObjectStorageObjectCreate) + router.Post("/api/object-storage/objects/upload", instance.handleObjectStorageObjectUpload) + router.Patch("/api/object-storage/objects", instance.handleObjectStorageObjectRename) + router.Delete("/api/object-storage/objects", instance.handleObjectStorageObjectDelete) + router.Get("/api/object-storage/content", instance.handleObjectStorageContent) return router } @@ -114,6 +819,62 @@ func (server *server) handleHealth(writer http.ResponseWriter, _ *http.Request) renderJSON(writer, http.StatusOK, map[string]string{"status": "ok", "service": "droboticflow-api"}) } +func workerAPIBaseURL() string { + return strings.TrimRight(strings.TrimSpace(workerAPIBaseURLResolver()), "/") +} + +func proxyWorkerAPI(writer http.ResponseWriter, request *http.Request, path string) { + baseURL := workerAPIBaseURL() + if baseURL == "" { + renderJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "worker API is not configured"}) + return + } + + upstreamURL := baseURL + path + upstreamRequest, err := http.NewRequestWithContext(request.Context(), request.Method, upstreamURL, request.Body) + if err != nil { + renderError(writer, http.StatusBadGateway, err) + return + } + upstreamRequest.Header = request.Header.Clone() + + response, err := http.DefaultClient.Do(upstreamRequest) + if err != nil { + renderJSON(writer, http.StatusBadGateway, map[string]string{ + "error": "worker API request failed", + "details": err.Error(), + }) + return + } + defer response.Body.Close() + + for key, values := range response.Header { + for _, value := range values { + writer.Header().Add(key, value) + } + } + writer.WriteHeader(response.StatusCode) + if _, err := io.Copy(writer, response.Body); err != nil { + return + } +} + +func (server *server) handleWorkerAPIHealth(writer http.ResponseWriter, request *http.Request) { + proxyWorkerAPI(writer, request, "/worker-api/health") +} + +func (server *server) handleWorkerAPIStats(writer http.ResponseWriter, request *http.Request) { + proxyWorkerAPI(writer, request, "/worker-api/stats") +} + +func (server *server) handleWorkerAPIProcessOnce(writer http.ResponseWriter, request *http.Request) { + proxyWorkerAPI(writer, request, "/worker-api/process-once") +} + +func (server *server) handleWorkerAPIRetry(writer http.ResponseWriter, request *http.Request) { + proxyWorkerAPI(writer, request, "/worker-api/runs/"+chi.URLParam(request, "id")+"/retry") +} + func (server *server) handleAIStatus(writer http.ResponseWriter, _ *http.Request) { renderJSON(writer, http.StatusOK, server.resolveAIService(context.Background()).Status()) } @@ -148,7 +909,7 @@ func (server *server) handleAIWorkflowGenerate(writer http.ResponseWriter, reque renderValidationError(writer, validationIssue{Field: "description", Message: "description is required"}) return } - operators, err := server.store.ListOperators(request.Context()) + operators, err := server.listAvailableOperators(request.Context()) if err != nil { renderError(writer, http.StatusInternalServerError, err) return @@ -226,7 +987,7 @@ func (server *server) handleProjectsList(writer http.ResponseWriter, request *ht renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, map[string]any{"items": items}) + renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeProjects(items)}) } func (server *server) handleProjectsCreate(writer http.ResponseWriter, request *http.Request) { @@ -243,7 +1004,7 @@ func (server *server) handleProjectsCreate(writer http.ResponseWriter, request * renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusCreated, project) + renderJSON(writer, http.StatusCreated, sanitizeProject(project)) } func (server *server) handleProjectsGet(writer http.ResponseWriter, request *http.Request) { @@ -261,7 +1022,7 @@ func (server *server) handleProjectsGet(writer http.ResponseWriter, request *htt renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeProject(item)) } type projectPatch struct { @@ -304,7 +1065,7 @@ func (server *server) handleProjectsUpdate(writer http.ResponseWriter, request * renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeProject(item)) } func (server *server) handleProjectsDelete(writer http.ResponseWriter, request *http.Request) { @@ -320,13 +1081,53 @@ func (server *server) handleProjectsDelete(writer http.ResponseWriter, request * writer.WriteHeader(http.StatusNoContent) } +func (server *server) handleProjectStoragePathFavoritesGet(writer http.ResponseWriter, request *http.Request) { + project, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if project == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"}) + return + } + renderJSON(writer, http.StatusOK, map[string]any{ + "items": sanitizeStoragePathFavorites(project.StoragePathFavorites), + }) +} + +func (server *server) handleProjectStoragePathFavoritesUpdate(writer http.ResponseWriter, request *http.Request) { + project, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if project == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"}) + return + } + var payload storagePathFavoritesPayload + if !decodeJSON(writer, request, &payload) { + return + } + project.StoragePathFavorites = sanitizeStoragePathFavorites(payload.Items) + project.UpdatedAt = nowISO() + if err := server.store.UpsertProject(request.Context(), *project); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{ + "items": project.StoragePathFavorites, + }) +} + func (server *server) handleProjectWorkflowsList(writer http.ResponseWriter, request *http.Request) { items, err := server.store.ListWorkflowsByProject(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, map[string]any{"items": items}) + renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)}) } func validWorkflowType(value domain.WorkflowType) bool { @@ -353,29 +1154,151 @@ func validateWorkflowExecutionConfig(maxConcurrentRuns int, nodeTimeoutSeconds i return issues } +func cloneWorkflowSchedule(schedule *domain.WorkflowSchedule) *domain.WorkflowSchedule { + if schedule == nil { + return nil + } + cloned := *schedule + return &cloned +} + +func normalizeWorkflowSchedule(existing *domain.WorkflowSchedule, incoming *domain.WorkflowSchedule, referenceTime string) *domain.WorkflowSchedule { + if incoming == nil { + return cloneWorkflowSchedule(existing) + } + next := *incoming + next.SourcePath = strings.TrimSpace(next.SourcePath) + next.TimeOfDay = strings.TrimSpace(next.TimeOfDay) + next.AnchorAt = strings.TrimSpace(next.AnchorAt) + next.Timezone = strings.TrimSpace(next.Timezone) + if normalizedMode := domain.NormalizeWorkflowScheduleMode(next.Mode); normalizedMode != "" { + next.Mode = normalizedMode + } else if strings.TrimSpace(string(next.Mode)) == "" { + next.Mode = domain.EffectiveWorkflowScheduleMode(&next) + } + effectiveMode := domain.EffectiveWorkflowScheduleMode(&next) + if effectiveMode == domain.WorkflowScheduleModeEveryDaysAtTime || next.Timezone != "" { + next.Timezone = domain.WorkflowScheduleTimezone(next.Timezone) + } + if (effectiveMode == domain.WorkflowScheduleModeEveryHours || effectiveMode == domain.WorkflowScheduleModeEveryDaysAtTime) && + domain.WorkflowScheduleEnabled(&next) { + if next.AnchorAt == "" || domain.EffectiveWorkflowScheduleMode(existing) != effectiveMode || !domain.WorkflowScheduleEnabled(existing) { + next.AnchorAt = referenceTime + } + } + return &next +} + +func validateWorkflowSchedule(schedule *domain.WorkflowSchedule) []validationIssue { + if schedule == nil { + return nil + } + issues := make([]validationIssue, 0, 6) + if rawMode := strings.TrimSpace(string(schedule.Mode)); rawMode != "" && domain.NormalizeWorkflowScheduleMode(schedule.Mode) == "" { + issues = append(issues, validationIssue{Field: "schedule.mode", Message: "schedule.mode must be interval_seconds, every_hours, or every_days_at_time"}) + } + if schedule.Interval < 0 { + issues = append(issues, validationIssue{Field: "schedule.interval", Message: "schedule.interval must be zero or greater"}) + } + if schedule.IntervalHours < 0 { + issues = append(issues, validationIssue{Field: "schedule.intervalHours", Message: "schedule.intervalHours must be zero or greater"}) + } + if schedule.DayInterval < 0 { + issues = append(issues, validationIssue{Field: "schedule.dayInterval", Message: "schedule.dayInterval must be zero or greater"}) + } + if timezone := strings.TrimSpace(schedule.Timezone); timezone != "" { + if _, err := time.LoadLocation(timezone); err != nil { + issues = append(issues, validationIssue{Field: "schedule.timezone", Message: "schedule.timezone must be a valid IANA timezone"}) + } + } + switch domain.EffectiveWorkflowScheduleMode(schedule) { + case domain.WorkflowScheduleModeEveryHours: + if schedule.IntervalHours < 1 { + issues = append(issues, validationIssue{Field: "schedule.intervalHours", Message: "schedule.intervalHours must be greater than zero"}) + } + case domain.WorkflowScheduleModeEveryDaysAtTime: + if schedule.DayInterval < 1 { + issues = append(issues, validationIssue{Field: "schedule.dayInterval", Message: "schedule.dayInterval must be greater than zero"}) + } + if !workflowScheduleTimeOfDayPattern.MatchString(strings.TrimSpace(schedule.TimeOfDay)) { + issues = append(issues, validationIssue{Field: "schedule.timeOfDay", Message: "schedule.timeOfDay must use HH:mm format"}) + } + } + return issues +} + func (server *server) handleProjectWorkflowsCreate(writer http.ResponseWriter, request *http.Request) { var payload domain.CreateWorkflowInput if !decodeJSON(writer, request, &payload) { return } payload.ProjectID = chi.URLParam(request, "id") - if strings.TrimSpace(payload.Name) == "" || !validWorkflowType(payload.Type) { - renderValidationError(writer, - validationIssue{Field: "name", Message: "name is required"}, - validationIssue{Field: "type", Message: "type must be batch or streaming"}, - ) + if payload.Type == "" { + payload.Type = domain.WorkflowTypeBatch + } + issues := make([]validationIssue, 0, 2) + if strings.TrimSpace(payload.Name) == "" { + issues = append(issues, validationIssue{Field: "name", Message: "name is required"}) + } + if !validWorkflowType(payload.Type) { + issues = append(issues, validationIssue{Field: "type", Message: "type must be batch or streaming"}) + } + if len(issues) > 0 { + renderValidationError(writer, issues...) return } if issues := validateWorkflowExecutionConfig(payload.MaxConcurrentRuns, payload.NodeTimeoutSeconds, payload.RunTimeoutSeconds, payload.MaxRetryAttempts, payload.RetryBackoffSeconds); len(issues) > 0 { renderValidationError(writer, issues...) return } + payload.Schedule = normalizeWorkflowSchedule(nil, payload.Schedule, nowISO()) + if issues := validateWorkflowSchedule(payload.Schedule); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if payload.Kind == domain.WorkflowKindScan || payload.ScanConfig != nil || domain.FindScanSourceNode(payload.Nodes) != nil { + requestPayload := scanConfigRequestFromSourceNode(payload.Name, payload.Description, payload.Schedule, payload.ScanConfig, payload.Nodes) + if issues := validateScanConfigDraftPayload(requestPayload); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if requestPayload.Interval > 0 && !scanConfigPayloadReady(requestPayload, nil) { + renderValidationError(writer, validateScanConfigPayload(requestPayload, nil)...) + return + } + payload.Kind = domain.WorkflowKindScan + payload.ScanConfig = normalizedScanConfigFromRequest(nil, requestPayload) + if issues := server.validateScanConnectionReference(request.Context(), payload.ScanConfig); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + payload.Schedule = normalizedScanSchedule(payload.Schedule, payload.ScanConfig) + payload.Type = normalizedScanWorkflowType(payload.ScanConfig) + payload.MaxConcurrentRuns = normalizedScanMaxConcurrentRuns(payload.MaxConcurrentRuns) + if len(payload.Nodes) == 0 { + scaffold, err := server.buildScanWorkflowInput(request.Context(), payload.ProjectID, requestPayload, nil) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + payload.Nodes = scaffold.Nodes + payload.Edges = scaffold.Edges + } + } + if issues := server.validateNodeConnectionReferences(request.Context(), payload.Nodes); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if issues := validateInputPortConnectionTopology(payload.Nodes, payload.Edges); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } item, err := server.store.CreateWorkflow(request.Context(), payload) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusCreated, item) + renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item)) } func (server *server) handleWorkflowsGet(writer http.ResponseWriter, request *http.Request) { @@ -388,13 +1311,64 @@ func (server *server) handleWorkflowsGet(writer http.ResponseWriter, request *ht renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"}) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeWorkflow(item)) +} + +func (server *server) handleWorkflowNodeSourceStateGet(writer http.ResponseWriter, request *http.Request) { + workflow, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + response, status, err := server.buildWorkflowNodeSourceStateResponse(request.Context(), workflow, chi.URLParam(request, "nodeId")) + if err != nil { + renderError(writer, status, err) + return + } + if response == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow node not found"}) + return + } + renderJSON(writer, status, response) +} + +func (server *server) handleWorkflowNodeSourceStateDelete(writer http.ResponseWriter, request *http.Request) { + workflow, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + response, status, err := server.buildWorkflowNodeSourceStateResponse(request.Context(), workflow, chi.URLParam(request, "nodeId")) + if err != nil { + renderError(writer, status, err) + return + } + if response == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow node not found"}) + return + } + if !response.Supported { + renderValidationError(writer, validationIssue{Field: "nodeId", Message: response.Reason}) + return + } + if !response.Stateful || strings.TrimSpace(response.CurrentFingerprint) == "" { + writer.WriteHeader(http.StatusNoContent) + return + } + if err := server.store.DeleteSourceState(request.Context(), workflow.ID, response.NodeID, response.CurrentFingerprint); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + writer.WriteHeader(http.StatusNoContent) } type workflowPatch struct { Name *string `json:"name"` Description *string `json:"description"` Type *domain.WorkflowType `json:"type"` + Kind *domain.WorkflowKind `json:"kind"` + ScanConfig *domain.ScanConfig `json:"scanConfig"` + Hyperparams *[]domain.HyperParam `json:"hyperparams"` MaxConcurrentRuns *int `json:"maxConcurrentRuns"` NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"` RunTimeoutSeconds *int `json:"runTimeoutSeconds"` @@ -406,6 +1380,241 @@ type workflowPatch struct { Status *domain.WorkflowStatus `json:"status"` } +func scanConfigRequestFromWorkflow(name string, description string, schedule *domain.WorkflowSchedule, scanConfig *domain.ScanConfig) scanConfigRequest { + requestPayload := scanConfigRequest{ + Name: strings.TrimSpace(name), + Description: strings.TrimSpace(description), + } + if schedule != nil { + requestPayload.Interval = schedule.Interval + requestPayload.SourcePath = strings.TrimSpace(schedule.SourcePath) + requestPayload.Enabled = domain.WorkflowScheduleEnabled(schedule) + } + if scanConfig == nil { + return requestPayload + } + requestPayload.DataSource = scanConfig.DataSource + if sourcePath := strings.TrimSpace(scanConfig.SourcePath); sourcePath != "" { + requestPayload.SourcePath = sourcePath + } + requestPayload.ConnectionID = scanConfig.ConnectionID + requestPayload.Bucket = scanConfig.Bucket + requestPayload.Prefix = scanConfig.Prefix + requestPayload.Endpoint = scanConfig.Endpoint + requestPayload.AccessKeyID = scanConfig.AccessKeyID + requestPayload.SecretAccessKey = scanConfig.SecretAccessKey + requestPayload.ScanMode = scanConfig.ScanMode + requestPayload.IncrementalUnit = scanConfig.IncrementalUnit + requestPayload.DirectoryDepth = scanConfig.DirectoryDepth + requestPayload.Pattern = scanConfig.Pattern + requestPayload.Notifications = scanConfig.Notifications + requestPayload.Enabled = scanConfig.Enabled || requestPayload.Enabled + return requestPayload +} + +func workflowHyperparamValue(node *domain.WorkflowNode, key string) string { + if node == nil { + return "" + } + target := strings.TrimSpace(key) + if target == "" { + return "" + } + for _, item := range node.Data.Hyperparams { + if strings.TrimSpace(item.Key) == target { + return strings.TrimSpace(item.Value) + } + } + return "" +} + +func scanConfigRequestFromSourceNode(name string, description string, schedule *domain.WorkflowSchedule, scanConfig *domain.ScanConfig, nodes []domain.WorkflowNode) scanConfigRequest { + requestPayload := scanConfigRequestFromWorkflow(name, description, schedule, scanConfig) + node := domain.FindScanSourceNode(nodes) + if node == nil { + return requestPayload + } + if requestPayload.DataSource == "" { + requestPayload.DataSource = chooseDataSource(workflowHyperparamValue(node, "DATA_SOURCE")) + } + if requestPayload.ConnectionID == "" { + requestPayload.ConnectionID = strings.TrimSpace(node.Data.ConnectionID) + } + if requestPayload.Bucket == "" { + requestPayload.Bucket = strings.TrimSpace(node.Data.Bucket) + } + if requestPayload.Prefix == "" { + requestPayload.Prefix = cleanBOSPrefix(node.Data.Prefix) + } + if requestPayload.SourcePath == "" { + requestPayload.SourcePath = strings.TrimSpace(workflowHyperparamValue(node, "SOURCE_PATH")) + } + if requestPayload.Endpoint == "" { + requestPayload.Endpoint = strings.TrimSpace(workflowHyperparamValue(node, "BOS_ENDPOINT")) + } + if requestPayload.AccessKeyID == "" { + requestPayload.AccessKeyID = strings.TrimSpace(workflowHyperparamValue(node, "BOS_ACCESS_KEY_ID")) + } + if requestPayload.SecretAccessKey == "" { + requestPayload.SecretAccessKey = strings.TrimSpace(workflowHyperparamValue(node, "BOS_SECRET_ACCESS_KEY")) + } + if requestPayload.ScanMode == "" { + requestPayload.ScanMode = chooseWorkflowScanMode(workflowHyperparamValue(node, "SCAN_MODE")) + } + if requestPayload.IncrementalUnit == "" { + requestPayload.IncrementalUnit = chooseWorkflowIncrementalUnit(workflowHyperparamValue(node, "INCREMENTAL_UNIT")) + } + if requestPayload.DirectoryDepth <= 0 { + if raw := strings.TrimSpace(workflowHyperparamValue(node, "INCREMENTAL_DIRECTORY_DEPTH")); raw != "" { + if parsed, err := strconv.Atoi(raw); err == nil { + requestPayload.DirectoryDepth = parsed + } + } + } + if requestPayload.Pattern == "" { + requestPayload.Pattern = strings.TrimSpace(node.Data.Pattern) + } + if requestPayload.Enabled { + requestPayload.Enabled = true + } + return requestPayload +} + +func normalizedScanConfigFromRequest(existing *domain.ScanConfig, requestPayload scanConfigRequest) *domain.ScanConfig { + dataSource := chooseDataSource(requestPayload.DataSource) + sourcePath := scanSourcePath(dataSource, requestPayload.SourcePath, requestPayload.Bucket, requestPayload.Prefix) + connectionID := "" + endpoint := "" + accessKeyID := "" + secretAccessKey := "" + if dataSource == "BOS" { + connectionID = mergeScanConnectionID(existing, requestPayload) + if connectionID == "" { + endpoint = chooseBOSEndpoint(requestPayload.Endpoint) + accessKeyID, secretAccessKey = mergeScanSecrets(existing, requestPayload) + } + } + return &domain.ScanConfig{ + DataSource: dataSource, + SourcePath: sourcePath, + ConnectionID: connectionID, + Bucket: strings.TrimSpace(requestPayload.Bucket), + Prefix: cleanBOSPrefix(requestPayload.Prefix), + Endpoint: endpoint, + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + ScanMode: chooseWorkflowScanMode(requestPayload.ScanMode), + IncrementalUnit: chooseWorkflowIncrementalUnit(requestPayload.IncrementalUnit), + DirectoryDepth: chooseWorkflowDirectoryDepth(requestPayload.DirectoryDepth), + Pattern: strings.TrimSpace(requestPayload.Pattern), + Enabled: requestPayload.Enabled, + Notifications: requestPayload.Notifications, + } +} + +func normalizedScanSchedule(existing *domain.WorkflowSchedule, scanConfig *domain.ScanConfig) *domain.WorkflowSchedule { + next := &domain.WorkflowSchedule{} + if existing != nil { + *next = *existing + } + if scanConfig == nil { + return next + } + next.SourcePath = scanConfig.SourcePath + if scanConfig.Enabled { + if !domain.WorkflowScheduleEnabled(next) { + next.Mode = domain.WorkflowScheduleModeIntervalSeconds + next.Interval = 60 + } + return next + } + if domain.EffectiveWorkflowScheduleMode(next) == domain.WorkflowScheduleModeIntervalSeconds { + next.Interval = 0 + } + return next +} + +func normalizedScanWorkflowType(scanConfig *domain.ScanConfig) domain.WorkflowType { + if scanConfig != nil && scanConfig.Enabled { + return domain.WorkflowTypeStreaming + } + return domain.WorkflowTypeBatch +} + +func normalizedScanMaxConcurrentRuns(value int) int { + if value > 0 { + return value + } + return 1 +} + +func (server *server) validateScanConnectionReference(ctx context.Context, scanConfig *domain.ScanConfig) []validationIssue { + if scanConfig == nil || strings.TrimSpace(scanConfig.ConnectionID) == "" { + return nil + } + if _, err := server.getStorageConnection(ctx, scanConfig.ConnectionID); err != nil { + return []validationIssue{{Field: "scanConfig.connectionId", Message: "connectionId must reference an existing storage connection"}} + } + return nil +} + +func (server *server) validateNodeConnectionReferences(ctx context.Context, nodes []domain.WorkflowNode) []validationIssue { + issues := make([]validationIssue, 0) + for index, node := range nodes { + storageConnectionID := strings.TrimSpace(node.Data.ConnectionID) + if storageConnectionID != "" { + if _, err := server.getStorageConnection(ctx, storageConnectionID); err != nil { + issues = append(issues, validationIssue{ + Field: "nodes[" + strconv.Itoa(index) + "].data.connectionId", + Message: "connectionId must reference an existing storage connection", + }) + } + } + + mongoConnectionID := "" + for _, item := range node.Data.Hyperparams { + if strings.EqualFold(strings.TrimSpace(item.Key), "MONGO_CONNECTION_ID") { + mongoConnectionID = strings.TrimSpace(item.Value) + break + } + } + if mongoConnectionID == "" { + continue + } + if _, err := server.getMongoConnection(ctx, mongoConnectionID); err != nil { + issues = append(issues, validationIssue{ + Field: "nodes[" + strconv.Itoa(index) + "].data.hyperparams", + Message: "MONGO_CONNECTION_ID must reference an existing mongo connection", + }) + } + } + return issues +} + +func validateInputPortConnectionTopology(nodes []domain.WorkflowNode, edges []domain.WorkflowEdge) []validationIssue { + conflicts := domain.FindInputPortConflicts(nodes, edges) + if len(conflicts) == 0 { + return nil + } + issues := make([]validationIssue, 0, len(conflicts)) + for _, conflict := range conflicts { + nodeLabel := strings.TrimSpace(conflict.TargetNodeLabel) + if nodeLabel == "" { + nodeLabel = conflict.TargetNodeID + } + portName := strings.TrimSpace(conflict.TargetPortName) + if portName == "" { + portName = "默认输入" + } + issues = append(issues, validationIssue{ + Field: "edges", + Message: "节点“" + nodeLabel + "”的输入端口“" + portName + "”连接了多条上游连线。" + + "每个输入端口只能连接一条上游连线;如需上传多个结果,请为每个结果添加单独的上传算子。", + }) + } + return issues +} + func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request *http.Request) { item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) if err != nil { @@ -430,6 +1639,12 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request if payload.Type != nil { item.Type = *payload.Type } + if payload.Kind != nil { + item.Kind = *payload.Kind + } + if payload.Hyperparams != nil { + item.Hyperparams = *payload.Hyperparams + } if payload.MaxConcurrentRuns != nil { if issues := validateWorkflowExecutionConfig(*payload.MaxConcurrentRuns, 0, 0, 0, 0); len(issues) > 0 { renderValidationError(writer, issues...) @@ -466,7 +1681,11 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request item.RetryBackoffSeconds = *payload.RetryBackoffSeconds } if payload.Schedule != nil { - item.Schedule = payload.Schedule + item.Schedule = normalizeWorkflowSchedule(item.Schedule, payload.Schedule, nowISO()) + if issues := validateWorkflowSchedule(item.Schedule); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } } if payload.Nodes != nil { item.Nodes = *payload.Nodes @@ -474,6 +1693,39 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request if payload.Edges != nil { item.Edges = *payload.Edges } + if item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil || payload.ScanConfig != nil || domain.FindScanSourceNode(item.Nodes) != nil { + existingScanConfig := item.ScanConfig + effectiveScanConfig := item.ScanConfig + if payload.ScanConfig != nil { + effectiveScanConfig = payload.ScanConfig + } + requestPayload := scanConfigRequestFromSourceNode(item.Name, item.Description, item.Schedule, effectiveScanConfig, item.Nodes) + if issues := validateScanConfigDraftPayload(requestPayload); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if requestPayload.Interval > 0 && !scanConfigPayloadReady(requestPayload, existingScanConfig) { + renderValidationError(writer, validateScanConfigPayload(requestPayload, existingScanConfig)...) + return + } + item.Kind = domain.WorkflowKindScan + item.ScanConfig = normalizedScanConfigFromRequest(existingScanConfig, requestPayload) + if issues := server.validateScanConnectionReference(request.Context(), item.ScanConfig); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.Schedule = normalizedScanSchedule(item.Schedule, item.ScanConfig) + item.Type = normalizedScanWorkflowType(item.ScanConfig) + item.MaxConcurrentRuns = normalizedScanMaxConcurrentRuns(item.MaxConcurrentRuns) + } + if issues := server.validateNodeConnectionReferences(request.Context(), item.Nodes); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if issues := validateInputPortConnectionTopology(item.Nodes, item.Edges); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } if payload.Status != nil { item.Status = *payload.Status } @@ -482,7 +1734,7 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeWorkflow(item)) } func (server *server) handleWorkflowsDelete(writer http.ResponseWriter, request *http.Request) { @@ -506,6 +1758,17 @@ func validRunTrigger(value domain.RunTrigger) bool { return value == domain.RunTriggerManual || value == domain.RunTriggerScheduled || value == domain.RunTriggerRetry } +type rerunSubtreeRequest struct { + NodeID string `json:"nodeId"` +} + +func nextRunAttemptForChild(run *domain.RunRecord) int { + if run == nil || run.Attempt <= 0 { + return 2 + } + return run.Attempt + 1 +} + func (server *server) handleWorkflowRun(writer http.ResponseWriter, request *http.Request) { payload := runRequest{Trigger: domain.RunTriggerManual} if request.ContentLength != 0 { @@ -517,6 +1780,30 @@ func (server *server) handleWorkflowRun(writer http.ResponseWriter, request *htt renderValidationError(writer, validationIssue{Field: "trigger", Message: "trigger must be manual, scheduled, or retry"}) return } + workflow, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if workflow == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"}) + return + } + if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil { + requestPayload := scanConfigRequestFromWorkflow(workflow.Name, workflow.Description, workflow.Schedule, workflow.ScanConfig) + if !scanConfigPayloadReady(requestPayload, workflow.ScanConfig) { + renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "扫描源配置未完成,请先在扫描源节点配置数据源、目标路径和访问密钥"}) + return + } + if issues := server.validateScanConnectionReference(request.Context(), workflow.ScanConfig); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + } + if issues := validateInputPortConnectionTopology(workflow.Nodes, workflow.Edges); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } run, err := server.store.CreateRun(request.Context(), chi.URLParam(request, "id"), payload.Trigger) if err != nil { if strings.Contains(err.Error(), "not found") { @@ -533,13 +1820,131 @@ func (server *server) handleWorkflowRun(writer http.ResponseWriter, request *htt renderJSON(writer, http.StatusAccepted, run) } +func (server *server) handleWorkflowRunsList(writer http.ResponseWriter, request *http.Request) { + workflowID := chi.URLParam(request, "id") + workflow, err := server.store.GetWorkflow(request.Context(), workflowID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if workflow == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"}) + return + } + items, err := server.store.ListRuns(request.Context(), "") + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + summaries := make([]domain.RunSummary, 0, len(items)) + for _, item := range items { + if item.WorkflowID != workflowID { + continue + } + summaries = append(summaries, runSummaryFromRecord(item)) + } + renderJSON(writer, http.StatusOK, map[string]any{"items": summaries}) +} + +func (server *server) handleRunsRerunSubtree(writer http.ResponseWriter, request *http.Request) { + sourceRun, err := server.loadRun(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if sourceRun == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) + return + } + if sourceRun.Status != domain.RunStatusFailed { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "只有失败的运行才支持从指定节点重新执行下游"}) + return + } + var payload rerunSubtreeRequest + if !decodeJSON(writer, request, &payload) { + return + } + payload.NodeID = strings.TrimSpace(payload.NodeID) + if payload.NodeID == "" { + renderValidationError(writer, validationIssue{Field: "nodeId", Message: "nodeId is required"}) + return + } + workflow, err := server.store.GetWorkflow(request.Context(), sourceRun.WorkflowID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if workflow == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"}) + return + } + if workflowNodeByID(workflow, payload.NodeID) == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Node not found"}) + return + } + + subtree := rerunSubtreeNodeSet(workflow.Edges, payload.NodeID) + baselineSet := rerunBaselineNodeSet(workflow.Edges, subtree) + sourceExecutionMap := make(map[string]domain.NodeExecutionRecord, len(sourceRun.NodeExecutions)) + for _, execution := range sourceRun.NodeExecutions { + sourceExecutionMap[execution.NodeID] = execution + } + baselineExecutions := make([]domain.NodeExecutionRecord, 0, len(baselineSet)) + for _, execution := range sourceRun.NodeExecutions { + if _, ok := baselineSet[execution.NodeID]; !ok { + continue + } + if execution.Status != domain.NodeExecutionStatusCompleted { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "选定节点缺少可复用的上游成功基线,无法从这里继续向后执行"}) + return + } + baselineExecutions = append(baselineExecutions, cloneNodeExecutionRecord(execution)) + } + for nodeID := range baselineSet { + execution, ok := sourceExecutionMap[nodeID] + if !ok || execution.Status != domain.NodeExecutionStatusCompleted { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "选定节点缺少可复用的上游成功基线,无法从这里继续向后执行"}) + return + } + } + + childRun, err := server.store.CreateRun(request.Context(), workflow.ID, domain.RunTriggerRerunSubtree) + if err != nil { + if strings.Contains(err.Error(), "not found") { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()}) + return + } + renderError(writer, http.StatusInternalServerError, err) + return + } + childRun.ParentRunID = sourceRun.ID + childRun.RestartFromNodeID = payload.NodeID + childRun.Attempt = nextRunAttemptForChild(sourceRun) + childRun.NodeExecutions = baselineExecutions + childRun.K8sRefs = []domain.K8sRef{} + childRun.Error = "" + childRun.CurrentNodeID = "" + childRun.StartedAt = "" + childRun.EndedAt = "" + childRun.DurationMS = 0 + childRun.Status = domain.RunStatusQueued + childRun.AvailableAt = nowISO() + childRun.UpdatedAt = nowISO() + rebuildRunAggregates(childRun) + if err := server.store.UpsertRun(request.Context(), *childRun); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusAccepted, childRun) +} + func (server *server) handleTemplatesList(writer http.ResponseWriter, request *http.Request) { items, err := server.store.ListTemplates(request.Context()) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, map[string]any{"items": items}) + renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeTemplates(items)}) } func (server *server) handleTemplatesCreate(writer http.ResponseWriter, request *http.Request) { @@ -560,16 +1965,42 @@ func (server *server) handleTemplatesCreate(writer http.ResponseWriter, request if configIssues := validateWorkflowExecutionConfig(payload.MaxConcurrentRuns, payload.NodeTimeoutSeconds, payload.RunTimeoutSeconds, payload.MaxRetryAttempts, payload.RetryBackoffSeconds); len(configIssues) > 0 { issues = append(issues, configIssues...) } + payload.Schedule = normalizeWorkflowSchedule(nil, payload.Schedule, nowISO()) + issues = append(issues, validateWorkflowSchedule(payload.Schedule)...) if len(issues) > 0 { renderValidationError(writer, issues...) return } + if payload.Kind == domain.WorkflowKindScan || payload.ScanConfig != nil || domain.FindScanSourceNode(payload.Nodes) != nil { + requestPayload := scanConfigRequestFromSourceNode(payload.Name, payload.Description, payload.Schedule, payload.ScanConfig, payload.Nodes) + if issues := validateScanConfigDraftPayload(requestPayload); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if requestPayload.Interval > 0 && !scanConfigPayloadReady(requestPayload, nil) { + renderValidationError(writer, validateScanConfigPayload(requestPayload, nil)...) + return + } + payload.Kind = domain.WorkflowKindScan + payload.ScanConfig = normalizedScanConfigFromRequest(nil, requestPayload) + if issues := server.validateScanConnectionReference(request.Context(), payload.ScanConfig); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + payload.Schedule = normalizedScanSchedule(payload.Schedule, payload.ScanConfig) + payload.Type = normalizedScanWorkflowType(payload.ScanConfig) + payload.MaxConcurrentRuns = normalizedScanMaxConcurrentRuns(payload.MaxConcurrentRuns) + } + if issues := validateInputPortConnectionTopology(payload.Nodes, payload.Edges); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } item, err := server.store.CreateTemplate(request.Context(), payload) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusCreated, item) + renderJSON(writer, http.StatusCreated, sanitizeTemplate(item)) } func (server *server) handleTemplatesGet(writer http.ResponseWriter, request *http.Request) { @@ -582,13 +2013,16 @@ func (server *server) handleTemplatesGet(writer http.ResponseWriter, request *ht renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Template not found"}) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeTemplate(item)) } type templatePatch struct { Name *string `json:"name"` Description *string `json:"description"` Type *domain.WorkflowType `json:"type"` + Kind *domain.WorkflowKind `json:"kind"` + ScanConfig *domain.ScanConfig `json:"scanConfig"` + Hyperparams *[]domain.HyperParam `json:"hyperparams"` MaxConcurrentRuns *int `json:"maxConcurrentRuns"` NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"` RunTimeoutSeconds *int `json:"runTimeoutSeconds"` @@ -626,6 +2060,12 @@ func (server *server) handleTemplatesUpdate(writer http.ResponseWriter, request } item.Type = *payload.Type } + if payload.Kind != nil { + item.Kind = *payload.Kind + } + if payload.Hyperparams != nil { + item.Hyperparams = *payload.Hyperparams + } if payload.MaxConcurrentRuns != nil { if issues := validateWorkflowExecutionConfig(*payload.MaxConcurrentRuns, 0, 0, 0, 0); len(issues) > 0 { renderValidationError(writer, issues...) @@ -662,7 +2102,11 @@ func (server *server) handleTemplatesUpdate(writer http.ResponseWriter, request item.RetryBackoffSeconds = *payload.RetryBackoffSeconds } if payload.Schedule != nil { - item.Schedule = payload.Schedule + item.Schedule = normalizeWorkflowSchedule(item.Schedule, payload.Schedule, nowISO()) + if issues := validateWorkflowSchedule(item.Schedule); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } } if payload.Nodes != nil { item.Nodes = *payload.Nodes @@ -670,12 +2114,41 @@ func (server *server) handleTemplatesUpdate(writer http.ResponseWriter, request if payload.Edges != nil { item.Edges = *payload.Edges } + if item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil || payload.ScanConfig != nil || domain.FindScanSourceNode(item.Nodes) != nil { + existingScanConfig := item.ScanConfig + effectiveScanConfig := item.ScanConfig + if payload.ScanConfig != nil { + effectiveScanConfig = payload.ScanConfig + } + requestPayload := scanConfigRequestFromSourceNode(item.Name, item.Description, item.Schedule, effectiveScanConfig, item.Nodes) + if issues := validateScanConfigDraftPayload(requestPayload); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if requestPayload.Interval > 0 && !scanConfigPayloadReady(requestPayload, existingScanConfig) { + renderValidationError(writer, validateScanConfigPayload(requestPayload, existingScanConfig)...) + return + } + item.Kind = domain.WorkflowKindScan + item.ScanConfig = normalizedScanConfigFromRequest(existingScanConfig, requestPayload) + if issues := server.validateScanConnectionReference(request.Context(), item.ScanConfig); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.Schedule = normalizedScanSchedule(item.Schedule, item.ScanConfig) + item.Type = normalizedScanWorkflowType(item.ScanConfig) + item.MaxConcurrentRuns = normalizedScanMaxConcurrentRuns(item.MaxConcurrentRuns) + } + if issues := validateInputPortConnectionTopology(item.Nodes, item.Edges); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } item.UpdatedAt = nowISO() if err := server.store.UpsertTemplate(request.Context(), *item); err != nil { renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeTemplate(item)) } func (server *server) handleTemplatesDelete(writer http.ResponseWriter, request *http.Request) { @@ -692,12 +2165,12 @@ func (server *server) handleTemplatesDelete(writer http.ResponseWriter, request } func (server *server) handleOperatorsList(writer http.ResponseWriter, request *http.Request) { - items, err := server.store.ListOperators(request.Context()) + items, err := server.listAvailableOperators(request.Context()) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, map[string]any{"items": items}) + renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeOperators(items)}) } func (server *server) handleOperatorsCreate(writer http.ResponseWriter, request *http.Request) { @@ -718,7 +2191,7 @@ func (server *server) handleOperatorsCreate(writer http.ResponseWriter, request renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusCreated, item) + renderJSON(writer, http.StatusCreated, sanitizeOperator(item)) } func (server *server) handleOperatorsGet(writer http.ResponseWriter, request *http.Request) { @@ -731,7 +2204,7 @@ func (server *server) handleOperatorsGet(writer http.ResponseWriter, request *ht renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Operator not found"}) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeOperator(item)) } type operatorPatch struct { @@ -800,11 +2273,32 @@ func (server *server) handleOperatorsUpdate(writer http.ResponseWriter, request renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeOperator(item)) } func (server *server) handleOperatorsDelete(writer http.ResponseWriter, request *http.Request) { - deleted, err := server.store.DeleteOperator(request.Context(), chi.URLParam(request, "id")) + id := chi.URLParam(request, "id") + item, err := server.store.GetOperator(request.Context(), id) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + writer.WriteHeader(http.StatusNotFound) + return + } + if strings.EqualFold(strings.TrimSpace(item.Type), "builtin") { + if err := server.store.UpsertBuiltinOperatorDeletion(request.Context(), domain.BuiltinOperatorDeletionRecord{ + Key: builtinOperatorDeletionKey(item.ID, item.Name), + OperatorID: item.ID, + OperatorName: item.Name, + DeletedAt: nowISO(), + }); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + } + deleted, err := server.store.DeleteOperator(request.Context(), id) if err != nil { renderError(writer, http.StatusInternalServerError, err) return @@ -817,7 +2311,7 @@ func (server *server) handleOperatorsDelete(writer http.ResponseWriter, request } func (server *server) handleRunsList(writer http.ResponseWriter, request *http.Request) { - items, err := server.store.ListRuns(request.Context(), request.URL.Query().Get("status")) + items, err := server.store.ListRunSummaries(request.Context(), request.URL.Query().Get("status")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return @@ -846,6 +2340,81 @@ func (server *server) handleRunsGet(writer http.ResponseWriter, request *http.Re renderJSON(writer, http.StatusOK, item) } +func (server *server) markWorkflowStopped(ctx context.Context, run *domain.RunRecord) error { + if run == nil || strings.TrimSpace(run.WorkflowID) == "" { + return nil + } + workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID) + if err != nil || workflow == nil { + return err + } + workflow.Status = domain.WorkflowStatusFailed + workflow.LastRunID = run.ID + workflow.LastRunAt = nowISO() + workflow.UpdatedAt = nowISO() + return server.store.UpsertWorkflow(ctx, *workflow) +} + +func (server *server) terminateRunJobs(ctx context.Context, run *domain.RunRecord) error { + if run == nil || strings.TrimSpace(run.ID) == "" { + return nil + } + settings, err := server.store.GetSettings(ctx) + if err != nil || settings == nil { + return err + } + launcher, err := runJobTerminatorFactory(settings.K8s.Namespace, "", settings.K8s.WorkspacePVCName, settings.K8s.WorkspaceMountPath) + if err != nil { + return err + } + if err := launcher.TerminateRun(ctx, settings.K8s.Namespace, run.ID); err != nil { + return err + } + for _, ref := range run.K8sRefs { + if err := launcher.TerminateJob(ctx, ref.Namespace, ref.JobName); err != nil { + return err + } + } + return nil +} + +func (server *server) handleRunStopWithMode(writer http.ResponseWriter, request *http.Request, mode domain.RunStopMode) { + run, err := server.store.RequestRunStop(request.Context(), chi.URLParam(request, "id"), mode) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if run == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) + return + } + if mode == domain.RunStopModeForce && run.Status == domain.RunStatusCancelled { + if err := server.terminateRunJobs(request.Context(), run); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if err := server.markWorkflowStopped(request.Context(), run); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + } + if mode == domain.RunStopModeGraceful && run.Status == domain.RunStatusCancelled { + if err := server.markWorkflowStopped(request.Context(), run); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + } + renderJSON(writer, http.StatusOK, run) +} + +func (server *server) handleRunsStop(writer http.ResponseWriter, request *http.Request) { + server.handleRunStopWithMode(writer, request, domain.RunStopModeGraceful) +} + +func (server *server) handleRunsForceStop(writer http.ResponseWriter, request *http.Request) { + server.handleRunStopWithMode(writer, request, domain.RunStopModeForce) +} + func (server *server) handleRunsLogs(writer http.ResponseWriter, request *http.Request) { item, err := server.loadRun(request.Context(), chi.URLParam(request, "id")) if err != nil { @@ -950,7 +2519,7 @@ func (server *server) handleUsersList(writer http.ResponseWriter, request *http. renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, map[string]any{"items": items}) + renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeUsers(items)}) } func normalizeUserInput(name string, email string) (string, string) { @@ -982,7 +2551,7 @@ func (server *server) handleUsersCreate(writer http.ResponseWriter, request *htt renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusCreated, item) + renderJSON(writer, http.StatusCreated, sanitizeUser(item)) } type userPatch struct { @@ -1032,7 +2601,7 @@ func (server *server) handleUsersUpdate(writer http.ResponseWriter, request *htt renderError(writer, http.StatusInternalServerError, err) return } - renderJSON(writer, http.StatusOK, item) + renderJSON(writer, http.StatusOK, sanitizeUser(item)) } func (server *server) handleUsersDelete(writer http.ResponseWriter, request *http.Request) { @@ -1048,6 +2617,332 @@ func (server *server) handleUsersDelete(writer http.ResponseWriter, request *htt writer.WriteHeader(http.StatusNoContent) } +type storagePathFavoritesPayload struct { + Items []domain.StoragePathFavoriteRecord `json:"items"` +} + +func (server *server) handleUserStoragePathFavoritesGet(writer http.ResponseWriter, request *http.Request) { + user, err := server.store.GetUser(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if user == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "User not found"}) + return + } + renderJSON(writer, http.StatusOK, map[string]any{ + "items": sanitizeStoragePathFavorites(user.StoragePathFavorites), + }) +} + +func (server *server) handleUserStoragePathFavoritesUpdate(writer http.ResponseWriter, request *http.Request) { + user, err := server.store.GetUser(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if user == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "User not found"}) + return + } + var payload storagePathFavoritesPayload + if !decodeJSON(writer, request, &payload) { + return + } + user.StoragePathFavorites = sanitizeStoragePathFavorites(payload.Items) + user.UpdatedAt = nowISO() + if err := server.store.UpsertUser(request.Context(), *user); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{ + "items": user.StoragePathFavorites, + }) +} + +type storageConnectionPatch struct { + Name *string `json:"name"` + Description *string `json:"description"` + Provider *domain.StorageProvider `json:"provider"` + Endpoint *string `json:"endpoint"` + Bucket *string `json:"bucket"` + AccessKeyID *string `json:"accessKeyId"` + SecretAccessKey *string `json:"secretAccessKey"` +} + +type mongoConnectionPatch struct { + Name *string `json:"name"` + Description *string `json:"description"` + URI *string `json:"uri"` + Database *string `json:"database"` +} + +func normalizeStorageProvider(value domain.StorageProvider) domain.StorageProvider { + switch domain.StorageProvider(strings.ToUpper(strings.TrimSpace(string(value)))) { + case domain.StorageProviderBOS: + return domain.StorageProviderBOS + default: + return "" + } +} + +func validStorageProvider(value domain.StorageProvider) bool { + return normalizeStorageProvider(value) != "" +} + +func storageEndpointForProvider(provider domain.StorageProvider, endpoint string) string { + switch normalizeStorageProvider(provider) { + case domain.StorageProviderBOS: + return chooseBOSEndpoint(endpoint) + default: + return strings.TrimSpace(endpoint) + } +} + +func (server *server) handleStorageConnectionsList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListStorageConnections(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeStorageConnections(items)}) +} + +func (server *server) handleStorageConnectionsCreate(writer http.ResponseWriter, request *http.Request) { + var payload domain.CreateStorageConnectionInput + if !decodeJSON(writer, request, &payload) { + return + } + payload.Name = strings.TrimSpace(payload.Name) + payload.Description = strings.TrimSpace(payload.Description) + payload.Provider = normalizeStorageProvider(payload.Provider) + payload.Endpoint = strings.TrimSpace(payload.Endpoint) + payload.Bucket = strings.Trim(strings.TrimSpace(payload.Bucket), "/") + payload.AccessKeyID = strings.TrimSpace(payload.AccessKeyID) + payload.SecretAccessKey = strings.TrimSpace(payload.SecretAccessKey) + if payload.Name == "" { + renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"}) + return + } + if !validStorageProvider(payload.Provider) { + renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS"}) + return + } + if payload.AccessKeyID == "" { + renderValidationError(writer, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required"}) + return + } + if payload.SecretAccessKey == "" { + renderValidationError(writer, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required"}) + return + } + payload.Endpoint = storageEndpointForProvider(payload.Provider, payload.Endpoint) + item, err := server.store.CreateStorageConnection(request.Context(), payload) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, sanitizeStorageConnection(item)) +} + +func (server *server) handleStorageConnectionsGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetStorageConnection(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Storage connection not found"}) + return + } + renderJSON(writer, http.StatusOK, sanitizeStorageConnection(item)) +} + +func (server *server) handleStorageConnectionsUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetStorageConnection(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Storage connection not found"}) + return + } + var payload storageConnectionPatch + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Name != nil { + item.Name = strings.TrimSpace(*payload.Name) + } + if payload.Description != nil { + item.Description = strings.TrimSpace(*payload.Description) + } + if payload.Provider != nil { + provider := normalizeStorageProvider(*payload.Provider) + if !validStorageProvider(provider) { + renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS"}) + return + } + item.Provider = provider + } + if payload.Endpoint != nil { + item.Endpoint = storageEndpointForProvider(item.Provider, *payload.Endpoint) + } + if payload.Bucket != nil { + item.Bucket = strings.Trim(strings.TrimSpace(*payload.Bucket), "/") + } + if payload.AccessKeyID != nil { + if trimmed := strings.TrimSpace(*payload.AccessKeyID); trimmed != "" { + item.AccessKeyID = trimmed + } + } + if payload.SecretAccessKey != nil { + if trimmed := strings.TrimSpace(*payload.SecretAccessKey); trimmed != "" { + item.SecretAccessKey = trimmed + } + } + if strings.TrimSpace(item.Name) == "" { + renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"}) + return + } + item.Endpoint = storageEndpointForProvider(item.Provider, item.Endpoint) + item.UpdatedAt = nowISO() + if err := server.store.UpsertStorageConnection(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, sanitizeStorageConnection(item)) +} + +func (server *server) handleStorageConnectionsDelete(writer http.ResponseWriter, request *http.Request) { + deleted, err := server.store.DeleteStorageConnection(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + writer.WriteHeader(http.StatusNotFound) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (server *server) handleMongoConnectionsList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListMongoConnections(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeMongoConnections(items)}) +} + +func (server *server) handleMongoConnectionsCreate(writer http.ResponseWriter, request *http.Request) { + var payload domain.CreateMongoConnectionInput + if !decodeJSON(writer, request, &payload) { + return + } + payload.Name = strings.TrimSpace(payload.Name) + payload.Description = strings.TrimSpace(payload.Description) + payload.URI = strings.TrimSpace(payload.URI) + payload.Database = strings.TrimSpace(payload.Database) + if payload.Name == "" { + renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"}) + return + } + if payload.URI == "" { + renderValidationError(writer, validationIssue{Field: "uri", Message: "uri is required"}) + return + } + if payload.Database == "" { + renderValidationError(writer, validationIssue{Field: "database", Message: "database is required"}) + return + } + item, err := server.store.CreateMongoConnection(request.Context(), payload) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, sanitizeMongoConnection(item)) +} + +func (server *server) handleMongoConnectionsGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetMongoConnection(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Mongo connection not found"}) + return + } + renderJSON(writer, http.StatusOK, sanitizeMongoConnection(item)) +} + +func (server *server) handleMongoConnectionsUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetMongoConnection(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Mongo connection not found"}) + return + } + var payload mongoConnectionPatch + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Name != nil { + item.Name = strings.TrimSpace(*payload.Name) + } + if payload.Description != nil { + item.Description = strings.TrimSpace(*payload.Description) + } + if payload.URI != nil { + if trimmed := strings.TrimSpace(*payload.URI); trimmed != "" { + item.URI = trimmed + } + } + if payload.Database != nil { + if trimmed := strings.TrimSpace(*payload.Database); trimmed != "" { + item.Database = trimmed + } + } + if strings.TrimSpace(item.Name) == "" { + renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"}) + return + } + if strings.TrimSpace(item.URI) == "" { + renderValidationError(writer, validationIssue{Field: "uri", Message: "uri is required"}) + return + } + if strings.TrimSpace(item.Database) == "" { + renderValidationError(writer, validationIssue{Field: "database", Message: "database is required"}) + return + } + item.UpdatedAt = nowISO() + if err := server.store.UpsertMongoConnection(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, sanitizeMongoConnection(item)) +} + +func (server *server) handleMongoConnectionsDelete(writer http.ResponseWriter, request *http.Request) { + deleted, err := server.store.DeleteMongoConnection(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + writer.WriteHeader(http.StatusNotFound) + return + } + writer.WriteHeader(http.StatusNoContent) +} + func (server *server) handleSettingsGet(writer http.ResponseWriter, request *http.Request) { item, err := server.store.GetSettings(request.Context()) if err != nil { @@ -1059,10 +2954,11 @@ func (server *server) handleSettingsGet(writer http.ResponseWriter, request *htt type settingsPatch struct { K8s *struct { - Enabled *bool `json:"enabled"` - Endpoint *string `json:"endpoint"` - Namespace *string `json:"namespace"` - Kubeconfig *string `json:"kubeconfig"` + Endpoint *string `json:"endpoint"` + Namespace *string `json:"namespace"` + WorkspacePVCName *string `json:"workspacePVCName"` + WorkspaceMountPath *string `json:"workspaceMountPath"` + Kubeconfig *string `json:"kubeconfig"` } `json:"k8s"` Worker *struct { PollIntervalMS *int `json:"pollIntervalMs"` @@ -1074,6 +2970,10 @@ type settingsPatch struct { Model *string `json:"model"` APIKey *string `json:"apiKey"` } `json:"ai"` + BusinessMongo *struct { + URI *string `json:"uri"` + Database *string `json:"database"` + } `json:"businessMongo"` Defaults *struct { CPURequest *string `json:"cpuRequest"` CPULimit *string `json:"cpuLimit"` @@ -1106,19 +3006,33 @@ func (server *server) handleSettingsUpdate(writer http.ResponseWriter, request * item.Timezone = timezone } if payload.K8s != nil { - if payload.K8s.Enabled != nil { - item.K8s.Enabled = *payload.K8s.Enabled - } if payload.K8s.Endpoint != nil { item.K8s.Endpoint = *payload.K8s.Endpoint } if payload.K8s.Namespace != nil { item.K8s.Namespace = *payload.K8s.Namespace } + if payload.K8s.WorkspacePVCName != nil { + item.K8s.WorkspacePVCName = strings.TrimSpace(*payload.K8s.WorkspacePVCName) + } + if payload.K8s.WorkspaceMountPath != nil { + item.K8s.WorkspaceMountPath = strings.TrimSpace(*payload.K8s.WorkspaceMountPath) + } if payload.K8s.Kubeconfig != nil { item.K8s.Kubeconfig = strings.TrimSpace(*payload.K8s.Kubeconfig) } } + if strings.TrimSpace(item.K8s.WorkspacePVCName) == "" { + renderValidationError(writer, validationIssue{Field: "k8s.workspacePVCName", Message: "workspacePVCName is required"}) + return + } + if strings.TrimSpace(item.K8s.WorkspaceMountPath) == "" { + item.K8s.WorkspaceMountPath = "/workspace" + } + if !strings.HasPrefix(item.K8s.WorkspaceMountPath, "/") { + renderValidationError(writer, validationIssue{Field: "k8s.workspaceMountPath", Message: "workspaceMountPath must be an absolute path"}) + return + } if payload.Worker != nil { if payload.Worker.PollIntervalMS != nil && *payload.Worker.PollIntervalMS > 0 { item.Worker.PollIntervalMS = *payload.Worker.PollIntervalMS @@ -1141,6 +3055,14 @@ func (server *server) handleSettingsUpdate(writer http.ResponseWriter, request * item.AI.APIKey = strings.TrimSpace(*payload.AI.APIKey) } } + if payload.BusinessMongo != nil { + if payload.BusinessMongo.URI != nil { + item.BusinessMongo.URI = strings.TrimSpace(*payload.BusinessMongo.URI) + } + if payload.BusinessMongo.Database != nil { + item.BusinessMongo.Database = strings.TrimSpace(*payload.BusinessMongo.Database) + } + } if payload.Defaults != nil { if payload.Defaults.CPURequest != nil { item.Defaults.CPURequest = *payload.Defaults.CPURequest @@ -1155,6 +3077,10 @@ func (server *server) handleSettingsUpdate(writer http.ResponseWriter, request * item.Defaults.MemoryLimit = *payload.Defaults.MemoryLimit } } + if strings.TrimSpace(item.BusinessMongo.URI) != "" && strings.TrimSpace(item.BusinessMongo.Database) == "" { + renderValidationError(writer, validationIssue{Field: "businessMongo.database", Message: "database is required when businessMongo.uri is configured"}) + return + } item.UpdatedAt = nowISO() if err := server.store.UpsertSettings(request.Context(), *item); err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -1182,13 +3108,14 @@ func sanitizeSettings(item *domain.SettingsRecord) *domain.SettingsView { return nil } view := &domain.SettingsView{ - ID: item.ID, - K8s: item.K8s, - Worker: item.Worker, - AI: item.AI, - Defaults: item.Defaults, - Timezone: item.Timezone, - UpdatedAt: item.UpdatedAt, + ID: item.ID, + K8s: item.K8s, + Worker: item.Worker, + AI: item.AI, + BusinessMongo: item.BusinessMongo, + Defaults: item.Defaults, + Timezone: item.Timezone, + UpdatedAt: item.UpdatedAt, } view.K8s.HasKubeconfig = strings.TrimSpace(view.K8s.Kubeconfig) != "" view.K8s.KubeconfigPreview = maskedSecretPreview(view.K8s.Kubeconfig) @@ -1196,13 +3123,35 @@ func sanitizeSettings(item *domain.SettingsRecord) *domain.SettingsView { view.AI.HasAPIKey = strings.TrimSpace(view.AI.APIKey) != "" view.AI.APIKeyPreview = maskedSecretPreview(view.AI.APIKey) view.AI.APIKey = "" + view.BusinessMongo.HasURI = strings.TrimSpace(view.BusinessMongo.URI) != "" + view.BusinessMongo.URIPreview = maskedSecretPreview(view.BusinessMongo.URI) + view.BusinessMongo.URI = "" return view } func (server *server) handleStorageBuckets(writer http.ResponseWriter, request *http.Request) { + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + if connectionID != "" { + connection, err := server.store.GetStorageConnection(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if connection == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Storage connection not found"}) + return + } + items, err := storageConnectionBucketsResolver(request.Context(), connection) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) + return + } sourcePath := strings.TrimSpace(request.URL.Query().Get("sourcePath")) if sourcePath == "" { - renderValidationError(writer, validationIssue{Field: "sourcePath", Message: "sourcePath is required"}) + renderValidationError(writer, validationIssue{Field: "sourcePath", Message: "sourcePath or connectionId is required"}) return } items, err := server.store.ListStorageBuckets(request.Context(), sourcePath) @@ -1214,9 +3163,33 @@ func (server *server) handleStorageBuckets(writer http.ResponseWriter, request * } func (server *server) handleStorageObjects(writer http.ResponseWriter, request *http.Request) { + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + if connectionID != "" { + connection, err := server.store.GetStorageConnection(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if connection == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Storage connection not found"}) + return + } + items, err := storageConnectionObjectsResolver( + request.Context(), + connection, + request.URL.Query().Get("bucket"), + request.URL.Query().Get("prefix"), + ) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) + return + } sourcePath := strings.TrimSpace(request.URL.Query().Get("sourcePath")) if sourcePath == "" { - renderValidationError(writer, validationIssue{Field: "sourcePath", Message: "sourcePath is required"}) + renderValidationError(writer, validationIssue{Field: "sourcePath", Message: "sourcePath or connectionId is required"}) return } items, err := server.store.ListStorageObjects( diff --git a/internal/api/server_test.go b/internal/api/server_test.go index d0eb2188..79030030 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -4,16 +4,156 @@ import ( "bytes" "context" "encoding/json" + "fmt" + "io" + "mime/multipart" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" + "sort" + "strings" "testing" "droboticflow/internal/domain" "droboticflow/internal/store" + + bosapi "github.com/baidubce/bce-sdk-go/services/bos/api" ) +func TestListStorageObjectsFromConnectionNormalizesDirectoryPrefixes(t *testing.T) { + tests := []struct { + name string + prefix string + expected string + }{ + {name: "empty", prefix: "", expected: ""}, + {name: "trimmed root child", prefix: "algo", expected: "algo/"}, + {name: "preserve nested directory", prefix: "algo/X-platform-test", expected: "algo/X-platform-test/"}, + {name: "trim slashes", prefix: "/algo/X-platform-test//", expected: "algo/X-platform-test/"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + requestPrefix := storageObjectRequestPrefix(test.prefix) + if requestPrefix != test.expected { + t.Fatalf("request prefix = %q, want %q", requestPrefix, test.expected) + } + }) + } +} + +func TestAppendStorageObjectItemsSkipsCurrentPrefixAndDedupesPages(t *testing.T) { + collected := make(map[string]struct{}) + appendStorageObjectItems( + collected, + &bosapi.ListObjectsResult{ + CommonPrefixes: []bosapi.PrefixType{ + {Prefix: "algo/"}, + {Prefix: "algo/X-platform-test/"}, + }, + Contents: []bosapi.ObjectSummaryType{ + {Key: "algo/"}, + {Key: "algo/manifest.json"}, + }, + }, + "algo", + "algo/", + ) + appendStorageObjectItems( + collected, + &bosapi.ListObjectsResult{ + CommonPrefixes: []bosapi.PrefixType{ + {Prefix: "algo/X-platform-test/"}, + {Prefix: "algo/raw/"}, + }, + Contents: []bosapi.ObjectSummaryType{ + {Key: "algo/manifest.json"}, + {Key: "algo/video.mp4"}, + }, + }, + "algo", + "algo/", + ) + + items := make([]string, 0, len(collected)) + for item := range collected { + items = append(items, item) + } + sort.Strings(items) + + expected := []string{ + "algo/X-platform-test/", + "algo/manifest.json", + "algo/raw/", + "algo/video.mp4", + } + if strings.Join(items, "|") != strings.Join(expected, "|") { + t.Fatalf("items = %#v, want %#v", items, expected) + } +} + +func TestWorkerAPIHealthIsProxiedThroughAPI(t *testing.T) { + worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/worker-api/health" { + t.Fatalf("unexpected upstream path: %s", request.URL.Path) + } + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"status":"ok","service":"droboticflow-worker"}`)) + })) + defer worker.Close() + + originalResolver := workerAPIBaseURLResolver + workerAPIBaseURLResolver = func() string { return worker.URL } + t.Cleanup(func() { + workerAPIBaseURLResolver = originalResolver + }) + + handler := NewHandler(store.NewMemoryStore(), nil) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String()) + } + if got := strings.TrimSpace(recorder.Body.String()); got != `{"status":"ok","service":"droboticflow-worker"}` { + t.Fatalf("unexpected worker health payload: %s", got) + } +} + +func TestWorkerAPIProcessOnceIsProxiedThroughAPI(t *testing.T) { + worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + t.Fatalf("unexpected upstream method: %s", request.Method) + } + if request.URL.Path != "/worker-api/process-once" { + t.Fatalf("unexpected upstream path: %s", request.URL.Path) + } + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusOK) + _, _ = writer.Write([]byte(`{"run":{"id":"run-1","status":"running"}}`)) + })) + defer worker.Close() + + originalResolver := workerAPIBaseURLResolver + workerAPIBaseURLResolver = func() string { return worker.URL } + t.Cleanup(func() { + workerAPIBaseURLResolver = originalResolver + }) + + handler := NewHandler(store.NewMemoryStore(), nil) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/worker-api/process-once", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String()) + } + if got := strings.TrimSpace(recorder.Body.String()); got != `{"run":{"id":"run-1","status":"running"}}` { + t.Fatalf("unexpected worker process-once payload: %s", got) + } +} + func TestAPIHandlerSupportsDashboardCRUDAndRunQueueing(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -70,7 +210,7 @@ func TestAPIHandlerSupportsDashboardCRUDAndRunQueueing(t *testing.T) { t.Fatalf("expected project id") } - workflowPayload := []byte(`{"name":"API 工作流","description":"workflow","type":"batch","nodes":[{"id":"node-source","type":"operator","data":{"label":"真实数据源","category":"IO","description":"source","inputs":[],"outputs":[{"id":"out-1","name":"file","type":"file"}],"hyperparams":[{"key":"SOURCE_TEXT","value":"api-test"}]},"position":{"x":0,"y":0}}],"edges":[]}`) + workflowPayload := []byte(`{"name":"API 工作流","description":"workflow","nodes":[{"id":"node-source","type":"operator","data":{"label":"真实数据源","category":"IO","description":"source","inputs":[],"outputs":[{"id":"out-1","name":"file","type":"file"}],"hyperparams":[{"key":"SOURCE_TEXT","value":"api-test"}]},"position":{"x":0,"y":0}}],"edges":[]}`) workflowRecorder := httptest.NewRecorder() handler.ServeHTTP(workflowRecorder, httptest.NewRequest(http.MethodPost, "/api/projects/"+projectID+"/workflows", bytes.NewReader(workflowPayload))) if workflowRecorder.Code != http.StatusCreated { @@ -80,6 +220,9 @@ func TestAPIHandlerSupportsDashboardCRUDAndRunQueueing(t *testing.T) { if err := json.Unmarshal(workflowRecorder.Body.Bytes(), &createdWorkflow); err != nil { t.Fatalf("decode workflow: %v", err) } + if createdWorkflow["type"].(string) != "batch" { + t.Fatalf("expected workflow type to default to batch, got %v", createdWorkflow["type"]) + } projectsRecorder := httptest.NewRecorder() handler.ServeHTTP(projectsRecorder, httptest.NewRequest(http.MethodGet, "/api/projects", nil)) @@ -199,6 +342,1334 @@ func TestAPIHandlerSupportsDashboardCRUDAndRunQueueing(t *testing.T) { } } +func TestOperatorsListEnsuresMongoWriteBuiltin(t *testing.T) { + handler := NewHandler(store.NewMemoryStore(), nil) + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/operators", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("operators list status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var response struct { + Items []domain.OperatorRecord `json:"items"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode operators list: %v", err) + } + + var mongoWrite *domain.OperatorRecord + for index := range response.Items { + item := response.Items[index] + if item.Name == "MongoDB 写入" { + mongoWrite = &item + break + } + } + if mongoWrite == nil { + t.Fatalf("expected MongoDB 写入 builtin operator to be present") + } + if mongoWrite.Category != "DATA" { + t.Fatalf("expected MongoDB 写入 category DATA, got %q", mongoWrite.Category) + } + if len(mongoWrite.Inputs) == 0 || len(mongoWrite.Outputs) == 0 { + t.Fatalf("expected MongoDB 写入 ports to be defined") + } + foundTarget := false + for _, param := range mongoWrite.Hyperparams { + if param.Key == "MONGO_TARGET" && param.Value == "business" { + foundTarget = true + break + } + } + if !foundTarget { + t.Fatalf("expected MongoDB 写入 to default MONGO_TARGET=business") + } +} + +func TestOperatorsListEnsuresRegressionBuiltinCatalog(t *testing.T) { + handler := NewHandler(store.NewMemoryStore(), nil) + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/operators", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("operators list status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var response struct { + Items []domain.OperatorRecord `json:"items"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode operators list: %v", err) + } + + byName := make(map[string]domain.OperatorRecord, len(response.Items)) + for _, item := range response.Items { + byName[item.Name] = item + } + + for _, name := range []string{ + "真实数据源", + "BOS 增量扫描", + "目录扫描", + "模型推理", + "X 平台数据集索引", + "采集目录校验", + "JSON 字段提取", + "文本模板渲染", + "JSON 过滤", + "JSON 映射重命名", + "JSON 转 CSV", + "CSV 转 JSON", + "条件分流", + "数据合并", + "视频质检", + "数据过滤", + "视频切片", + "视频标注", + "成品打包", + "抽检采样", + "人工审核", + "MongoDB 写入", + "LeRobot 数据集概览", + "ROSBag 数据概览", + "结果上传", + "Shell 命令执行", + } { + if _, ok := byName[name]; !ok { + t.Fatalf("expected builtin operator %q to be present", name) + } + } + + sampling := byName["抽检采样"] + foundRules := false + for _, param := range sampling.Hyperparams { + if param.Key == "RULES_JSON" { + foundRules = true + break + } + } + if !foundRules { + t.Fatalf("expected 抽检采样 to expose RULES_JSON") + } +} + +func TestOperatorsListRefreshesBuiltinOperatorImages(t *testing.T) { + t.Setenv("OPERATOR_RUNNER_IMAGE", "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:20260416-workspacefix-amd64") + ctx := context.Background() + dataStore := store.NewMemoryStore() + item, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ + Name: "BOS 增量扫描", + Type: "builtin", + Category: "IO", + Description: "source", + Image: "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:old", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, + }) + if err != nil { + t.Fatalf("create operator: %v", err) + } + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/operators", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("operators list status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + stored, err := dataStore.GetOperator(ctx, item.ID) + if err != nil { + t.Fatalf("get operator: %v", err) + } + if stored == nil { + t.Fatalf("expected operator to exist") + } + if stored.Image != "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:20260416-workspacefix-amd64" { + t.Fatalf("expected builtin operator image to refresh, got %q", stored.Image) + } +} + +func TestOperatorsDeleteKeepsBuiltinOperatorDeleted(t *testing.T) { + handler := NewHandler(store.NewMemoryStore(), nil) + + listRecorder := httptest.NewRecorder() + handler.ServeHTTP(listRecorder, httptest.NewRequest(http.MethodGet, "/api/operators", nil)) + if listRecorder.Code != http.StatusOK { + t.Fatalf("operators list status: %d body=%s", listRecorder.Code, listRecorder.Body.String()) + } + + var listResponse struct { + Items []domain.OperatorRecord `json:"items"` + } + if err := json.Unmarshal(listRecorder.Body.Bytes(), &listResponse); err != nil { + t.Fatalf("decode operators list: %v", err) + } + + var builtin *domain.OperatorRecord + for index := range listResponse.Items { + item := listResponse.Items[index] + if item.Type == "builtin" && item.Name == "真实数据源" { + builtin = &item + break + } + } + if builtin == nil { + t.Fatalf("expected builtin operator to exist before delete") + } + + deleteRecorder := httptest.NewRecorder() + handler.ServeHTTP( + deleteRecorder, + httptest.NewRequest(http.MethodDelete, "/api/operators/"+builtin.ID, nil), + ) + if deleteRecorder.Code != http.StatusNoContent { + t.Fatalf("delete builtin status: %d body=%s", deleteRecorder.Code, deleteRecorder.Body.String()) + } + + listAfterDeleteRecorder := httptest.NewRecorder() + handler.ServeHTTP(listAfterDeleteRecorder, httptest.NewRequest(http.MethodGet, "/api/operators", nil)) + if listAfterDeleteRecorder.Code != http.StatusOK { + t.Fatalf("operators list after delete status: %d body=%s", listAfterDeleteRecorder.Code, listAfterDeleteRecorder.Body.String()) + } + + var listAfterDeleteResponse struct { + Items []domain.OperatorRecord `json:"items"` + } + if err := json.Unmarshal(listAfterDeleteRecorder.Body.Bytes(), &listAfterDeleteResponse); err != nil { + t.Fatalf("decode operators list after delete: %v", err) + } + for _, item := range listAfterDeleteResponse.Items { + if item.Name == builtin.Name { + t.Fatalf("expected builtin operator %q to stay deleted", builtin.Name) + } + } +} + +func TestDatabaseBrowserListsCollectionsAndQueriesDocuments(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + if _, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "数据库浏览项目", + Description: "用于数据库浏览测试", + }); err != nil { + t.Fatalf("create project: %v", err) + } + if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ + Name: "JSON 测试算子", + Type: "builtin", + Category: "DATA", + Description: "用于数据库过滤查询测试", + Inputs: []domain.PortDefinition{{ID: "input", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "output", Name: "output", Type: domain.PortTypeJSON}}, + }); err != nil { + t.Fatalf("create operator: %v", err) + } + + handler := NewHandler(dataStore, nil) + + overviewRecorder := httptest.NewRecorder() + handler.ServeHTTP(overviewRecorder, httptest.NewRequest(http.MethodGet, "/api/database?target=platform", nil)) + if overviewRecorder.Code != http.StatusOK { + t.Fatalf("database overview status: %d body=%s", overviewRecorder.Code, overviewRecorder.Body.String()) + } + + var overview struct { + Database string `json:"database"` + Collections []struct { + Name string `json:"name"` + DocumentCount int64 `json:"documentCount"` + } `json:"collections"` + } + if err := json.Unmarshal(overviewRecorder.Body.Bytes(), &overview); err != nil { + t.Fatalf("decode database overview: %v", err) + } + if overview.Database == "" { + t.Fatalf("expected database name in overview response") + } + if len(overview.Collections) == 0 { + t.Fatalf("expected collections in overview response") + } + + foundOperatorsCollection := false + for _, item := range overview.Collections { + if item.Name == "operators" { + foundOperatorsCollection = true + if item.DocumentCount < 1 { + t.Fatalf("expected operators collection to have documents, got %d", item.DocumentCount) + } + } + } + if !foundOperatorsCollection { + t.Fatalf("expected operators collection in overview") + } + + filter := url.QueryEscape(`{"category":"DATA"}`) + queryRecorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/api/database/collections/operators/documents?target=platform&filter="+filter+"&limit=10", nil) + handler.ServeHTTP(queryRecorder, request) + if queryRecorder.Code != http.StatusOK { + t.Fatalf("database documents status: %d body=%s", queryRecorder.Code, queryRecorder.Body.String()) + } + + var result struct { + Collection string `json:"collection"` + Documents []map[string]any `json:"documents"` + Limit int `json:"limit"` + Total int64 `json:"total"` + Filter map[string]any `json:"filter"` + } + if err := json.Unmarshal(queryRecorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode database documents: %v", err) + } + if result.Collection != "operators" { + t.Fatalf("expected operators collection, got %q", result.Collection) + } + if result.Total < 1 || len(result.Documents) < 1 { + t.Fatalf("expected at least one operator document, total=%d len=%d", result.Total, len(result.Documents)) + } + if result.Filter["category"] != "DATA" { + t.Fatalf("expected filter to round-trip, got %#v", result.Filter) + } +} + +func TestDatabaseBrowserDefaultsToBusinessTargetWhenUnspecified(t *testing.T) { + handler := NewHandler(store.NewMemoryStore(), nil) + + overviewRecorder := httptest.NewRecorder() + handler.ServeHTTP(overviewRecorder, httptest.NewRequest(http.MethodGet, "/api/database", nil)) + if overviewRecorder.Code != http.StatusOK { + t.Fatalf("database overview status: %d body=%s", overviewRecorder.Code, overviewRecorder.Body.String()) + } + if !strings.Contains(overviewRecorder.Body.String(), `"target":"business"`) { + t.Fatalf("expected default target=business, got %s", overviewRecorder.Body.String()) + } + if !strings.Contains(overviewRecorder.Body.String(), `"configured":false`) { + t.Fatalf("expected configured=false for unconfigured business mongo, got %s", overviewRecorder.Body.String()) + } +} + +func TestDatabaseBrowserUsesMongoConnectionWhenProvided(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{ + Name: "业务库", + URI: "mongodb://business.example:27017", + Database: "business", + }) + if err != nil { + t.Fatalf("create mongo connection: %v", err) + } + + originalResolver := mongoConnectionStoreResolver + t.Cleanup(func() { + mongoConnectionStoreResolver = originalResolver + }) + businessStore := store.NewMemoryStore() + if _, err := businessStore.CreateProject(ctx, domain.CreateProjectInput{Name: "业务项目", Description: "业务库"}); err != nil { + t.Fatalf("create business project: %v", err) + } + mongoConnectionStoreResolver = func(_ context.Context, item *domain.MongoConnectionRecord) (databaseBrowserStore, func(), error) { + if item == nil || item.ID != connection.ID || item.URI != "mongodb://business.example:27017" || item.Database != "business" { + t.Fatalf("unexpected mongo connection: %#v", item) + } + return businessStore, func() {}, nil + } + + handler := NewHandler(dataStore, nil) + + overviewRecorder := httptest.NewRecorder() + handler.ServeHTTP(overviewRecorder, httptest.NewRequest(http.MethodGet, "/api/database?connectionId="+connection.ID, nil)) + if overviewRecorder.Code != http.StatusOK { + t.Fatalf("business database overview status: %d body=%s", overviewRecorder.Code, overviewRecorder.Body.String()) + } + if !strings.Contains(overviewRecorder.Body.String(), `"target":"business"`) { + t.Fatalf("expected business target in overview, got %s", overviewRecorder.Body.String()) + } + if !strings.Contains(overviewRecorder.Body.String(), `"connectionId":"`+connection.ID+`"`) { + t.Fatalf("expected connection id in overview, got %s", overviewRecorder.Body.String()) + } + + queryRecorder := httptest.NewRecorder() + handler.ServeHTTP(queryRecorder, httptest.NewRequest(http.MethodGet, "/api/database/collections/projects/documents?connectionId="+connection.ID+"&limit=5", nil)) + if queryRecorder.Code != http.StatusOK { + t.Fatalf("business database documents status: %d body=%s", queryRecorder.Code, queryRecorder.Body.String()) + } + if !strings.Contains(queryRecorder.Body.String(), `"configured":true`) { + t.Fatalf("expected configured=true, got %s", queryRecorder.Body.String()) + } + if !strings.Contains(queryRecorder.Body.String(), `"target":"business"`) { + t.Fatalf("expected business target in query result, got %s", queryRecorder.Body.String()) + } +} + +func TestDatabaseBrowserBusinessCollectionAndDocumentCRUD(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{ + Name: "业务写入库", + URI: "mongodb://business.example:27017", + Database: "business", + }) + if err != nil { + t.Fatalf("create mongo connection: %v", err) + } + + originalResolver := mongoConnectionStoreResolver + t.Cleanup(func() { + mongoConnectionStoreResolver = originalResolver + }) + businessStore := store.NewMemoryStore() + mongoConnectionStoreResolver = func(_ context.Context, item *domain.MongoConnectionRecord) (databaseBrowserStore, func(), error) { + if item == nil || item.ID != connection.ID { + t.Fatalf("unexpected mongo connection: %#v", item) + } + return businessStore, func() {}, nil + } + + handler := NewHandler(dataStore, nil) + connectionQuery := "?connectionId=" + connection.ID + + createCollectionRecorder := httptest.NewRecorder() + createCollectionBody := bytes.NewBufferString(`{"name":"orders"}`) + handler.ServeHTTP( + createCollectionRecorder, + httptest.NewRequest(http.MethodPost, "/api/database/collections"+connectionQuery, createCollectionBody), + ) + if createCollectionRecorder.Code != http.StatusCreated { + t.Fatalf("create collection status: %d body=%s", createCollectionRecorder.Code, createCollectionRecorder.Body.String()) + } + + createDocumentRecorder := httptest.NewRecorder() + createDocumentBody := bytes.NewBufferString(`{"document":{"orderNo":"SO-001","status":"new","amount":12}}`) + handler.ServeHTTP( + createDocumentRecorder, + httptest.NewRequest(http.MethodPost, "/api/database/collections/orders/documents"+connectionQuery, createDocumentBody), + ) + if createDocumentRecorder.Code != http.StatusCreated { + t.Fatalf("create document status: %d body=%s", createDocumentRecorder.Code, createDocumentRecorder.Body.String()) + } + var createdDocument struct { + Collection string `json:"collection"` + Document map[string]any `json:"document"` + } + if err := json.Unmarshal(createDocumentRecorder.Body.Bytes(), &createdDocument); err != nil { + t.Fatalf("decode created document: %v", err) + } + documentID, _ := createdDocument.Document["_id"].(string) + if documentID == "" { + t.Fatalf("expected created document id, got %#v", createdDocument.Document["_id"]) + } + + queryRecorder := httptest.NewRecorder() + filter := url.QueryEscape(`{"orderNo":"SO-001"}`) + handler.ServeHTTP( + queryRecorder, + httptest.NewRequest(http.MethodGet, "/api/database/collections/orders/documents"+connectionQuery+"&filter="+filter, nil), + ) + if queryRecorder.Code != http.StatusOK { + t.Fatalf("query documents status: %d body=%s", queryRecorder.Code, queryRecorder.Body.String()) + } + if !strings.Contains(queryRecorder.Body.String(), `"total":1`) { + t.Fatalf("expected one query result, got %s", queryRecorder.Body.String()) + } + + updateDocumentRecorder := httptest.NewRecorder() + updateDocumentBody := bytes.NewBufferString(`{"document":{"orderNo":"SO-001","status":"done","amount":18}}`) + handler.ServeHTTP( + updateDocumentRecorder, + httptest.NewRequest(http.MethodPut, "/api/database/collections/orders/documents/"+documentID+connectionQuery, updateDocumentBody), + ) + if updateDocumentRecorder.Code != http.StatusOK { + t.Fatalf("update document status: %d body=%s", updateDocumentRecorder.Code, updateDocumentRecorder.Body.String()) + } + if !strings.Contains(updateDocumentRecorder.Body.String(), `"status":"done"`) { + t.Fatalf("expected updated status in response, got %s", updateDocumentRecorder.Body.String()) + } + + renameCollectionRecorder := httptest.NewRecorder() + renameCollectionBody := bytes.NewBufferString(`{"newName":"ordersArchive"}`) + handler.ServeHTTP( + renameCollectionRecorder, + httptest.NewRequest(http.MethodPatch, "/api/database/collections/orders"+connectionQuery, renameCollectionBody), + ) + if renameCollectionRecorder.Code != http.StatusOK { + t.Fatalf("rename collection status: %d body=%s", renameCollectionRecorder.Code, renameCollectionRecorder.Body.String()) + } + if !strings.Contains(renameCollectionRecorder.Body.String(), `"collection":"ordersArchive"`) { + t.Fatalf("expected renamed collection in response, got %s", renameCollectionRecorder.Body.String()) + } + + deleteDocumentRecorder := httptest.NewRecorder() + handler.ServeHTTP( + deleteDocumentRecorder, + httptest.NewRequest(http.MethodDelete, "/api/database/collections/ordersArchive/documents/"+documentID+connectionQuery, nil), + ) + if deleteDocumentRecorder.Code != http.StatusOK { + t.Fatalf("delete document status: %d body=%s", deleteDocumentRecorder.Code, deleteDocumentRecorder.Body.String()) + } + + deleteCollectionRecorder := httptest.NewRecorder() + handler.ServeHTTP( + deleteCollectionRecorder, + httptest.NewRequest(http.MethodDelete, "/api/database/collections/ordersArchive"+connectionQuery, nil), + ) + if deleteCollectionRecorder.Code != http.StatusOK { + t.Fatalf("delete collection status: %d body=%s", deleteCollectionRecorder.Code, deleteCollectionRecorder.Body.String()) + } + + overviewRecorder := httptest.NewRecorder() + handler.ServeHTTP(overviewRecorder, httptest.NewRequest(http.MethodGet, "/api/database"+connectionQuery, nil)) + if overviewRecorder.Code != http.StatusOK { + t.Fatalf("overview status: %d body=%s", overviewRecorder.Code, overviewRecorder.Body.String()) + } + if strings.Contains(overviewRecorder.Body.String(), "ordersArchive") { + t.Fatalf("expected deleted collection to be absent, got %s", overviewRecorder.Body.String()) + } +} + +func TestDatabaseBrowserBusinessIndexCRUD(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{ + Name: "业务索引库", + URI: "mongodb://business.example:27017", + Database: "business", + }) + if err != nil { + t.Fatalf("create mongo connection: %v", err) + } + + originalResolver := mongoConnectionStoreResolver + t.Cleanup(func() { + mongoConnectionStoreResolver = originalResolver + }) + businessStore := store.NewMemoryStore() + if _, err := businessStore.CreateDatabaseCollection(ctx, "events"); err != nil { + t.Fatalf("create collection: %v", err) + } + mongoConnectionStoreResolver = func(_ context.Context, item *domain.MongoConnectionRecord) (databaseBrowserStore, func(), error) { + if item == nil || item.ID != connection.ID { + t.Fatalf("unexpected mongo connection: %#v", item) + } + return businessStore, func() {}, nil + } + + handler := NewHandler(dataStore, nil) + connectionQuery := "?connectionId=" + connection.ID + + createIndexRecorder := httptest.NewRecorder() + createIndexBody := bytes.NewBufferString(`{"name":"idx-record-key","keys":{"recordKey":1},"unique":true}`) + handler.ServeHTTP( + createIndexRecorder, + httptest.NewRequest(http.MethodPost, "/api/database/collections/events/indexes"+connectionQuery, createIndexBody), + ) + if createIndexRecorder.Code != http.StatusCreated { + t.Fatalf("create index status: %d body=%s", createIndexRecorder.Code, createIndexRecorder.Body.String()) + } + if !strings.Contains(createIndexRecorder.Body.String(), `"name":"idx-record-key"`) { + t.Fatalf("expected created index name, got %s", createIndexRecorder.Body.String()) + } + + listIndexesRecorder := httptest.NewRecorder() + handler.ServeHTTP( + listIndexesRecorder, + httptest.NewRequest(http.MethodGet, "/api/database/collections/events/indexes"+connectionQuery, nil), + ) + if listIndexesRecorder.Code != http.StatusOK { + t.Fatalf("list indexes status: %d body=%s", listIndexesRecorder.Code, listIndexesRecorder.Body.String()) + } + if !strings.Contains(listIndexesRecorder.Body.String(), `"idx-record-key"`) { + t.Fatalf("expected created index in list, got %s", listIndexesRecorder.Body.String()) + } + + deleteIndexRecorder := httptest.NewRecorder() + handler.ServeHTTP( + deleteIndexRecorder, + httptest.NewRequest(http.MethodDelete, "/api/database/collections/events/indexes/idx-record-key"+connectionQuery, nil), + ) + if deleteIndexRecorder.Code != http.StatusOK { + t.Fatalf("delete index status: %d body=%s", deleteIndexRecorder.Code, deleteIndexRecorder.Body.String()) + } + if strings.Contains(deleteIndexRecorder.Body.String(), `"deleted":false`) { + t.Fatalf("expected index deletion success, got %s", deleteIndexRecorder.Body.String()) + } +} + +func TestMongoConnectionsTestEndpointUsesDraftOrStoredConnection(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + storedConnection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{ + Name: "业务测试库", + URI: "mongodb://business.example:27017", + Database: "business", + }) + if err != nil { + t.Fatalf("create mongo connection: %v", err) + } + + originalResolver := mongoConnectionStoreResolver + t.Cleanup(func() { + mongoConnectionStoreResolver = originalResolver + }) + probeStore := store.NewMemoryStore() + if _, err := probeStore.CreateDatabaseCollection(ctx, "probeCollection"); err != nil { + t.Fatalf("create probe collection: %v", err) + } + mongoConnectionStoreResolver = func(_ context.Context, item *domain.MongoConnectionRecord) (databaseBrowserStore, func(), error) { + if item == nil { + t.Fatalf("expected mongo connection") + } + return probeStore, func() {}, nil + } + + handler := NewHandler(dataStore, nil) + + draftRecorder := httptest.NewRecorder() + draftBody := bytes.NewBufferString(`{"uri":"mongodb://draft.example:27017","database":"draft-db"}`) + handler.ServeHTTP(draftRecorder, httptest.NewRequest(http.MethodPost, "/api/mongo/connections/test", draftBody)) + if draftRecorder.Code != http.StatusOK { + t.Fatalf("draft connection test status: %d body=%s", draftRecorder.Code, draftRecorder.Body.String()) + } + if !strings.Contains(draftRecorder.Body.String(), `"probeCollection"`) { + t.Fatalf("expected probe collection in test response, got %s", draftRecorder.Body.String()) + } + + storedRecorder := httptest.NewRecorder() + storedBody := bytes.NewBufferString(`{"id":"` + storedConnection.ID + `"}`) + handler.ServeHTTP(storedRecorder, httptest.NewRequest(http.MethodPost, "/api/mongo/connections/test", storedBody)) + if storedRecorder.Code != http.StatusOK { + t.Fatalf("stored connection test status: %d body=%s", storedRecorder.Code, storedRecorder.Body.String()) + } + if !strings.Contains(storedRecorder.Body.String(), `"connectionId":"`+storedConnection.ID+`"`) { + t.Fatalf("expected stored connection id in response, got %s", storedRecorder.Body.String()) + } +} + +func TestRunsListReturnsSummaryPayload(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + run := domain.RunRecord{ + ID: "run-summary-1", + WorkflowID: "workflow-1", + WorkflowName: "Summary Workflow", + ProjectID: "project-1", + ProjectName: "Summary Project", + Trigger: domain.RunTriggerScheduled, + Status: domain.RunStatusRunning, + CurrentNodeID: "node-quality", + NodeExecutions: []domain.NodeExecutionRecord{ + { + NodeID: "node-quality", + NodeLabel: "视频质检", + Status: domain.NodeExecutionStatusRunning, + Logs: []domain.LogEntryRecord{ + {Timestamp: "2026-04-13T01:00:00Z", Level: "INFO", Message: "heavy node log"}, + }, + Artifacts: []domain.ExecutionArtifact{ + {ID: "artifact-1", RunID: "run-summary-1", NodeID: "node-quality", Name: "quality-report.json", Type: domain.PortTypeJSON, Size: "2 KB", Path: "/tmp/quality-report.json"}, + }, + ResourceSeries: []domain.ResourceSample{ + {Timestamp: "2026-04-13T01:00:00Z", CPU: 90, Memory: 1024}, + }, + }, + }, + Logs: []domain.LogEntryRecord{ + {Timestamp: "2026-04-13T01:00:00Z", Level: "INFO", Message: "top level log"}, + }, + ResourceSeries: []domain.ResourceSample{ + {Timestamp: "2026-04-13T01:00:00Z", CPU: 80, Memory: 2048}, + }, + Artifacts: []domain.ExecutionArtifact{ + {ID: "artifact-2", RunID: "run-summary-1", NodeID: "node-quality", Name: "clip.mp4", Type: domain.PortTypeVideo, Size: "120 MB", Path: "/tmp/clip.mp4"}, + }, + K8sRefs: []domain.K8sRef{ + {NodeID: "node-quality", JobName: "quality-job", Namespace: "droboticflow"}, + }, + CreatedAt: "2026-04-13T01:00:00Z", + UpdatedAt: "2026-04-13T01:00:00Z", + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/runs", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("runs list status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + body := recorder.Body.String() + for _, forbidden := range []string{"\"logs\"", "\"artifacts\"", "\"resourceSeries\"", "\"nodeExecutions\"", "\"k8sRefs\""} { + if strings.Contains(body, forbidden) { + t.Fatalf("runs list should not include %s in summary payload: %s", forbidden, body) + } + } + + var response struct { + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode runs response: %v", err) + } + if len(response.Items) != 1 { + t.Fatalf("expected 1 run summary, got %d", len(response.Items)) + } + if response.Items[0]["id"] != run.ID { + t.Fatalf("summary id = %v, want %s", response.Items[0]["id"], run.ID) + } + if response.Items[0]["currentNodeId"] != run.CurrentNodeID { + t.Fatalf("summary currentNodeId = %v, want %s", response.Items[0]["currentNodeId"], run.CurrentNodeID) + } +} + +func TestWorkflowRunsListReturnsWorkflowOnlySummaries(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "历史项目", Description: "history"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflowA, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "周期工作流 A", + Description: "a", + Type: domain.WorkflowTypeStreaming, + Schedule: &domain.WorkflowSchedule{IntervalHours: 2, Mode: domain.WorkflowScheduleModeEveryHours}, + Nodes: []domain.WorkflowNode{}, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflowA: %v", err) + } + workflowB, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "周期工作流 B", + Description: "b", + Type: domain.WorkflowTypeStreaming, + Schedule: &domain.WorkflowSchedule{IntervalHours: 3, Mode: domain.WorkflowScheduleModeEveryHours}, + Nodes: []domain.WorkflowNode{}, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflowB: %v", err) + } + if err := dataStore.UpsertRun(ctx, domain.RunRecord{ + ID: "run-a1", + WorkflowID: workflowA.ID, + WorkflowName: workflowA.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerScheduled, + Status: domain.RunStatusCompleted, + CreatedAt: "2026-04-22T01:00:00Z", + UpdatedAt: "2026-04-22T01:00:00Z", + NodeExecutions: []domain.NodeExecutionRecord{}, + Logs: []domain.LogEntryRecord{}, + ResourceSeries: []domain.ResourceSample{}, + Artifacts: []domain.ExecutionArtifact{}, + K8sRefs: []domain.K8sRef{}, + }); err != nil { + t.Fatalf("upsert workflowA run: %v", err) + } + if err := dataStore.UpsertRun(ctx, domain.RunRecord{ + ID: "run-b1", + WorkflowID: workflowB.ID, + WorkflowName: workflowB.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerScheduled, + Status: domain.RunStatusFailed, + CreatedAt: "2026-04-22T02:00:00Z", + UpdatedAt: "2026-04-22T02:00:00Z", + NodeExecutions: []domain.NodeExecutionRecord{}, + Logs: []domain.LogEntryRecord{}, + ResourceSeries: []domain.ResourceSample{}, + Artifacts: []domain.ExecutionArtifact{}, + K8sRefs: []domain.K8sRef{}, + }); err != nil { + t.Fatalf("upsert workflowB run: %v", err) + } + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/workflows/"+workflowA.ID+"/runs", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("workflow runs list status: %d body=%s", recorder.Code, recorder.Body.String()) + } + var response struct { + Items []domain.RunSummary `json:"items"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode workflow runs: %v", err) + } + if len(response.Items) != 1 { + t.Fatalf("expected 1 workflow run summary, got %d", len(response.Items)) + } + if response.Items[0].ID != "run-a1" { + t.Fatalf("expected workflowA run summary, got %#v", response.Items[0]) + } +} + +func TestRunsRerunSubtreeCreatesQueuedChildRunWithBaseline(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "补跑项目", Description: "rerun"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "失败补跑工作流", + Description: "rerun subtree", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + {ID: "a", Type: "operator", Data: domain.WorkflowNodeData{Label: "A", Category: "DATA", Description: "a"}}, + {ID: "b", Type: "operator", Data: domain.WorkflowNodeData{Label: "B", Category: "DATA", Description: "b"}}, + {ID: "c", Type: "operator", Data: domain.WorkflowNodeData{Label: "C", Category: "DATA", Description: "c"}}, + {ID: "d", Type: "operator", Data: domain.WorkflowNodeData{Label: "D", Category: "DATA", Description: "d"}}, + }, + Edges: []domain.WorkflowEdge{ + {ID: "a-b", Source: "a", Target: "b"}, + {ID: "b-d", Source: "b", Target: "d"}, + {ID: "c-d", Source: "c", Target: "d"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + sourceRun := domain.RunRecord{ + ID: "run-failed-1", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerScheduled, + Status: domain.RunStatusFailed, + CurrentNodeID: "b", + NodeExecutions: []domain.NodeExecutionRecord{ + {NodeID: "a", NodeLabel: "A", Status: domain.NodeExecutionStatusCompleted, Logs: []domain.LogEntryRecord{{Message: "a ok"}}, Artifacts: []domain.ExecutionArtifact{{ID: "artifact-a", Name: "a.json"}}}, + {NodeID: "c", NodeLabel: "C", Status: domain.NodeExecutionStatusCompleted, Logs: []domain.LogEntryRecord{{Message: "c ok"}}, Artifacts: []domain.ExecutionArtifact{{ID: "artifact-c", Name: "c.json"}}}, + {NodeID: "b", NodeLabel: "B", Status: domain.NodeExecutionStatusFailed, Error: "boom"}, + }, + CreatedAt: "2026-04-22T03:00:00Z", + UpdatedAt: "2026-04-22T03:05:00Z", + } + if err := dataStore.UpsertRun(ctx, sourceRun); err != nil { + t.Fatalf("upsert source run: %v", err) + } + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + body := bytes.NewBufferString(`{"nodeId":"b"}`) + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/runs/"+sourceRun.ID+"/rerun-subtree", body)) + if recorder.Code != http.StatusAccepted { + t.Fatalf("rerun subtree status: %d body=%s", recorder.Code, recorder.Body.String()) + } + var child domain.RunRecord + if err := json.Unmarshal(recorder.Body.Bytes(), &child); err != nil { + t.Fatalf("decode child run: %v", err) + } + if child.Trigger != domain.RunTriggerRerunSubtree { + t.Fatalf("expected rerun_subtree trigger, got %s", child.Trigger) + } + if child.ParentRunID != sourceRun.ID { + t.Fatalf("expected parent run %s, got %s", sourceRun.ID, child.ParentRunID) + } + if child.RestartFromNodeID != "b" { + t.Fatalf("expected restartFromNodeId=b, got %s", child.RestartFromNodeID) + } + if child.Status != domain.RunStatusQueued { + t.Fatalf("expected queued child run, got %s", child.Status) + } + if len(child.NodeExecutions) != 2 { + t.Fatalf("expected 2 baseline executions, got %#v", child.NodeExecutions) + } + nodeIDs := []string{child.NodeExecutions[0].NodeID, child.NodeExecutions[1].NodeID} + sort.Strings(nodeIDs) + if strings.Join(nodeIDs, ",") != "a,c" { + t.Fatalf("expected baseline nodes a,c, got %v", nodeIDs) + } +} + +func TestRunsRerunSubtreeRejectsMissingCompletedBaseline(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "补跑项目", Description: "rerun"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "失败补跑工作流", + Description: "rerun subtree", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + {ID: "a", Type: "operator", Data: domain.WorkflowNodeData{Label: "A", Category: "DATA", Description: "a"}}, + {ID: "b", Type: "operator", Data: domain.WorkflowNodeData{Label: "B", Category: "DATA", Description: "b"}}, + {ID: "c", Type: "operator", Data: domain.WorkflowNodeData{Label: "C", Category: "DATA", Description: "c"}}, + {ID: "d", Type: "operator", Data: domain.WorkflowNodeData{Label: "D", Category: "DATA", Description: "d"}}, + }, + Edges: []domain.WorkflowEdge{ + {ID: "a-b", Source: "a", Target: "b"}, + {ID: "b-d", Source: "b", Target: "d"}, + {ID: "c-d", Source: "c", Target: "d"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + sourceRun := domain.RunRecord{ + ID: "run-failed-2", + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: domain.RunTriggerScheduled, + Status: domain.RunStatusFailed, + CurrentNodeID: "b", + NodeExecutions: []domain.NodeExecutionRecord{ + {NodeID: "a", NodeLabel: "A", Status: domain.NodeExecutionStatusCompleted}, + {NodeID: "c", NodeLabel: "C", Status: domain.NodeExecutionStatusFailed, Error: "side branch failed"}, + {NodeID: "b", NodeLabel: "B", Status: domain.NodeExecutionStatusFailed, Error: "boom"}, + }, + CreatedAt: "2026-04-22T03:00:00Z", + UpdatedAt: "2026-04-22T03:05:00Z", + } + if err := dataStore.UpsertRun(ctx, sourceRun); err != nil { + t.Fatalf("upsert source run: %v", err) + } + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + body := bytes.NewBufferString(`{"nodeId":"b"}`) + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/runs/"+sourceRun.ID+"/rerun-subtree", body)) + if recorder.Code != http.StatusConflict { + t.Fatalf("expected 409 for invalid baseline, got %d body=%s", recorder.Code, recorder.Body.String()) + } +} + +func TestWorkflowNodeSourceStateEndpointsExposeAndResetCommittedState(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + Name: "扫描状态工作流", + Description: "scan source state", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + ScanConfig: &domain.ScanConfig{ + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test/source-a", + Bucket: "datastorage", + Prefix: "algo/X-platform-test/source-a", + ScanMode: "INCREMENTAL", + }, + Nodes: []domain.WorkflowNode{ + { + ID: "scan-source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Category: "IO", + Description: "source", + }, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + currentFingerprint := domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{ + ScanMode: "INCREMENTAL", + IncrementalUnit: "OBJECT", + DirectoryDepth: 1, + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test/source-a", + Bucket: "datastorage", + Prefix: "algo/X-platform-test/source-a", + }) + legacyFingerprint := domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{ + ScanMode: "INCREMENTAL", + IncrementalUnit: "OBJECT", + DirectoryDepth: 1, + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test/source-legacy", + Bucket: "datastorage", + Prefix: "algo/X-platform-test/source-legacy", + }) + + if err := dataStore.UpsertSourceState(ctx, domain.SourceStateRecord{ + ID: workflow.ID + ":scan-source:" + legacyFingerprint, + WorkflowID: workflow.ID, + NodeID: "scan-source", + SourceFingerprint: legacyFingerprint, + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test/source-legacy", + Bucket: "datastorage", + Prefix: "algo/X-platform-test/source-legacy", + ScanMode: "INCREMENTAL", + IncrementalUnit: "OBJECT", + DirectoryDepth: 1, + SeenFingerprints: []string{"legacy-1"}, + LastRunID: "run-legacy", + LastListedCount: 3, + LastNewCount: 1, + LastScanCompletedAt: "2026-04-12T00:00:00Z", + UpdatedAt: "2026-04-12T00:00:01Z", + }); err != nil { + t.Fatalf("seed legacy source state: %v", err) + } + if err := dataStore.UpsertSourceState(ctx, domain.SourceStateRecord{ + ID: workflow.ID + ":scan-source:" + currentFingerprint, + WorkflowID: workflow.ID, + NodeID: "scan-source", + SourceFingerprint: currentFingerprint, + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test/source-a", + Bucket: "datastorage", + Prefix: "algo/X-platform-test/source-a", + ScanMode: "INCREMENTAL", + IncrementalUnit: "OBJECT", + DirectoryDepth: 1, + SeenFingerprints: []string{"entry-1", "entry-2"}, + LastRunID: "run-current", + LastListedCount: 8, + LastNewCount: 2, + LastScanCompletedAt: "2026-04-12T01:00:00Z", + UpdatedAt: "2026-04-12T01:00:01Z", + }); err != nil { + t.Fatalf("seed current source state: %v", err) + } + + handler := NewHandler(dataStore, nil) + + getRecorder := httptest.NewRecorder() + handler.ServeHTTP(getRecorder, httptest.NewRequest(http.MethodGet, "/api/workflows/"+workflow.ID+"/nodes/scan-source/source-state", nil)) + if getRecorder.Code != http.StatusOK { + t.Fatalf("source state get status: %d body=%s", getRecorder.Code, getRecorder.Body.String()) + } + + var body workflowNodeSourceStateResponse + if err := json.Unmarshal(getRecorder.Body.Bytes(), &body); err != nil { + t.Fatalf("decode source state response: %v", err) + } + if !body.Supported || !body.Stateful { + t.Fatalf("expected supported stateful response, got %+v", body) + } + if body.Current == nil || body.Current.LastRunID != "run-current" { + t.Fatalf("expected current source state, got %+v", body.Current) + } + if body.CurrentFingerprint != currentFingerprint { + t.Fatalf("expected current fingerprint %s, got %s", currentFingerprint, body.CurrentFingerprint) + } + if len(body.Versions) != 2 { + t.Fatalf("expected 2 source state versions, got %+v", body.Versions) + } + if body.Versions[0].LastRunID != "run-current" { + t.Fatalf("expected latest source state first, got %+v", body.Versions) + } + + deleteRecorder := httptest.NewRecorder() + handler.ServeHTTP(deleteRecorder, httptest.NewRequest(http.MethodDelete, "/api/workflows/"+workflow.ID+"/nodes/scan-source/source-state", nil)) + if deleteRecorder.Code != http.StatusNoContent { + t.Fatalf("source state delete status: %d body=%s", deleteRecorder.Code, deleteRecorder.Body.String()) + } + + stored, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", currentFingerprint) + if err != nil { + t.Fatalf("get deleted current state: %v", err) + } + if stored != nil { + t.Fatalf("expected current source state removed, got %+v", stored) + } + legacyStored, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", legacyFingerprint) + if err != nil { + t.Fatalf("get legacy source state: %v", err) + } + if legacyStored == nil { + t.Fatalf("expected legacy source state to remain") + } +} + +func TestWorkflowNodeSourceStateEndpointsSupportDedicatedBosScanOperatorNodes(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + Name: "BOS 扫描状态工作流", + Description: "dedicated bos scan state", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + ScanConfig: &domain.ScanConfig{ + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test", + Bucket: "datastorage", + Prefix: "algo/X-platform-test", + ScanMode: "INCREMENTAL", + }, + Nodes: []domain.WorkflowNode{ + { + ID: "node-bos-scan", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Category: "IO", + Description: "source", + }, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + currentFingerprint := domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{ + ScanMode: "INCREMENTAL", + IncrementalUnit: "OBJECT", + DirectoryDepth: 1, + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test", + Bucket: "datastorage", + Prefix: "algo/X-platform-test", + }) + + if err := dataStore.UpsertSourceState(ctx, domain.SourceStateRecord{ + ID: workflow.ID + ":node-bos-scan:" + currentFingerprint, + WorkflowID: workflow.ID, + NodeID: "node-bos-scan", + SourceFingerprint: currentFingerprint, + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test", + Bucket: "datastorage", + Prefix: "algo/X-platform-test", + ScanMode: "INCREMENTAL", + IncrementalUnit: "OBJECT", + DirectoryDepth: 1, + SeenFingerprints: []string{"entry-1"}, + LastRunID: "run-bos-current", + LastListedCount: 6, + LastNewCount: 1, + LastScanCompletedAt: "2026-04-12T02:00:00Z", + UpdatedAt: "2026-04-12T02:00:01Z", + }); err != nil { + t.Fatalf("seed bos source state: %v", err) + } + + handler := NewHandler(dataStore, nil) + + getRecorder := httptest.NewRecorder() + handler.ServeHTTP(getRecorder, httptest.NewRequest(http.MethodGet, "/api/workflows/"+workflow.ID+"/nodes/node-bos-scan/source-state", nil)) + if getRecorder.Code != http.StatusOK { + t.Fatalf("source state get status: %d body=%s", getRecorder.Code, getRecorder.Body.String()) + } + + var body workflowNodeSourceStateResponse + if err := json.Unmarshal(getRecorder.Body.Bytes(), &body); err != nil { + t.Fatalf("decode source state response: %v", err) + } + if !body.Supported || !body.Stateful { + t.Fatalf("expected supported stateful response, got %+v", body) + } + if body.Current == nil || body.Current.LastRunID != "run-bos-current" { + t.Fatalf("expected current bos source state, got %+v", body.Current) + } + if body.CurrentFingerprint != currentFingerprint { + t.Fatalf("expected current fingerprint %s, got %s", currentFingerprint, body.CurrentFingerprint) + } +} + +func TestUserStoragePathFavoritesAPIStoresNormalizedFavoritesWithoutLeakingToUsersList(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + user, err := dataStore.CreateUser(ctx, domain.CreateUserInput{ + Name: "收藏用户", + Email: "favorite-user@example.com", + Role: domain.UserRoleOperator, + }) + if err != nil { + t.Fatalf("create user: %v", err) + } + + handler := NewHandler(dataStore, nil) + + updatePayload := []byte(`{ + "items": [ + {"scopeKey":"connection:storage-a","bucket":"datastorage","prefix":"algo/X-platform-test","label":" "}, + {"scopeKey":"connection:storage-a","bucket":"datastorage","prefix":"algo/X-platform-test/","label":"重复"}, + {"scopeKey":"connection:storage-a","bucket":"datastorage","prefix":"processed"}, + {"scopeKey":"","bucket":"datastorage","prefix":"invalid"}, + {"scopeKey":"source:bos://datastorage/root","bucket":"archive","prefix":" clips "} + ] + }`) + updateRecorder := httptest.NewRecorder() + handler.ServeHTTP( + updateRecorder, + httptest.NewRequest(http.MethodPut, "/api/users/"+user.ID+"/storage-path-favorites", bytes.NewReader(updatePayload)), + ) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("favorites update status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + var updateBody struct { + Items []domain.StoragePathFavoriteRecord `json:"items"` + } + if err := json.Unmarshal(updateRecorder.Body.Bytes(), &updateBody); err != nil { + t.Fatalf("decode favorites update: %v", err) + } + expected := []domain.StoragePathFavoriteRecord{ + { + ScopeKey: "connection:storage-a", + Bucket: "datastorage", + Prefix: "algo/X-platform-test", + Label: "datastorage/algo/X-platform-test", + }, + { + ScopeKey: "connection:storage-a", + Bucket: "datastorage", + Prefix: "processed", + Label: "datastorage/processed", + }, + { + ScopeKey: "source:bos://datastorage/root", + Bucket: "archive", + Prefix: "clips", + Label: "archive/clips", + }, + } + if got, want := len(updateBody.Items), len(expected); got != want { + t.Fatalf("favorites count = %d, want %d", got, want) + } + for index := range expected { + if updateBody.Items[index] != expected[index] { + t.Fatalf("favorite[%d] = %#v, want %#v", index, updateBody.Items[index], expected[index]) + } + } + + getRecorder := httptest.NewRecorder() + handler.ServeHTTP( + getRecorder, + httptest.NewRequest(http.MethodGet, "/api/users/"+user.ID+"/storage-path-favorites", nil), + ) + if getRecorder.Code != http.StatusOK { + t.Fatalf("favorites get status: %d body=%s", getRecorder.Code, getRecorder.Body.String()) + } + var getBody struct { + Items []domain.StoragePathFavoriteRecord `json:"items"` + } + if err := json.Unmarshal(getRecorder.Body.Bytes(), &getBody); err != nil { + t.Fatalf("decode favorites get: %v", err) + } + if strings.Join(func() []string { + values := make([]string, 0, len(getBody.Items)) + for _, item := range getBody.Items { + values = append(values, item.ScopeKey+"|"+item.Bucket+"|"+item.Prefix+"|"+item.Label) + } + return values + }(), "\n") != strings.Join(func() []string { + values := make([]string, 0, len(expected)) + for _, item := range expected { + values = append(values, item.ScopeKey+"|"+item.Bucket+"|"+item.Prefix+"|"+item.Label) + } + return values + }(), "\n") { + t.Fatalf("favorites get body = %#v, want %#v", getBody.Items, expected) + } + + usersRecorder := httptest.NewRecorder() + handler.ServeHTTP(usersRecorder, httptest.NewRequest(http.MethodGet, "/api/users", nil)) + if usersRecorder.Code != http.StatusOK { + t.Fatalf("users list status: %d body=%s", usersRecorder.Code, usersRecorder.Body.String()) + } + if strings.Contains(usersRecorder.Body.String(), "storagePathFavorites") { + t.Fatalf("expected users list to omit storagePathFavorites, got %s", usersRecorder.Body.String()) + } + + storedUser, err := dataStore.GetUser(ctx, user.ID) + if err != nil { + t.Fatalf("get stored user: %v", err) + } + if storedUser == nil || len(storedUser.StoragePathFavorites) != len(expected) { + t.Fatalf("stored favorites = %#v, want %#v", storedUser, expected) + } +} + +func TestProjectStoragePathFavoritesAPIStoresNormalizedFavoritesWithoutLeakingToProjectResponses(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "共享收藏项目", + Description: "project favorites", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + handler := NewHandler(dataStore, nil) + + updatePayload := []byte(`{ + "items": [ + {"scopeKey":"connection:storage-a","bucket":"datastorage","prefix":"algo/X-platform-test"}, + {"scopeKey":"connection:storage-a","bucket":"datastorage","prefix":"algo/X-platform-test/"}, + {"scopeKey":"connection:storage-a","bucket":"datastorage","prefix":"processed"}, + {"scopeKey":"source:bos://datastorage/root","bucket":"archive","prefix":" clips "} + ] + }`) + updateRecorder := httptest.NewRecorder() + handler.ServeHTTP( + updateRecorder, + httptest.NewRequest(http.MethodPut, "/api/projects/"+project.ID+"/storage-path-favorites", bytes.NewReader(updatePayload)), + ) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("project favorites update status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + var updateBody struct { + Items []domain.StoragePathFavoriteRecord `json:"items"` + } + if err := json.Unmarshal(updateRecorder.Body.Bytes(), &updateBody); err != nil { + t.Fatalf("decode project favorites update: %v", err) + } + if len(updateBody.Items) != 3 { + t.Fatalf("expected 3 normalized project favorites, got %#v", updateBody.Items) + } + + getRecorder := httptest.NewRecorder() + handler.ServeHTTP( + getRecorder, + httptest.NewRequest(http.MethodGet, "/api/projects/"+project.ID+"/storage-path-favorites", nil), + ) + if getRecorder.Code != http.StatusOK { + t.Fatalf("project favorites get status: %d body=%s", getRecorder.Code, getRecorder.Body.String()) + } + if strings.Contains(getRecorder.Body.String(), "\"prefix\":\"clips \"") { + t.Fatalf("expected project favorites prefix to be normalized, got %s", getRecorder.Body.String()) + } + + projectGetRecorder := httptest.NewRecorder() + handler.ServeHTTP(projectGetRecorder, httptest.NewRequest(http.MethodGet, "/api/projects/"+project.ID, nil)) + if projectGetRecorder.Code != http.StatusOK { + t.Fatalf("project get status: %d body=%s", projectGetRecorder.Code, projectGetRecorder.Body.String()) + } + if strings.Contains(projectGetRecorder.Body.String(), "storagePathFavorites") { + t.Fatalf("expected project get to omit storagePathFavorites, got %s", projectGetRecorder.Body.String()) + } + + projectsListRecorder := httptest.NewRecorder() + handler.ServeHTTP(projectsListRecorder, httptest.NewRequest(http.MethodGet, "/api/projects", nil)) + if projectsListRecorder.Code != http.StatusOK { + t.Fatalf("projects list status: %d body=%s", projectsListRecorder.Code, projectsListRecorder.Body.String()) + } + if strings.Contains(projectsListRecorder.Body.String(), "storagePathFavorites") { + t.Fatalf("expected projects list to omit storagePathFavorites, got %s", projectsListRecorder.Body.String()) + } +} + func TestSettingsAIConfigurationDrivesAIStatus(t *testing.T) { dataStore := store.NewMemoryStore() handler := NewHandler(dataStore, nil) @@ -207,6 +1678,8 @@ func TestSettingsAIConfigurationDrivesAIStatus(t *testing.T) { "k8s": { "endpoint": "https://120.48.161.230:6443", "namespace": "droboticflow", + "workspacePVCName": "droboticflow-workspace", + "workspaceMountPath": "/workspace", "kubeconfig": "apiVersion: v1\nkind: Config" }, "ai": { @@ -215,6 +1688,10 @@ func TestSettingsAIConfigurationDrivesAIStatus(t *testing.T) { "model": "glm-5", "apiKey": "test-key" }, + "businessMongo": { + "uri": "mongodb://business-user:business-pass@business.example:27017/?authSource=admin", + "database": "business_data" + }, "timezone": "Asia/Tokyo" }`) @@ -257,15 +1734,23 @@ func TestSettingsAIConfigurationDrivesAIStatus(t *testing.T) { } var settingsBody struct { K8s struct { - Kubeconfig string `json:"kubeconfig"` - KubeconfigPreview string `json:"kubeconfigPreview"` - HasKubeconfig bool `json:"hasKubeconfig"` + WorkspacePVCName string `json:"workspacePVCName"` + WorkspaceMountPath string `json:"workspaceMountPath"` + Kubeconfig string `json:"kubeconfig"` + KubeconfigPreview string `json:"kubeconfigPreview"` + HasKubeconfig bool `json:"hasKubeconfig"` } `json:"k8s"` AI struct { APIKey string `json:"apiKey"` APIKeyPreview string `json:"apiKeyPreview"` HasAPIKey bool `json:"hasAPIKey"` } `json:"ai"` + BusinessMongo struct { + URI string `json:"uri"` + URIPreview string `json:"uriPreview"` + HasURI bool `json:"hasUri"` + Database string `json:"database"` + } `json:"businessMongo"` Timezone string `json:"timezone"` Storage any `json:"storage"` } @@ -281,6 +1766,12 @@ func TestSettingsAIConfigurationDrivesAIStatus(t *testing.T) { if settingsBody.K8s.KubeconfigPreview == "" { t.Fatalf("expected kubeconfig preview to be returned") } + if settingsBody.K8s.WorkspacePVCName != "droboticflow-workspace" { + t.Fatalf("expected workspace pvc name to roundtrip, got %q", settingsBody.K8s.WorkspacePVCName) + } + if settingsBody.K8s.WorkspaceMountPath != "/workspace" { + t.Fatalf("expected workspace mount path to roundtrip, got %q", settingsBody.K8s.WorkspaceMountPath) + } if settingsBody.AI.APIKey != "" { t.Fatalf("expected settings apiKey to be sanitized") } @@ -290,6 +1781,18 @@ func TestSettingsAIConfigurationDrivesAIStatus(t *testing.T) { if settingsBody.AI.APIKeyPreview != "te****ey" { t.Fatalf("expected apiKey preview to be masked, got %q", settingsBody.AI.APIKeyPreview) } + if settingsBody.BusinessMongo.URI != "" { + t.Fatalf("expected business mongo uri to be sanitized") + } + if !settingsBody.BusinessMongo.HasURI { + t.Fatalf("expected hasUri to be true for business mongo") + } + if settingsBody.BusinessMongo.URIPreview == "" { + t.Fatalf("expected business mongo uri preview to be returned") + } + if settingsBody.BusinessMongo.Database != "business_data" { + t.Fatalf("expected business mongo database to roundtrip, got %q", settingsBody.BusinessMongo.Database) + } if settingsBody.Timezone != "Asia/Tokyo" { t.Fatalf("expected timezone to roundtrip, got %q", settingsBody.Timezone) } @@ -306,6 +1809,7 @@ func TestTemplateExecutionConfigValidationAndPersistence(t *testing.T) { "name":"执行保障模板", "description":"template execution config", "type":"streaming", + "hyperparams":[{"key":"GLOBAL_ENV","value":"template-create"}], "maxConcurrentRuns":2, "nodeTimeoutSeconds":12, "runTimeoutSeconds":34, @@ -338,6 +1842,10 @@ func TestTemplateExecutionConfigValidationAndPersistence(t *testing.T) { if created["retryBackoffSeconds"].(float64) != 56 { t.Fatalf("expected retryBackoffSeconds=56, got %v", created["retryBackoffSeconds"]) } + createdHyperparams := created["hyperparams"].([]any) + if len(createdHyperparams) != 1 || createdHyperparams[0].(map[string]any)["value"].(string) != "template-create" { + t.Fatalf("expected template hyperparams to persist on create, got %#v", created["hyperparams"]) + } templateID := created["id"].(string) getRecorder := httptest.NewRecorder() @@ -357,9 +1865,14 @@ func TestTemplateExecutionConfigValidationAndPersistence(t *testing.T) { if schedule["interval"].(float64) != 90 { t.Fatalf("expected interval=90, got %v", schedule["interval"]) } + loadedHyperparams := loaded["hyperparams"].([]any) + if len(loadedHyperparams) != 1 || loadedHyperparams[0].(map[string]any)["key"].(string) != "GLOBAL_ENV" { + t.Fatalf("expected template hyperparams to survive load, got %#v", loaded["hyperparams"]) + } updatePayload := []byte(`{ "type":"batch", + "hyperparams":[{"key":"GLOBAL_ENV","value":"template-update"},{"key":"SHARED_MODE","value":"true"}], "maxConcurrentRuns":3, "nodeTimeoutSeconds":6, "runTimeoutSeconds":18, @@ -388,6 +1901,10 @@ func TestTemplateExecutionConfigValidationAndPersistence(t *testing.T) { if updated["retryBackoffSeconds"].(float64) != 9 { t.Fatalf("expected retryBackoffSeconds=9 after update, got %v", updated["retryBackoffSeconds"]) } + updatedHyperparams := updated["hyperparams"].([]any) + if len(updatedHyperparams) != 2 || updatedHyperparams[1].(map[string]any)["key"].(string) != "SHARED_MODE" { + t.Fatalf("expected template hyperparams to update, got %#v", updated["hyperparams"]) + } invalidPayload := []byte(`{"maxRetryAttempts":-1}`) invalidRecorder := httptest.NewRecorder() @@ -417,6 +1934,7 @@ func TestWorkflowExecutionConfigValidationAndPersistence(t *testing.T) { "name":"执行配置工作流", "description":"workflow", "type":"batch", + "hyperparams":[{"key":"GLOBAL_ENV","value":"workflow-create"}], "maxConcurrentRuns":2, "nodeTimeoutSeconds":30, "runTimeoutSeconds":120, @@ -449,8 +1967,17 @@ func TestWorkflowExecutionConfigValidationAndPersistence(t *testing.T) { if created.RetryBackoffSeconds != 15 { t.Fatalf("expected retryBackoffSeconds=15, got %d", created.RetryBackoffSeconds) } + if len(created.Hyperparams) != 1 || created.Hyperparams[0].Value != "workflow-create" { + t.Fatalf("expected workflow hyperparams persisted on create, got %#v", created.Hyperparams) + } - updatePayload := []byte(`{"nodeTimeoutSeconds":12,"runTimeoutSeconds":45,"maxRetryAttempts":1,"retryBackoffSeconds":9}`) + updatePayload := []byte(`{ + "hyperparams":[{"key":"GLOBAL_ENV","value":"workflow-update"},{"key":"SHARED_MODE","value":"true"}], + "nodeTimeoutSeconds":12, + "runTimeoutSeconds":45, + "maxRetryAttempts":1, + "retryBackoffSeconds":9 + }`) updateRecorder := httptest.NewRecorder() updateRequest := httptest.NewRequest(http.MethodPut, "/api/workflows/"+created.ID, bytes.NewReader(updatePayload)) handler.ServeHTTP(updateRecorder, updateRequest) @@ -474,6 +2001,9 @@ func TestWorkflowExecutionConfigValidationAndPersistence(t *testing.T) { if updated.RetryBackoffSeconds != 9 { t.Fatalf("expected updated retryBackoffSeconds=9, got %d", updated.RetryBackoffSeconds) } + if len(updated.Hyperparams) != 2 || updated.Hyperparams[1].Key != "SHARED_MODE" { + t.Fatalf("expected workflow hyperparams updated, got %#v", updated.Hyperparams) + } invalidCreatePayload := []byte(`{ "name":"非法工作流", @@ -502,6 +2032,259 @@ func TestWorkflowExecutionConfigValidationAndPersistence(t *testing.T) { } } +func TestCreateAndUpdatePreserveOperatorRunnerImage(t *testing.T) { + t.Setenv("OPERATOR_RUNNER_IMAGE", "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:20260412151842-amd64") + + ctx := context.Background() + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + oldRunnerImage := "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:20260410181157-amd64" + customImage := "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/custom-operator:1" + + operatorCreatePayload := []byte(`{ + "name":"镜像归一化算子", + "type":"builtin", + "category":"IO", + "description":"normalize operator image", + "image":"` + oldRunnerImage + `", + "inputs":[], + "outputs":[{"id":"out-1","name":"output","type":"json"}] + }`) + operatorCreateRecorder := httptest.NewRecorder() + handler.ServeHTTP(operatorCreateRecorder, httptest.NewRequest(http.MethodPost, "/api/operators", bytes.NewReader(operatorCreatePayload))) + if operatorCreateRecorder.Code != http.StatusCreated { + t.Fatalf("operator create status: %d body=%s", operatorCreateRecorder.Code, operatorCreateRecorder.Body.String()) + } + + var createdOperator domain.OperatorRecord + if err := json.Unmarshal(operatorCreateRecorder.Body.Bytes(), &createdOperator); err != nil { + t.Fatalf("decode created operator: %v", err) + } + if createdOperator.Image != oldRunnerImage { + t.Fatalf("expected created operator image %q, got %q", oldRunnerImage, createdOperator.Image) + } + + storedOperator, err := dataStore.GetOperator(ctx, createdOperator.ID) + if err != nil { + t.Fatalf("get stored operator: %v", err) + } + if storedOperator == nil || storedOperator.Image != oldRunnerImage { + t.Fatalf("expected stored operator image %q, got %+v", oldRunnerImage, storedOperator) + } + + operatorUpdatePayload := []byte(`{"image":"` + oldRunnerImage + `"}`) + operatorUpdateRecorder := httptest.NewRecorder() + handler.ServeHTTP(operatorUpdateRecorder, httptest.NewRequest(http.MethodPut, "/api/operators/"+createdOperator.ID, bytes.NewReader(operatorUpdatePayload))) + if operatorUpdateRecorder.Code != http.StatusOK { + t.Fatalf("operator update status: %d body=%s", operatorUpdateRecorder.Code, operatorUpdateRecorder.Body.String()) + } + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "镜像归一化项目", + Description: "normalize workflow images", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflowCreatePayload := []byte(`{ + "name":"镜像归一化工作流", + "description":"normalize workflow node image", + "type":"batch", + "nodes":[ + { + "id":"runner-node", + "type":"operator", + "data":{ + "label":"Runner Node", + "category":"IO", + "description":"runner node", + "image":"` + oldRunnerImage + `", + "inputs":[], + "outputs":[{"id":"out-1","name":"output","type":"json"}] + }, + "position":{"x":0,"y":0} + }, + { + "id":"custom-node", + "type":"operator", + "data":{ + "label":"Custom Node", + "category":"IO", + "description":"custom node", + "image":"` + customImage + `", + "inputs":[], + "outputs":[{"id":"out-2","name":"output","type":"json"}] + }, + "position":{"x":180,"y":0} + } + ], + "edges":[] + }`) + workflowCreateRecorder := httptest.NewRecorder() + handler.ServeHTTP(workflowCreateRecorder, httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(workflowCreatePayload))) + if workflowCreateRecorder.Code != http.StatusCreated { + t.Fatalf("workflow create status: %d body=%s", workflowCreateRecorder.Code, workflowCreateRecorder.Body.String()) + } + + var createdWorkflow domain.WorkflowRecord + if err := json.Unmarshal(workflowCreateRecorder.Body.Bytes(), &createdWorkflow); err != nil { + t.Fatalf("decode created workflow: %v", err) + } + if createdWorkflow.Nodes[0].Data.Image != oldRunnerImage { + t.Fatalf("expected created workflow runner image %q, got %q", oldRunnerImage, createdWorkflow.Nodes[0].Data.Image) + } + if createdWorkflow.Nodes[1].Data.Image != customImage { + t.Fatalf("expected custom workflow image to remain %q, got %q", customImage, createdWorkflow.Nodes[1].Data.Image) + } + + storedWorkflow, err := dataStore.GetWorkflow(ctx, createdWorkflow.ID) + if err != nil { + t.Fatalf("get stored workflow: %v", err) + } + if storedWorkflow == nil || storedWorkflow.Nodes[0].Data.Image != oldRunnerImage { + t.Fatalf("expected stored workflow runner image %q, got %+v", oldRunnerImage, storedWorkflow) + } + if storedWorkflow.Nodes[1].Data.Image != customImage { + t.Fatalf("expected stored custom workflow image to remain %q, got %q", customImage, storedWorkflow.Nodes[1].Data.Image) + } + + workflowUpdatePayload := []byte(`{ + "nodes":[ + { + "id":"runner-node", + "type":"operator", + "data":{ + "label":"Runner Node", + "category":"IO", + "description":"runner node", + "image":"` + oldRunnerImage + `", + "inputs":[], + "outputs":[{"id":"out-1","name":"output","type":"json"}] + }, + "position":{"x":0,"y":0} + }, + { + "id":"custom-node", + "type":"operator", + "data":{ + "label":"Custom Node", + "category":"IO", + "description":"custom node", + "image":"` + customImage + `", + "inputs":[], + "outputs":[{"id":"out-2","name":"output","type":"json"}] + }, + "position":{"x":180,"y":0} + } + ] + }`) + workflowUpdateRecorder := httptest.NewRecorder() + handler.ServeHTTP(workflowUpdateRecorder, httptest.NewRequest(http.MethodPut, "/api/workflows/"+createdWorkflow.ID, bytes.NewReader(workflowUpdatePayload))) + if workflowUpdateRecorder.Code != http.StatusOK { + t.Fatalf("workflow update status: %d body=%s", workflowUpdateRecorder.Code, workflowUpdateRecorder.Body.String()) + } + + templateCreatePayload := []byte(`{ + "name":"镜像归一化模板", + "description":"normalize template node image", + "type":"batch", + "nodes":[ + { + "id":"runner-node", + "type":"operator", + "data":{ + "label":"Runner Node", + "category":"IO", + "description":"runner node", + "image":"` + oldRunnerImage + `", + "inputs":[], + "outputs":[{"id":"out-1","name":"output","type":"json"}] + }, + "position":{"x":0,"y":0} + }, + { + "id":"custom-node", + "type":"operator", + "data":{ + "label":"Custom Node", + "category":"IO", + "description":"custom node", + "image":"` + customImage + `", + "inputs":[], + "outputs":[{"id":"out-2","name":"output","type":"json"}] + }, + "position":{"x":180,"y":0} + } + ], + "edges":[] + }`) + templateCreateRecorder := httptest.NewRecorder() + handler.ServeHTTP(templateCreateRecorder, httptest.NewRequest(http.MethodPost, "/api/templates", bytes.NewReader(templateCreatePayload))) + if templateCreateRecorder.Code != http.StatusCreated { + t.Fatalf("template create status: %d body=%s", templateCreateRecorder.Code, templateCreateRecorder.Body.String()) + } + + var createdTemplate domain.TemplateRecord + if err := json.Unmarshal(templateCreateRecorder.Body.Bytes(), &createdTemplate); err != nil { + t.Fatalf("decode created template: %v", err) + } + if createdTemplate.Nodes[0].Data.Image != oldRunnerImage { + t.Fatalf("expected created template runner image %q, got %q", oldRunnerImage, createdTemplate.Nodes[0].Data.Image) + } + if createdTemplate.Nodes[1].Data.Image != customImage { + t.Fatalf("expected custom template image to remain %q, got %q", customImage, createdTemplate.Nodes[1].Data.Image) + } + + storedTemplate, err := dataStore.GetTemplate(ctx, createdTemplate.ID) + if err != nil { + t.Fatalf("get stored template: %v", err) + } + if storedTemplate == nil || storedTemplate.Nodes[0].Data.Image != oldRunnerImage { + t.Fatalf("expected stored template runner image %q, got %+v", oldRunnerImage, storedTemplate) + } + if storedTemplate.Nodes[1].Data.Image != customImage { + t.Fatalf("expected stored custom template image to remain %q, got %q", customImage, storedTemplate.Nodes[1].Data.Image) + } + + templateUpdatePayload := []byte(`{ + "nodes":[ + { + "id":"runner-node", + "type":"operator", + "data":{ + "label":"Runner Node", + "category":"IO", + "description":"runner node", + "image":"` + oldRunnerImage + `", + "inputs":[], + "outputs":[{"id":"out-1","name":"output","type":"json"}] + }, + "position":{"x":0,"y":0} + }, + { + "id":"custom-node", + "type":"operator", + "data":{ + "label":"Custom Node", + "category":"IO", + "description":"custom node", + "image":"` + customImage + `", + "inputs":[], + "outputs":[{"id":"out-2","name":"output","type":"json"}] + }, + "position":{"x":180,"y":0} + } + ] + }`) + templateUpdateRecorder := httptest.NewRecorder() + handler.ServeHTTP(templateUpdateRecorder, httptest.NewRequest(http.MethodPut, "/api/templates/"+createdTemplate.ID, bytes.NewReader(templateUpdatePayload))) + if templateUpdateRecorder.Code != http.StatusOK { + t.Fatalf("template update status: %d body=%s", templateUpdateRecorder.Code, templateUpdateRecorder.Body.String()) + } +} + func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -562,6 +2345,2007 @@ func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { } } +func TestProjectlessScanWorkflowCanRun(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + Name: "无项目扫描工作流", + Description: "projectless scan workflow", + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindScan, + ScanConfig: &domain.ScanConfig{ + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test", + Bucket: "datastorage", + Prefix: "algo/X-platform-test", + Endpoint: "https://bj.bcebos.com", + AccessKeyID: "ak-projectless-1234", + SecretAccessKey: "sk-projectless-5678", + Enabled: false, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + handler := NewHandler(dataStore, nil) + runRecorder := httptest.NewRecorder() + runRequest := httptest.NewRequest( + http.MethodPost, + "/api/workflows/"+workflow.ID+"/run", + bytes.NewReader([]byte(`{"trigger":"manual"}`)), + ) + handler.ServeHTTP(runRecorder, runRequest) + if runRecorder.Code != http.StatusAccepted { + t.Fatalf("expected 202 for projectless scan workflow run, got %d body=%s", runRecorder.Code, runRecorder.Body.String()) + } + + var run domain.RunRecord + if err := json.Unmarshal(runRecorder.Body.Bytes(), &run); err != nil { + t.Fatalf("decode run response: %v", err) + } + if run.ProjectID != "" || run.ProjectName != "" { + t.Fatalf("expected blank project scope for projectless run, got %+v", run) + } +} + +func TestRunningWorkflowCanBeStoppedGracefully(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "停止项目", + Description: "graceful stop", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "停止工作流", + Description: "stoppable", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "数据源", + Category: "IO", + Description: "source", + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + run.Status = domain.RunStatusRunning + run.StartedAt = nowISO() + run.UpdatedAt = nowISO() + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert running run: %v", err) + } + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/runs/"+run.ID+"/stop", nil) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("expected 200 from graceful stop, got %d body=%s", recorder.Code, recorder.Body.String()) + } + + stored, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get stored run: %v", err) + } + if stored == nil || stored.Status != domain.RunStatus("stopping") { + t.Fatalf("expected run to enter stopping state, got %#v", stored) + } +} + +func TestRunningWorkflowCanBeForceStopped(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + originalFactory := runJobTerminatorFactory + runJobTerminatorFactory = func(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string) (runJobTerminator, error) { + return testRunJobTerminator{}, nil + } + defer func() { + runJobTerminatorFactory = originalFactory + }() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "强制停止项目", + Description: "force stop", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "强制停止工作流", + Description: "force stoppable", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "数据源", + Category: "IO", + Description: "source", + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + run.Status = domain.RunStatusRunning + run.StartedAt = nowISO() + run.UpdatedAt = nowISO() + run.K8sRefs = []domain.K8sRef{{NodeID: "source", JobName: "run-source-1", Namespace: "droboticflow"}} + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert running run: %v", err) + } + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/runs/"+run.ID+"/force-stop", nil) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("expected 200 from force stop, got %d body=%s", recorder.Code, recorder.Body.String()) + } + + stored, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get stored run: %v", err) + } + if stored == nil || stored.Status != domain.RunStatusCancelled { + t.Fatalf("expected run to be cancelled, got %#v", stored) + } + if stored.EndedAt == "" { + t.Fatalf("expected cancelled run to be ended") + } +} + +type testRunJobTerminator struct{} + +func (testRunJobTerminator) TerminateRun(context.Context, string, string) error { return nil } + +func (testRunJobTerminator) TerminateJob(context.Context, string, string) error { return nil } + +func TestProjectWorkflowAPISupportsNativeScanWorkflows(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} { + if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ + Name: name, + Type: "builtin", + Category: "IO", + Description: name, + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}}, + }); err != nil { + t.Fatalf("create operator %s: %v", name, err) + } + } + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "扫描项目", + Description: "native workflow scan api", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + handler := NewHandler(dataStore, nil) + createPayload := []byte(`{ + "name":"X-platform BOS 扫描", + "description":"扫描 workflow 原生创建", + "type":"batch", + "kind":"scan", + "schedule":{"interval":900}, + "scanConfig":{ + "dataSource":"BOS", + "bucket":"datastorage", + "prefix":"algo/X-platform-test/", + "endpoint":"https://bj.bcebos.com", + "accessKeyId":"ak-native-1234", + "secretAccessKey":"sk-native-5678", + "scanMode":"INCREMENTAL", + "incrementalUnit":"DIRECTORY", + "directoryDepth":2, + "pattern":"*.mp4", + "enabled":true + }, + "nodes":[], + "edges":[] + }`) + createRecorder := httptest.NewRecorder() + handler.ServeHTTP( + createRecorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(createPayload)), + ) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create scan workflow status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var created domain.WorkflowRecord + if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created workflow: %v", err) + } + if created.Kind != domain.WorkflowKindScan { + t.Fatalf("expected scan workflow kind, got %s", created.Kind) + } + if created.Type != domain.WorkflowTypeStreaming { + t.Fatalf("expected streaming scan workflow, got %s", created.Type) + } + if created.ScanConfig == nil { + t.Fatalf("expected scan config in response") + } + if created.ScanConfig.AccessKeyID != "ak-native-1234" || created.ScanConfig.SecretAccessKey != "sk-native-5678" { + t.Fatalf("expected raw secrets in response: %+v", created.ScanConfig) + } + if !created.ScanConfig.HasAccessKeyID || !created.ScanConfig.HasSecretAccessKey { + t.Fatalf("expected secret presence flags in response: %+v", created.ScanConfig) + } + if created.ScanConfig.AccessKeyPreview == "" || created.ScanConfig.SecretAccessKeyPreview == "" { + t.Fatalf("expected secret previews in response: %+v", created.ScanConfig) + } + if created.ScanConfig.ScanMode != "INCREMENTAL" { + t.Fatalf("expected incremental scan mode, got %s", created.ScanConfig.ScanMode) + } + if created.ScanConfig.IncrementalUnit != "DIRECTORY" || created.ScanConfig.DirectoryDepth != 2 { + t.Fatalf("expected directory incremental config, got %+v", created.ScanConfig) + } + if len(created.Nodes) != 3 || len(created.Edges) != 2 { + t.Fatalf("expected default scan graph scaffold, got %d nodes and %d edges", len(created.Nodes), len(created.Edges)) + } + + stored, err := dataStore.GetWorkflow(ctx, created.ID) + if err != nil { + t.Fatalf("get stored workflow: %v", err) + } + if stored == nil || stored.ScanConfig == nil { + t.Fatalf("expected stored workflow scan config") + } + if stored.ScanConfig.AccessKeyID != "ak-native-1234" || stored.ScanConfig.SecretAccessKey != "sk-native-5678" { + t.Fatalf("expected secrets preserved in store: %+v", stored.ScanConfig) + } + if stored.ScanConfig.IncrementalUnit != "DIRECTORY" || stored.ScanConfig.DirectoryDepth != 2 { + t.Fatalf("expected stored directory incremental config, got %+v", stored.ScanConfig) + } + if stored.Schedule == nil || stored.Schedule.Interval != 900 { + t.Fatalf("expected schedule interval persisted, got %+v", stored.Schedule) + } + if stored.Schedule.SourcePath != "bos://datastorage/algo/X-platform-test" { + t.Fatalf("unexpected stored source path: %+v", stored.Schedule) + } + + updatePayload := []byte(`{ + "schedule":{"interval":0}, + "scanConfig":{ + "dataSource":"BOS", + "bucket":"datastorage", + "prefix":"algo/X-platform-test", + "endpoint":"https://bj.bcebos.com", + "accessKeyId":"", + "secretAccessKey":"", + "scanMode":"FULL", + "incrementalUnit":"OBJECT", + "directoryDepth":1, + "pattern":"*.json", + "enabled":false + } + }`) + updateRecorder := httptest.NewRecorder() + handler.ServeHTTP( + updateRecorder, + httptest.NewRequest(http.MethodPut, "/api/workflows/"+created.ID, bytes.NewReader(updatePayload)), + ) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("update scan workflow status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + + updated, err := dataStore.GetWorkflow(ctx, created.ID) + if err != nil { + t.Fatalf("get updated workflow: %v", err) + } + if updated == nil || updated.ScanConfig == nil { + t.Fatalf("expected updated scan config") + } + if updated.Type != domain.WorkflowTypeBatch { + t.Fatalf("expected batch after disabling scan schedule, got %s", updated.Type) + } + if updated.ScanConfig.AccessKeyID != "ak-native-1234" || updated.ScanConfig.SecretAccessKey != "sk-native-5678" { + t.Fatalf("expected secrets preserved on generic workflow update: %+v", updated.ScanConfig) + } + if updated.ScanConfig.ScanMode != "FULL" { + t.Fatalf("expected full scan mode after update, got %s", updated.ScanConfig.ScanMode) + } + if updated.ScanConfig.IncrementalUnit != "OBJECT" || updated.ScanConfig.DirectoryDepth != 1 { + t.Fatalf("expected updated incremental config, got %+v", updated.ScanConfig) + } + if updated.Schedule == nil || updated.Schedule.Interval != 0 { + t.Fatalf("expected schedule interval reset to 0, got %+v", updated.Schedule) + } +} + +func TestProjectWorkflowCreateInfersScanConfigFromIncrementalSourceNode(t *testing.T) { + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(context.Background(), domain.CreateProjectInput{ + Name: "增量项目", + Description: "generic incremental dag", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + handler := NewHandler(dataStore, nil) + + createPayload := []byte(`{ + "name":"增量全链路", + "description":"generic incremental dag", + "type":"streaming", + "schedule":{"interval":45}, + "nodes":[ + { + "id":"source", + "type":"operator", + "data":{ + "label":"BOS 增量扫描", + "category":"IO", + "description":"source", + "inputs":[], + "outputs":[{"id":"out-1","name":"manifest","type":"json"}], + "bucket":"wjzh-data-migration", + "prefix":"wjzh-0330-Data-migration", + "pattern":"*.mp4", + "hyperparams":[ + {"key":"DATA_SOURCE","value":"BOS"}, + {"key":"SOURCE_PATH","value":"bos://wjzh-data-migration/wjzh-0330-Data-migration"}, + {"key":"BOS_ENDPOINT","value":"https://bj.bcebos.com"}, + {"key":"BOS_ACCESS_KEY_ID","value":"ak-node-123"}, + {"key":"BOS_SECRET_ACCESS_KEY","value":"sk-node-456"}, + {"key":"SCAN_MODE","value":"INCREMENTAL"} + ] + }, + "position":{"x":0,"y":0} + }, + { + "id":"quality", + "type":"operator", + "data":{ + "label":"视频质检", + "category":"VIDEO", + "description":"quality", + "inputs":[{"id":"in-1","name":"input","type":"json"}], + "outputs":[{"id":"out-1","name":"quality","type":"json"}] + }, + "position":{"x":200,"y":0} + } + ], + "edges":[ + {"id":"source-quality","source":"source","sourceHandle":"out-1","target":"quality","targetHandle":"in-1"} + ] + }`) + + recorder := httptest.NewRecorder() + handler.ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(createPayload)), + ) + if recorder.Code != http.StatusCreated { + t.Fatalf("create incremental workflow status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var created domain.WorkflowRecord + if err := json.Unmarshal(recorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created workflow: %v", err) + } + if created.Kind != domain.WorkflowKindScan { + t.Fatalf("expected scan kind, got %s", created.Kind) + } + if created.Type != domain.WorkflowTypeStreaming { + t.Fatalf("expected streaming workflow, got %s", created.Type) + } + if created.ScanConfig == nil { + t.Fatalf("expected inferred scan config") + } + if created.ScanConfig.SourcePath != "bos://wjzh-data-migration/wjzh-0330-Data-migration" { + t.Fatalf("unexpected inferred source path: %+v", created.ScanConfig) + } + if created.ScanConfig.Bucket != "wjzh-data-migration" || created.ScanConfig.Prefix != "wjzh-0330-Data-migration" { + t.Fatalf("unexpected inferred bucket/prefix: %+v", created.ScanConfig) + } + if created.ScanConfig.AccessKeyID != "ak-node-123" || created.ScanConfig.SecretAccessKey != "sk-node-456" { + t.Fatalf("expected inferred raw secrets, got %+v", created.ScanConfig) + } + if created.ScanConfig.ScanMode != "INCREMENTAL" { + t.Fatalf("expected inferred incremental scan mode, got %+v", created.ScanConfig) + } + if created.MaxConcurrentRuns != 1 { + t.Fatalf("expected inferred scan workflow maxConcurrentRuns=1, got %d", created.MaxConcurrentRuns) + } + if created.Schedule == nil || created.Schedule.Interval != 45 || created.Schedule.SourcePath != "bos://wjzh-data-migration/wjzh-0330-Data-migration" { + t.Fatalf("unexpected inferred schedule: %+v", created.Schedule) + } + if len(created.Nodes) != 2 || len(created.Edges) != 1 { + t.Fatalf("expected original graph to be preserved, got %d nodes %d edges", len(created.Nodes), len(created.Edges)) + } +} + +func TestProjectWorkflowCreatePreservesBOSSourcePathWhenUsingConnectionReference(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "连接引用扫描项目", + Description: "connection + sourcePath", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "WJZH Connection", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + Bucket: "wjzh-data-migration", + AccessKeyID: "ak-conn", + SecretAccessKey: "sk-conn", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + + handler := NewHandler(dataStore, nil) + createPayload := []byte(fmt.Sprintf(`{ + "name":"连接引用增量烟测", + "description":"scan config should keep explicit sourcePath", + "type":"batch", + "nodes":[ + { + "id":"source", + "type":"operator", + "data":{ + "label":"BOS 增量扫描", + "category":"IO", + "description":"source", + "inputs":[], + "outputs":[{"id":"out-1","name":"manifest","type":"json"}], + "connectionId":"%s", + "hyperparams":[ + {"key":"DATA_SOURCE","value":"BOS"}, + {"key":"SOURCE_PATH","value":"bos://wjzh-data-migration/wjzh-0330-Data-migration"}, + {"key":"SCAN_MODE","value":"FULL"} + ] + }, + "position":{"x":0,"y":0} + } + ], + "edges":[] + }`, connection.ID)) + + recorder := httptest.NewRecorder() + handler.ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(createPayload)), + ) + if recorder.Code != http.StatusCreated { + t.Fatalf("create workflow status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var created domain.WorkflowRecord + if err := json.Unmarshal(recorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created workflow: %v", err) + } + if created.ScanConfig == nil { + t.Fatalf("expected scan config") + } + if created.ScanConfig.ConnectionID != connection.ID { + t.Fatalf("expected connection reference to be preserved, got %+v", created.ScanConfig) + } + if created.ScanConfig.SourcePath != "bos://wjzh-data-migration/wjzh-0330-Data-migration" { + t.Fatalf("expected explicit bos sourcePath to be preserved, got %+v", created.ScanConfig) + } + + runRecorder := httptest.NewRecorder() + handler.ServeHTTP( + runRecorder, + httptest.NewRequest(http.MethodPost, "/api/workflows/"+created.ID+"/run", bytes.NewReader([]byte(`{"trigger":"manual"}`))), + ) + if runRecorder.Code != http.StatusCreated && runRecorder.Code != http.StatusAccepted { + t.Fatalf("expected connectionId + sourcePath workflow to run, got %d body=%s", runRecorder.Code, runRecorder.Body.String()) + } +} + +func TestWorkflowUpdateInfersScanConfigFromIncrementalSourceNode(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "增量更新项目", + Description: "generic incremental dag", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "待升级增量工作流", + Description: "incremental update", + Type: domain.WorkflowTypeBatch, + Schedule: &domain.WorkflowSchedule{Interval: 0}, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Category: "IO", + Description: "source", + Bucket: "wjzh-data-migration", + Prefix: "wjzh-0330-Data-migration", + Pattern: "*.json", + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "BOS"}, + {Key: "SOURCE_PATH", Value: "bos://wjzh-data-migration/wjzh-0330-Data-migration"}, + {Key: "BOS_ENDPOINT", Value: "https://bj.bcebos.com"}, + {Key: "BOS_ACCESS_KEY_ID", Value: "ak-update-123"}, + {Key: "BOS_SECRET_ACCESS_KEY", Value: "sk-update-456"}, + {Key: "SCAN_MODE", Value: "FULL"}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + handler := NewHandler(dataStore, nil) + updatePayload := []byte(`{ + "type":"streaming", + "schedule":{"interval":75} + }`) + recorder := httptest.NewRecorder() + handler.ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPut, "/api/workflows/"+workflow.ID, bytes.NewReader(updatePayload)), + ) + if recorder.Code != http.StatusOK { + t.Fatalf("update incremental workflow status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + updated, err := dataStore.GetWorkflow(ctx, workflow.ID) + if err != nil { + t.Fatalf("get updated workflow: %v", err) + } + if updated == nil || updated.ScanConfig == nil { + t.Fatalf("expected inferred scan config after update") + } + if updated.Kind != domain.WorkflowKindScan { + t.Fatalf("expected scan kind after update, got %s", updated.Kind) + } + if updated.Type != domain.WorkflowTypeStreaming { + t.Fatalf("expected streaming type after update, got %s", updated.Type) + } + if updated.ScanConfig.AccessKeyID != "ak-update-123" || updated.ScanConfig.SecretAccessKey != "sk-update-456" { + t.Fatalf("expected secrets derived from node hyperparams, got %+v", updated.ScanConfig) + } + if updated.ScanConfig.ScanMode != "FULL" { + t.Fatalf("expected scan mode derived from node hyperparams, got %+v", updated.ScanConfig) + } + if updated.MaxConcurrentRuns != 1 { + t.Fatalf("expected inferred scan workflow maxConcurrentRuns=1 after update, got %d", updated.MaxConcurrentRuns) + } + if updated.Schedule == nil || updated.Schedule.Interval != 75 || updated.Schedule.SourcePath != "bos://wjzh-data-migration/wjzh-0330-Data-migration" { + t.Fatalf("unexpected updated schedule: %+v", updated.Schedule) + } +} + +func TestTemplateCreateInfersScanConfigFromIncrementalSourceNode(t *testing.T) { + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + createPayload := []byte(`{ + "name":"增量模板", + "description":"template incremental dag", + "type":"streaming", + "schedule":{"interval":30}, + "nodes":[ + { + "id":"source", + "type":"operator", + "data":{ + "label":"BOS 增量扫描", + "category":"IO", + "description":"source", + "inputs":[], + "outputs":[{"id":"out-1","name":"manifest","type":"json"}], + "bucket":"wjzh-data-migration", + "prefix":"wjzh-0330-Data-migration", + "pattern":"*.mp4", + "hyperparams":[ + {"key":"DATA_SOURCE","value":"BOS"}, + {"key":"SOURCE_PATH","value":"bos://wjzh-data-migration/wjzh-0330-Data-migration"}, + {"key":"BOS_ENDPOINT","value":"https://bj.bcebos.com"}, + {"key":"BOS_ACCESS_KEY_ID","value":"ak-template-123"}, + {"key":"BOS_SECRET_ACCESS_KEY","value":"sk-template-456"}, + {"key":"SCAN_MODE","value":"INCREMENTAL"} + ] + }, + "position":{"x":0,"y":0} + } + ], + "edges":[] + }`) + + recorder := httptest.NewRecorder() + handler.ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/templates", bytes.NewReader(createPayload)), + ) + if recorder.Code != http.StatusCreated { + t.Fatalf("create incremental template status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var created domain.TemplateRecord + if err := json.Unmarshal(recorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created template: %v", err) + } + if created.Kind != domain.WorkflowKindScan { + t.Fatalf("expected scan template kind, got %s", created.Kind) + } + if created.Type != domain.WorkflowTypeStreaming { + t.Fatalf("expected streaming template type, got %s", created.Type) + } + if created.ScanConfig == nil { + t.Fatalf("expected inferred template scan config") + } + if created.ScanConfig.AccessKeyID != "ak-template-123" || created.ScanConfig.SecretAccessKey != "sk-template-456" { + t.Fatalf("expected inferred template secrets, got %+v", created.ScanConfig) + } + if created.MaxConcurrentRuns != 1 { + t.Fatalf("expected inferred scan template maxConcurrentRuns=1, got %d", created.MaxConcurrentRuns) + } + if created.Schedule == nil || created.Schedule.Interval != 30 || created.Schedule.SourcePath != "bos://wjzh-data-migration/wjzh-0330-Data-migration" { + t.Fatalf("unexpected template schedule: %+v", created.Schedule) + } +} + +func TestTemplateUpdateInfersScanConfigFromIncrementalSourceNode(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + template, err := dataStore.CreateTemplate(ctx, domain.CreateTemplateInput{ + Name: "待升级增量模板", + Description: "incremental template update", + Type: domain.WorkflowTypeBatch, + Schedule: &domain.WorkflowSchedule{Interval: 0}, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Category: "IO", + Description: "source", + Bucket: "wjzh-data-migration", + Prefix: "wjzh-0330-Data-migration", + Pattern: "*.json", + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "BOS"}, + {Key: "SOURCE_PATH", Value: "bos://wjzh-data-migration/wjzh-0330-Data-migration"}, + {Key: "BOS_ENDPOINT", Value: "https://bj.bcebos.com"}, + {Key: "BOS_ACCESS_KEY_ID", Value: "ak-template-update-123"}, + {Key: "BOS_SECRET_ACCESS_KEY", Value: "sk-template-update-456"}, + {Key: "SCAN_MODE", Value: "INCREMENTAL"}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create template: %v", err) + } + + handler := NewHandler(dataStore, nil) + updatePayload := []byte(`{ + "type":"streaming", + "schedule":{"interval":90} + }`) + recorder := httptest.NewRecorder() + handler.ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPut, "/api/templates/"+template.ID, bytes.NewReader(updatePayload)), + ) + if recorder.Code != http.StatusOK { + t.Fatalf("update incremental template status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + updated, err := dataStore.GetTemplate(ctx, template.ID) + if err != nil { + t.Fatalf("get updated template: %v", err) + } + if updated == nil || updated.ScanConfig == nil { + t.Fatalf("expected inferred template scan config after update") + } + if updated.Kind != domain.WorkflowKindScan { + t.Fatalf("expected scan template kind after update, got %s", updated.Kind) + } + if updated.Type != domain.WorkflowTypeStreaming { + t.Fatalf("expected streaming template type after update, got %s", updated.Type) + } + if updated.ScanConfig.AccessKeyID != "ak-template-update-123" || updated.ScanConfig.SecretAccessKey != "sk-template-update-456" { + t.Fatalf("expected template secrets derived from node hyperparams, got %+v", updated.ScanConfig) + } + if updated.ScanConfig.ScanMode != "INCREMENTAL" { + t.Fatalf("expected template scan mode derived from node hyperparams, got %+v", updated.ScanConfig) + } + if updated.MaxConcurrentRuns != 1 { + t.Fatalf("expected inferred scan template maxConcurrentRuns=1 after update, got %d", updated.MaxConcurrentRuns) + } + if updated.Schedule == nil || updated.Schedule.Interval != 90 || updated.Schedule.SourcePath != "bos://wjzh-data-migration/wjzh-0330-Data-migration" { + t.Fatalf("unexpected updated template schedule: %+v", updated.Schedule) + } +} + +func TestProjectWorkflowCreatePersistsEveryHoursSchedule(t *testing.T) { + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(context.Background(), domain.CreateProjectInput{ + Name: "小时调度项目", + Description: "hourly schedule", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + handler := NewHandler(dataStore, nil) + + payload := []byte(`{ + "name":"小时调度工作流", + "description":"hourly schedule", + "type":"streaming", + "schedule":{ + "mode":"every_hours", + "intervalHours":4 + }, + "nodes":[], + "edges":[] + }`) + + recorder := httptest.NewRecorder() + handler.ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(payload)), + ) + if recorder.Code != http.StatusCreated { + t.Fatalf("create hourly workflow status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var created domain.WorkflowRecord + if err := json.Unmarshal(recorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created workflow: %v", err) + } + if created.Schedule == nil { + t.Fatalf("expected schedule persisted") + } + if created.Schedule.Mode != domain.WorkflowScheduleModeEveryHours { + t.Fatalf("expected every_hours mode, got %+v", created.Schedule) + } + if created.Schedule.IntervalHours != 4 { + t.Fatalf("expected intervalHours=4, got %+v", created.Schedule) + } + if strings.TrimSpace(created.Schedule.AnchorAt) == "" { + t.Fatalf("expected anchorAt populated, got %+v", created.Schedule) + } +} + +func TestTemplateCreatePersistsEveryDaysScheduleDefaultsTimezone(t *testing.T) { + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + payload := []byte(`{ + "name":"按天模板", + "description":"daily schedule", + "type":"streaming", + "schedule":{ + "mode":"every_days_at_time", + "dayInterval":2, + "timeOfDay":"18:30" + }, + "nodes":[], + "edges":[] + }`) + + recorder := httptest.NewRecorder() + handler.ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/templates", bytes.NewReader(payload)), + ) + if recorder.Code != http.StatusCreated { + t.Fatalf("create daily template status: %d body=%s", recorder.Code, recorder.Body.String()) + } + + var created domain.TemplateRecord + if err := json.Unmarshal(recorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created template: %v", err) + } + if created.Schedule == nil { + t.Fatalf("expected schedule persisted") + } + if created.Schedule.Mode != domain.WorkflowScheduleModeEveryDaysAtTime { + t.Fatalf("expected every_days_at_time mode, got %+v", created.Schedule) + } + if created.Schedule.DayInterval != 2 || created.Schedule.TimeOfDay != "18:30" { + t.Fatalf("unexpected daily schedule payload: %+v", created.Schedule) + } + if created.Schedule.Timezone != domain.DefaultWorkflowScheduleTimezone { + t.Fatalf("expected default timezone %q, got %+v", domain.DefaultWorkflowScheduleTimezone, created.Schedule) + } + if strings.TrimSpace(created.Schedule.AnchorAt) == "" { + t.Fatalf("expected anchorAt populated, got %+v", created.Schedule) + } +} + +func TestProjectWorkflowCreateRejectsInvalidEveryDaysSchedule(t *testing.T) { + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(context.Background(), domain.CreateProjectInput{ + Name: "非法调度项目", + Description: "invalid schedule", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + handler := NewHandler(dataStore, nil) + + payload := []byte(`{ + "name":"非法按天工作流", + "description":"invalid daily schedule", + "type":"streaming", + "schedule":{ + "mode":"every_days_at_time", + "dayInterval":1, + "timeOfDay":"25:61" + }, + "nodes":[], + "edges":[] + }`) + + recorder := httptest.NewRecorder() + handler.ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(payload)), + ) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected invalid daily schedule rejected, got %d body=%s", recorder.Code, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "schedule.timeOfDay") { + t.Fatalf("expected timeOfDay validation issue, body=%s", recorder.Body.String()) + } +} + +func TestStorageConnectionsAPIStoresSecretsMasked(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + createPayload := []byte(`{ + "name":"BOS 主连接", + "description":"扫描用 BOS 连接", + "provider":"BOS", + "endpoint":"https://bj.bcebos.com", + "bucket":"datastorage", + "accessKeyId":"ak-connection-1234", + "secretAccessKey":"sk-connection-5678" + }`) + createRecorder := httptest.NewRecorder() + handler.ServeHTTP( + createRecorder, + httptest.NewRequest(http.MethodPost, "/api/storage/connections", bytes.NewReader(createPayload)), + ) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create storage connection status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var created map[string]any + if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created connection: %v", err) + } + if _, ok := created["accessKeyId"]; ok { + t.Fatalf("expected masked accessKeyId to be omitted: %#v", created) + } + if _, ok := created["secretAccessKey"]; ok { + t.Fatalf("expected masked secretAccessKey to be omitted: %#v", created) + } + if created["hasAccessKeyId"] != true || created["hasSecretAccessKey"] != true { + t.Fatalf("expected secret presence flags in response: %#v", created) + } + if created["accessKeyPreview"] == "" || created["secretAccessKeyPreview"] == "" { + t.Fatalf("expected secret previews in response: %#v", created) + } + + stored, err := dataStore.GetStorageConnection(ctx, created["id"].(string)) + if err != nil { + t.Fatalf("get stored storage connection: %v", err) + } + if stored == nil { + t.Fatalf("expected stored connection") + } + if stored.Bucket != "datastorage" { + t.Fatalf("expected bucket preserved in store: %#v", stored) + } + if stored.AccessKeyID != "ak-connection-1234" || stored.SecretAccessKey != "sk-connection-5678" { + t.Fatalf("expected real secrets preserved in store: %#v", stored) + } + + updatePayload := []byte(`{ + "description":"更新后的连接描述", + "endpoint":"https://gz.bcebos.com", + "bucket":"archive", + "accessKeyId":"", + "secretAccessKey":"" + }`) + updateRecorder := httptest.NewRecorder() + handler.ServeHTTP( + updateRecorder, + httptest.NewRequest(http.MethodPut, "/api/storage/connections/"+created["id"].(string), bytes.NewReader(updatePayload)), + ) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("update storage connection status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + + updated, err := dataStore.GetStorageConnection(ctx, created["id"].(string)) + if err != nil { + t.Fatalf("get updated storage connection: %v", err) + } + if updated == nil { + t.Fatalf("expected updated connection") + } + if updated.Description != "更新后的连接描述" || updated.Endpoint != "https://gz.bcebos.com" || updated.Bucket != "archive" { + t.Fatalf("unexpected updated connection: %#v", updated) + } + if updated.AccessKeyID != "ak-connection-1234" || updated.SecretAccessKey != "sk-connection-5678" { + t.Fatalf("expected blank update payload to preserve secrets: %#v", updated) + } + + listRecorder := httptest.NewRecorder() + handler.ServeHTTP(listRecorder, httptest.NewRequest(http.MethodGet, "/api/storage/connections", nil)) + if listRecorder.Code != http.StatusOK { + t.Fatalf("list storage connections status: %d body=%s", listRecorder.Code, listRecorder.Body.String()) + } + var listBody struct { + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(listRecorder.Body.Bytes(), &listBody); err != nil { + t.Fatalf("decode storage connection list: %v", err) + } + if len(listBody.Items) != 1 { + t.Fatalf("expected 1 storage connection, got %d", len(listBody.Items)) + } +} + +func TestMongoConnectionsAPIStoresSecretsMasked(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + createPayload := []byte(`{ + "name":"业务 MongoDB", + "description":"测试业务库", + "uri":"mongodb://business-user:business-pass@business.example:27017", + "database":"business_data" + }`) + createRecorder := httptest.NewRecorder() + handler.ServeHTTP( + createRecorder, + httptest.NewRequest(http.MethodPost, "/api/mongo/connections", bytes.NewReader(createPayload)), + ) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create mongo connection status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var created map[string]any + if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created mongo connection: %v", err) + } + if _, ok := created["uri"]; ok { + t.Fatalf("expected masked uri to be omitted: %#v", created) + } + if created["hasUri"] != true || created["uriPreview"] == "" { + t.Fatalf("expected uri presence metadata: %#v", created) + } + + stored, err := dataStore.GetMongoConnection(ctx, created["id"].(string)) + if err != nil { + t.Fatalf("get stored mongo connection: %v", err) + } + if stored == nil { + t.Fatalf("expected stored mongo connection") + } + if stored.URI != "mongodb://business-user:business-pass@business.example:27017" { + t.Fatalf("expected real uri preserved in store: %#v", stored) + } + + updatePayload := []byte(`{ + "description":"更新后的测试业务库", + "uri":"", + "database":"business_data_v2" + }`) + updateRecorder := httptest.NewRecorder() + handler.ServeHTTP( + updateRecorder, + httptest.NewRequest(http.MethodPut, "/api/mongo/connections/"+created["id"].(string), bytes.NewReader(updatePayload)), + ) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("update mongo connection status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + + updated, err := dataStore.GetMongoConnection(ctx, created["id"].(string)) + if err != nil { + t.Fatalf("get updated mongo connection: %v", err) + } + if updated == nil { + t.Fatalf("expected updated mongo connection") + } + if updated.Description != "更新后的测试业务库" || updated.Database != "business_data_v2" { + t.Fatalf("unexpected updated mongo connection: %#v", updated) + } + if updated.URI != "mongodb://business-user:business-pass@business.example:27017" { + t.Fatalf("expected blank update payload to preserve uri: %#v", updated) + } + + listRecorder := httptest.NewRecorder() + handler.ServeHTTP(listRecorder, httptest.NewRequest(http.MethodGet, "/api/mongo/connections", nil)) + if listRecorder.Code != http.StatusOK { + t.Fatalf("list mongo connections status: %d body=%s", listRecorder.Code, listRecorder.Body.String()) + } + var listBody struct { + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(listRecorder.Body.Bytes(), &listBody); err != nil { + t.Fatalf("decode mongo connection list: %v", err) + } + if len(listBody.Items) != 1 { + t.Fatalf("expected 1 mongo connection, got %d", len(listBody.Items)) + } +} + +func TestStorageBrowserSupportsConnectionID(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 主连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + AccessKeyID: "ak-browser-1234", + SecretAccessKey: "sk-browser-5678", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + + originalBucketsResolver := storageConnectionBucketsResolver + originalObjectsResolver := storageConnectionObjectsResolver + storageConnectionBucketsResolver = func(ctx context.Context, item *domain.StorageConnectionRecord) ([]string, error) { + if item == nil || item.ID != connection.ID { + t.Fatalf("unexpected storage connection for buckets: %#v", item) + } + return []string{"datastorage", "incoming"}, nil + } + storageConnectionObjectsResolver = func(ctx context.Context, item *domain.StorageConnectionRecord, bucket string, prefix string) ([]string, error) { + if item == nil || item.ID != connection.ID { + t.Fatalf("unexpected storage connection for objects: %#v", item) + } + if bucket != "datastorage" || prefix != "algo/X-platform-test" { + t.Fatalf("unexpected bucket/prefix: %s %s", bucket, prefix) + } + return []string{"algo/X-platform-test/BJ_001/", "algo/X-platform-test/meta.json"}, nil + } + t.Cleanup(func() { + storageConnectionBucketsResolver = originalBucketsResolver + storageConnectionObjectsResolver = originalObjectsResolver + }) + + handler := NewHandler(dataStore, nil) + + bucketsRecorder := httptest.NewRecorder() + handler.ServeHTTP( + bucketsRecorder, + httptest.NewRequest(http.MethodGet, "/api/storage/buckets?connectionId="+connection.ID, nil), + ) + if bucketsRecorder.Code != http.StatusOK { + t.Fatalf("connection buckets status: %d body=%s", bucketsRecorder.Code, bucketsRecorder.Body.String()) + } + + objectsRecorder := httptest.NewRecorder() + handler.ServeHTTP( + objectsRecorder, + httptest.NewRequest(http.MethodGet, "/api/storage/objects?connectionId="+connection.ID+"&bucket=datastorage&prefix=algo/X-platform-test", nil), + ) + if objectsRecorder.Code != http.StatusOK { + t.Fatalf("connection objects status: %d body=%s", objectsRecorder.Code, objectsRecorder.Body.String()) + } +} + +type fakeObjectStorageBrowser struct { + buckets map[string]struct{} + objects map[string]domain.ObjectStorageObjectSummary + content map[string][]byte + readErr map[string]error + readCalls map[string]int + readPreviewCalls map[string]int + lastPreviewMaxLen map[string]int64 +} + +func newFakeObjectStorageBrowser() *fakeObjectStorageBrowser { + return &fakeObjectStorageBrowser{ + buckets: map[string]struct{}{ + "datastorage": {}, + }, + objects: map[string]domain.ObjectStorageObjectSummary{ + "datastorage|algo/demo.json": { + Bucket: "datastorage", + Key: "algo/demo.json", + Name: "demo.json", + Kind: "json", + SizeBytes: 13, + ContentType: "application/json", + }, + }, + content: map[string][]byte{ + "datastorage|algo/demo.json": []byte(`{"demo":true}`), + }, + readErr: map[string]error{}, + readCalls: map[string]int{}, + readPreviewCalls: map[string]int{}, + lastPreviewMaxLen: map[string]int64{}, + } +} + +func (browser *fakeObjectStorageBrowser) ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error) { + items := make([]domain.ObjectStorageBucketSummary, 0, len(browser.buckets)) + for bucket := range browser.buckets { + items = append(items, domain.ObjectStorageBucketSummary{Name: bucket}) + } + sort.Slice(items, func(left int, right int) bool { return items[left].Name < items[right].Name }) + return items, nil +} + +func (browser *fakeObjectStorageBrowser) ListObjects(_ context.Context, bucket string, prefix string) ([]domain.ObjectStorageObjectSummary, error) { + items := make([]domain.ObjectStorageObjectSummary, 0) + normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/") + for _, item := range browser.objects { + if item.Bucket != bucket { + continue + } + if normalizedPrefix != "" && !strings.HasPrefix(item.Key, normalizedPrefix) { + continue + } + items = append(items, item) + } + sort.Slice(items, func(left int, right int) bool { return items[left].Key < items[right].Key }) + return items, nil +} + +func (browser *fakeObjectStorageBrowser) CreateBucket(_ context.Context, bucket string) (domain.ObjectStorageBucketSummary, error) { + browser.buckets[bucket] = struct{}{} + return domain.ObjectStorageBucketSummary{Name: bucket}, nil +} + +func (browser *fakeObjectStorageBrowser) DeleteBucket(_ context.Context, bucket string) error { + delete(browser.buckets, bucket) + return nil +} + +func (browser *fakeObjectStorageBrowser) PutObject(_ context.Context, bucket string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) { + browser.buckets[bucket] = struct{}{} + item := domain.ObjectStorageObjectSummary{ + Bucket: bucket, + Key: key, + Name: filepath.Base(key), + Kind: "file", + SizeBytes: int64(len(payload)), + ContentType: contentType, + } + if strings.HasSuffix(strings.ToLower(key), ".json") { + item.Kind = "json" + } + browser.objects[bucket+"|"+key] = item + browser.content[bucket+"|"+key] = append([]byte(nil), payload...) + return item, nil +} + +func (browser *fakeObjectStorageBrowser) RenameObject(_ context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) { + item := browser.objects[bucket+"|"+key] + delete(browser.objects, bucket+"|"+key) + item.Key = newKey + item.Name = filepath.Base(newKey) + browser.objects[bucket+"|"+newKey] = item + browser.content[bucket+"|"+newKey] = browser.content[bucket+"|"+key] + delete(browser.content, bucket+"|"+key) + return item, nil +} + +func (browser *fakeObjectStorageBrowser) DeleteObject(_ context.Context, bucket string, key string) error { + delete(browser.objects, bucket+"|"+key) + delete(browser.content, bucket+"|"+key) + return nil +} + +func (browser *fakeObjectStorageBrowser) DeletePrefix(_ context.Context, bucket string, prefix string) error { + normalized := strings.Trim(strings.TrimSpace(prefix), "/") + for objectKey := range browser.objects { + if !strings.HasPrefix(objectKey, bucket+"|") { + continue + } + key := strings.TrimPrefix(objectKey, bucket+"|") + if key == normalized || strings.HasPrefix(key, normalized+"/") { + delete(browser.objects, objectKey) + delete(browser.content, objectKey) + } + } + return nil +} + +func (browser *fakeObjectStorageBrowser) GetObjectSummary(_ context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) { + item, ok := browser.objects[bucket+"|"+key] + if !ok { + return domain.ObjectStorageObjectSummary{}, os.ErrNotExist + } + return item, nil +} + +func (browser *fakeObjectStorageBrowser) ReadObject(_ context.Context, bucket string, key string) ([]byte, string, error) { + browser.readCalls[bucket+"|"+key]++ + if err, ok := browser.readErr[bucket+"|"+key]; ok && err != nil { + return nil, "", err + } + item, ok := browser.objects[bucket+"|"+key] + if !ok { + return nil, "", os.ErrNotExist + } + payload, ok := browser.content[bucket+"|"+key] + if !ok { + return nil, "", os.ErrNotExist + } + return append([]byte(nil), payload...), item.ContentType, nil +} + +func (browser *fakeObjectStorageBrowser) ReadObjectPreview(_ context.Context, bucket string, key string, maxBytes int64) ([]byte, string, error) { + browser.readPreviewCalls[bucket+"|"+key]++ + browser.lastPreviewMaxLen[bucket+"|"+key] = maxBytes + if err, ok := browser.readErr[bucket+"|"+key]; ok && err != nil { + return nil, "", err + } + item, ok := browser.objects[bucket+"|"+key] + if !ok { + return nil, "", os.ErrNotExist + } + payload, ok := browser.content[bucket+"|"+key] + if !ok { + return nil, "", os.ErrNotExist + } + if maxBytes > 0 && int64(len(payload)) > maxBytes { + payload = payload[:maxBytes] + } + return append([]byte(nil), payload...), item.ContentType, nil +} + +func (browser *fakeObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucket string, key string) (string, error) { + if _, ok := browser.objects[bucket+"|"+key]; !ok { + return "", os.ErrNotExist + } + return "https://example.com/preview/" + url.PathEscape(bucket) + "/" + url.PathEscape(key), nil +} + +func TestObjectStorageBrowserSupportsConnectionCRUDAndPreview(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 管理连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + Bucket: "datastorage", + AccessKeyID: "ak-storage-browser", + SecretAccessKey: "sk-storage-browser", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + + fakeBrowser := newFakeObjectStorageBrowser() + originalResolver := storageConnectionBrowserResolver + storageConnectionBrowserResolver = func(_ context.Context, item *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + if item == nil || item.ID != connection.ID { + t.Fatalf("unexpected storage connection: %#v", item) + } + return fakeBrowser, func() {}, nil + } + t.Cleanup(func() { + storageConnectionBrowserResolver = originalResolver + }) + + handler := NewHandler(dataStore, nil) + + overviewRecorder := httptest.NewRecorder() + handler.ServeHTTP( + overviewRecorder, + httptest.NewRequest(http.MethodGet, "/api/object-storage?connectionId="+connection.ID, nil), + ) + if overviewRecorder.Code != http.StatusOK { + t.Fatalf("object storage overview status: %d body=%s", overviewRecorder.Code, overviewRecorder.Body.String()) + } + + createObjectRecorder := httptest.NewRecorder() + handler.ServeHTTP( + createObjectRecorder, + httptest.NewRequest(http.MethodPost, "/api/object-storage/objects?connectionId="+connection.ID, strings.NewReader(`{"key":"algo/created.json","content":"{\"ok\":true}","contentType":"application/json"}`)), + ) + if createObjectRecorder.Code != http.StatusOK { + t.Fatalf("create object status: %d body=%s", createObjectRecorder.Code, createObjectRecorder.Body.String()) + } + + renameRecorder := httptest.NewRecorder() + handler.ServeHTTP( + renameRecorder, + httptest.NewRequest(http.MethodPatch, "/api/object-storage/objects?connectionId="+connection.ID, strings.NewReader(`{"key":"algo/created.json","newKey":"algo/renamed.json"}`)), + ) + if renameRecorder.Code != http.StatusOK { + t.Fatalf("rename object status: %d body=%s", renameRecorder.Code, renameRecorder.Body.String()) + } + + objectsRecorder := httptest.NewRecorder() + handler.ServeHTTP( + objectsRecorder, + httptest.NewRequest(http.MethodGet, "/api/object-storage/objects?connectionId="+connection.ID+"&prefix=algo", nil), + ) + if objectsRecorder.Code != http.StatusOK { + t.Fatalf("list objects status: %d body=%s", objectsRecorder.Code, objectsRecorder.Body.String()) + } + + contentRecorder := httptest.NewRecorder() + handler.ServeHTTP( + contentRecorder, + httptest.NewRequest(http.MethodGet, "/api/object-storage/content?connectionId="+connection.ID+"&key=algo/renamed.json", nil), + ) + if contentRecorder.Code != http.StatusOK { + t.Fatalf("object content status: %d body=%s", contentRecorder.Code, contentRecorder.Body.String()) + } + if got := strings.TrimSpace(contentRecorder.Header().Get("Content-Type")); !strings.HasPrefix(got, "application/json") { + t.Fatalf("unexpected content type: %s", got) + } + + var multipartBody bytes.Buffer + multipartWriter := multipart.NewWriter(&multipartBody) + fileWriter, err := multipartWriter.CreateFormFile("file", "hello.txt") + if err != nil { + t.Fatalf("create multipart file: %v", err) + } + if _, err := io.WriteString(fileWriter, "hello object storage"); err != nil { + t.Fatalf("write multipart file: %v", err) + } + if err := multipartWriter.Close(); err != nil { + t.Fatalf("close multipart writer: %v", err) + } + uploadRequest := httptest.NewRequest(http.MethodPost, "/api/object-storage/objects/upload?connectionId="+connection.ID+"&key=algo/nested/uploaded.txt", &multipartBody) + uploadRequest.Header.Set("Content-Type", multipartWriter.FormDataContentType()) + uploadRecorder := httptest.NewRecorder() + handler.ServeHTTP(uploadRecorder, uploadRequest) + if uploadRecorder.Code != http.StatusOK { + t.Fatalf("upload object status: %d body=%s", uploadRecorder.Code, uploadRecorder.Body.String()) + } + + deleteObjectRecorder := httptest.NewRecorder() + handler.ServeHTTP( + deleteObjectRecorder, + httptest.NewRequest(http.MethodDelete, "/api/object-storage/objects?connectionId="+connection.ID+"&key=algo/renamed.json", nil), + ) + if deleteObjectRecorder.Code != http.StatusOK { + t.Fatalf("delete object status: %d body=%s", deleteObjectRecorder.Code, deleteObjectRecorder.Body.String()) + } + + deleteDirectoryRecorder := httptest.NewRecorder() + handler.ServeHTTP( + deleteDirectoryRecorder, + httptest.NewRequest(http.MethodDelete, "/api/object-storage/objects?connectionId="+connection.ID+"&key=algo/nested&recursive=true", nil), + ) + if deleteDirectoryRecorder.Code != http.StatusOK { + t.Fatalf("delete directory status: %d body=%s", deleteDirectoryRecorder.Code, deleteDirectoryRecorder.Body.String()) + } + if _, ok := fakeBrowser.objects["datastorage|algo/nested/uploaded.txt"]; ok { + t.Fatalf("expected recursive directory delete to remove nested object") + } +} + +func TestObjectStorageContentStreamsVideoPreviewWithoutReadingWholeObject(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 管理连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + Bucket: "datastorage", + AccessKeyID: "ak-storage-browser", + SecretAccessKey: "sk-storage-browser", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + + fakeBrowser := newFakeObjectStorageBrowser() + videoKey := "algo/demo.mp4" + fakeBrowser.objects["datastorage|"+videoKey] = domain.ObjectStorageObjectSummary{ + Bucket: "datastorage", + Key: videoKey, + Name: "demo.mp4", + Kind: "video", + SizeBytes: 7 << 30, + ContentType: "video/mp4", + } + fakeBrowser.readErr["datastorage|"+videoKey] = fmt.Errorf("read should not be called for redirected video preview") + + originalResolver := storageConnectionBrowserResolver + storageConnectionBrowserResolver = func(ctx context.Context, item *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + if item == nil || item.ID != connection.ID { + t.Fatalf("unexpected storage connection: %#v", item) + } + return fakeBrowser, func() {}, nil + } + defer func() { storageConnectionBrowserResolver = originalResolver }() + + previewFile, err := os.CreateTemp(t.TempDir(), "object-storage-preview-*.mp4") + if err != nil { + t.Fatalf("create preview file: %v", err) + } + if _, err := previewFile.Write([]byte("preview-video")); err != nil { + t.Fatalf("write preview file: %v", err) + } + if err := previewFile.Close(); err != nil { + t.Fatalf("close preview file: %v", err) + } + + originalPreviewResolver := sourceBrowserVideoPreviewPath + sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { + if request.RedirectURL == "" { + t.Fatalf("expected redirect url for video preview request") + } + if len(request.Payload) != 0 { + t.Fatalf("expected video preview to avoid in-memory payload reads") + } + return previewFile.Name(), nil + } + defer func() { sourceBrowserVideoPreviewPath = originalPreviewResolver }() + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + request := httptest.NewRequest( + http.MethodGet, + "/api/object-storage/content?connectionId="+url.QueryEscape(connection.ID)+"&bucket=datastorage&key="+url.QueryEscape(videoKey)+"&transcode=browser", + nil, + ) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("video preview status: %d body=%s", recorder.Code, recorder.Body.String()) + } + if contentType := recorder.Header().Get("Content-Type"); contentType != "video/mp4" { + t.Fatalf("unexpected content type: %s", contentType) + } + if body := recorder.Body.String(); body != "preview-video" { + t.Fatalf("unexpected preview body: %q", body) + } +} + +func TestObjectStorageContentPreviewTruncatesLargeJSONObjects(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 管理连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + Bucket: "datastorage", + AccessKeyID: "ak-storage-browser", + SecretAccessKey: "sk-storage-browser", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + + fakeBrowser := newFakeObjectStorageBrowser() + largeKey := "algo/large.json" + largePayload := []byte(strings.Repeat(`{"imu":1}`+"\n", 40000)) + fakeBrowser.objects["datastorage|"+largeKey] = domain.ObjectStorageObjectSummary{ + Bucket: "datastorage", + Key: largeKey, + Name: "large.json", + Kind: "json", + SizeBytes: int64(len(largePayload)), + ContentType: "application/json", + } + fakeBrowser.content["datastorage|"+largeKey] = largePayload + + originalResolver := storageConnectionBrowserResolver + storageConnectionBrowserResolver = func(_ context.Context, item *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + if item == nil || item.ID != connection.ID { + t.Fatalf("unexpected storage connection: %#v", item) + } + return fakeBrowser, func() {}, nil + } + defer func() { storageConnectionBrowserResolver = originalResolver }() + + handler := NewHandler(dataStore, nil) + recorder := httptest.NewRecorder() + request := httptest.NewRequest( + http.MethodGet, + "/api/object-storage/content?connectionId="+url.QueryEscape(connection.ID)+"&bucket=datastorage&key="+url.QueryEscape(largeKey)+"&preview=true", + nil, + ) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("content preview status: %d body=%s", recorder.Code, recorder.Body.String()) + } + if fakeBrowser.readCalls["datastorage|"+largeKey] != 0 { + t.Fatalf("expected no full-object reads, got %d", fakeBrowser.readCalls["datastorage|"+largeKey]) + } + if fakeBrowser.readPreviewCalls["datastorage|"+largeKey] != 1 { + t.Fatalf("expected one preview read, got %d", fakeBrowser.readPreviewCalls["datastorage|"+largeKey]) + } + if got := fakeBrowser.lastPreviewMaxLen["datastorage|"+largeKey]; got != objectStoragePartialPreviewBytes { + t.Fatalf("preview max bytes = %d, want %d", got, objectStoragePartialPreviewBytes) + } + if recorder.Header().Get(objectStoragePreviewTruncatedHeader) != "true" { + t.Fatalf("expected truncated header true, got %q", recorder.Header().Get(objectStoragePreviewTruncatedHeader)) + } + if got := recorder.Header().Get(objectStoragePreviewOriginalSizeHeader); got != fmt.Sprintf("%d", len(largePayload)) { + t.Fatalf("unexpected original size header: %q", got) + } + if recorder.Body.Len() != int(objectStoragePartialPreviewBytes) { + t.Fatalf("preview body length = %d, want %d", recorder.Body.Len(), objectStoragePartialPreviewBytes) + } +} + +func TestProjectWorkflowAPICreatesConnectionBackedScanWorkflow(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} { + if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ + Name: name, + Type: "builtin", + Category: "IO", + Description: name, + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}}, + }); err != nil { + t.Fatalf("create operator %s: %v", name, err) + } + } + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "连接型扫描项目", + Description: "scan with storage connection", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 主连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + AccessKeyID: "ak-conn-1234", + SecretAccessKey: "sk-conn-5678", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + + handler := NewHandler(dataStore, nil) + createPayload := []byte(`{ + "name":"连接型 BOS 扫描", + "description":"使用 connectionId 的扫描 workflow", + "type":"batch", + "kind":"scan", + "schedule":{"interval":600}, + "scanConfig":{ + "dataSource":"BOS", + "connectionId":"` + connection.ID + `", + "bucket":"datastorage", + "prefix":"algo/X-platform-test/", + "scanMode":"INCREMENTAL", + "incrementalUnit":"DIRECTORY", + "directoryDepth":3, + "enabled":true + }, + "nodes":[], + "edges":[] + }`) + createRecorder := httptest.NewRecorder() + handler.ServeHTTP( + createRecorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(createPayload)), + ) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create connection-backed scan workflow status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var created domain.WorkflowRecord + if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created workflow: %v", err) + } + if created.ScanConfig == nil || created.ScanConfig.ConnectionID != connection.ID { + t.Fatalf("expected connection-backed scan config, got %+v", created.ScanConfig) + } + if created.ScanConfig.IncrementalUnit != "DIRECTORY" || created.ScanConfig.DirectoryDepth != 3 { + t.Fatalf("expected connection-backed directory incremental config, got %+v", created.ScanConfig) + } + if created.ScanConfig.AccessKeyID != "" || created.ScanConfig.SecretAccessKey != "" { + t.Fatalf("expected workflow response not to expose secrets: %+v", created.ScanConfig) + } + if created.Type != domain.WorkflowTypeStreaming { + t.Fatalf("expected enabled connection-backed scan workflow to be streaming, got %s", created.Type) + } + + stored, err := dataStore.GetWorkflow(ctx, created.ID) + if err != nil { + t.Fatalf("get stored workflow: %v", err) + } + if stored == nil || stored.ScanConfig == nil { + t.Fatalf("expected stored scan config") + } + if stored.ScanConfig.ConnectionID != connection.ID { + t.Fatalf("expected stored connection id, got %+v", stored.ScanConfig) + } + if stored.ScanConfig.IncrementalUnit != "DIRECTORY" || stored.ScanConfig.DirectoryDepth != 3 { + t.Fatalf("expected stored connection-backed directory incremental config, got %+v", stored.ScanConfig) + } + if stored.ScanConfig.AccessKeyID != "" || stored.ScanConfig.SecretAccessKey != "" { + t.Fatalf("expected workflow store to keep connection ref instead of duplicating secrets: %+v", stored.ScanConfig) + } + + runRecorder := httptest.NewRecorder() + handler.ServeHTTP( + runRecorder, + httptest.NewRequest(http.MethodPost, "/api/workflows/"+created.ID+"/run", bytes.NewReader([]byte(`{"trigger":"manual"}`))), + ) + if runRecorder.Code != http.StatusAccepted { + t.Fatalf("run connection-backed scan workflow status: %d body=%s", runRecorder.Code, runRecorder.Body.String()) + } +} + +func TestProjectWorkflowAPIRejectsUnknownNodeStorageConnection(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "连接校验项目", + Description: "reject invalid node connection", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + handler := NewHandler(dataStore, nil) + createPayload := []byte(`{ + "name":"无效连接工作流", + "description":"节点 connectionId 应该校验", + "type":"batch", + "nodes":[ + { + "id":"node-oss", + "type":"operator", + "data":{ + "label":"OSS 数据源", + "category":"IO", + "description":"source", + "connectionId":"connection-missing", + "inputs":[], + "outputs":[], + "hyperparams":[] + }, + "position":{"x":0,"y":0} + } + ], + "edges":[] + }`) + createRecorder := httptest.NewRecorder() + handler.ServeHTTP( + createRecorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(createPayload)), + ) + if createRecorder.Code != http.StatusBadRequest { + t.Fatalf("expected validation failure, got %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + if !strings.Contains(createRecorder.Body.String(), "connectionId") { + t.Fatalf("expected connectionId issue, body=%s", createRecorder.Body.String()) + } +} + +func duplicateInputEdgeWorkflowNodes() []domain.WorkflowNode { + return []domain.WorkflowNode{ + { + ID: "source-a", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "上游 A", + Category: "IO", + Description: "source a", + Inputs: []domain.PortDefinition{}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "source-b", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "上游 B", + Category: "IO", + Description: "source b", + Inputs: []domain.PortDefinition{}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 120, Y: 0}, + }, + { + ID: "upload", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "结果上传", + Category: "IO", + Description: "upload", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "result", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "uploaded", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 240, Y: 0}, + }, + } +} + +func duplicateInputEdgeWorkflowEdges() []domain.WorkflowEdge { + return []domain.WorkflowEdge{ + {ID: "edge-a", Source: "source-a", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, + {ID: "edge-b", Source: "source-b", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, + } +} + +func TestProjectWorkflowAPIRejectsDuplicateIncomingEdgesPerInputPort(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "重复输入端口项目", + Description: "reject duplicate input edges", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + payload, err := json.Marshal(domain.CreateWorkflowInput{ + Name: "非法上传工作流", + Description: "duplicate input edges", + Type: domain.WorkflowTypeBatch, + Nodes: duplicateInputEdgeWorkflowNodes(), + Edges: duplicateInputEdgeWorkflowEdges(), + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + recorder := httptest.NewRecorder() + NewHandler(dataStore, nil).ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(payload)), + ) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected validation failure, got %d body=%s", recorder.Code, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "每个输入端口只能连接一条上游连线") { + t.Fatalf("expected duplicate input edge validation, body=%s", recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "上传算子") { + t.Fatalf("expected upload node guidance, body=%s", recorder.Body.String()) + } +} + +func TestWorkflowUpdateRejectsDuplicateIncomingEdgesPerInputPort(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "更新重复输入端口项目", + Description: "reject duplicate input edges on update", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "合法工作流", + Description: "valid before update", + Type: domain.WorkflowTypeBatch, + Nodes: duplicateInputEdgeWorkflowNodes(), + Edges: []domain.WorkflowEdge{ + {ID: "edge-a", Source: "source-a", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + payload, err := json.Marshal(map[string]any{ + "nodes": duplicateInputEdgeWorkflowNodes(), + "edges": duplicateInputEdgeWorkflowEdges(), + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + recorder := httptest.NewRecorder() + NewHandler(dataStore, nil).ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPut, "/api/workflows/"+workflow.ID, bytes.NewReader(payload)), + ) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected update validation failure, got %d body=%s", recorder.Code, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "每个输入端口只能连接一条上游连线") { + t.Fatalf("expected duplicate input edge validation, body=%s", recorder.Body.String()) + } +} + +func TestWorkflowRunRejectsDuplicateIncomingEdgesPerInputPort(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "运行重复输入端口项目", + Description: "reject duplicate input edges on run", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "非法运行工作流", + Description: "invalid graph", + Type: domain.WorkflowTypeBatch, + Nodes: duplicateInputEdgeWorkflowNodes(), + Edges: duplicateInputEdgeWorkflowEdges(), + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + recorder := httptest.NewRecorder() + NewHandler(dataStore, nil).ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/workflows/"+workflow.ID+"/run", bytes.NewReader([]byte(`{"trigger":"manual"}`))), + ) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected run validation failure, got %d body=%s", recorder.Code, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "每个输入端口只能连接一条上游连线") { + t.Fatalf("expected duplicate input edge validation, body=%s", recorder.Body.String()) + } +} + +func TestTemplateCreateRejectsDuplicateIncomingEdgesPerInputPort(t *testing.T) { + dataStore := store.NewMemoryStore() + + payload, err := json.Marshal(domain.CreateTemplateInput{ + Name: "非法模板", + Description: "duplicate input edges", + Type: domain.WorkflowTypeBatch, + Nodes: duplicateInputEdgeWorkflowNodes(), + Edges: duplicateInputEdgeWorkflowEdges(), + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + recorder := httptest.NewRecorder() + NewHandler(dataStore, nil).ServeHTTP( + recorder, + httptest.NewRequest(http.MethodPost, "/api/templates", bytes.NewReader(payload)), + ) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected template validation failure, got %d body=%s", recorder.Code, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "每个输入端口只能连接一条上游连线") { + t.Fatalf("expected duplicate input edge validation, body=%s", recorder.Body.String()) + } +} + +func TestProjectWorkflowAPICreatesScanDraftAndBlocksRunUntilConfigured(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} { + if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ + Name: name, + Type: "builtin", + Category: "IO", + Description: name, + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}}, + }); err != nil { + t.Fatalf("create operator %s: %v", name, err) + } + } + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "扫描草稿项目", + Description: "scan draft", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + handler := NewHandler(dataStore, nil) + createPayload := []byte(`{ + "name":"扫描草稿", + "description":"先创建后配置", + "type":"batch", + "kind":"scan", + "schedule":{"interval":0}, + "scanConfig":{ + "dataSource":"BOS", + "endpoint":"https://bj.bcebos.com", + "scanMode":"INCREMENTAL", + "enabled":false + }, + "nodes":[], + "edges":[] + }`) + createRecorder := httptest.NewRecorder() + handler.ServeHTTP( + createRecorder, + httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(createPayload)), + ) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create scan draft status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var created domain.WorkflowRecord + if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode scan draft: %v", err) + } + if created.Kind != domain.WorkflowKindScan { + t.Fatalf("expected scan kind, got %s", created.Kind) + } + if created.ScanConfig == nil { + t.Fatalf("expected scan config") + } + if created.ScanConfig.Bucket != "" || created.ScanConfig.Prefix != "" { + t.Fatalf("expected incomplete draft scan target, got %+v", created.ScanConfig) + } + if created.Type != domain.WorkflowTypeBatch { + t.Fatalf("expected draft scan workflow to stay batch, got %s", created.Type) + } + if len(created.Nodes) != 3 { + t.Fatalf("expected scaffolded scan nodes, got %d", len(created.Nodes)) + } + + runRecorder := httptest.NewRecorder() + handler.ServeHTTP( + runRecorder, + httptest.NewRequest(http.MethodPost, "/api/workflows/"+created.ID+"/run", bytes.NewReader([]byte(`{"trigger":"manual"}`))), + ) + if runRecorder.Code != http.StatusBadRequest { + t.Fatalf("expected scan draft run to be rejected, got %d body=%s", runRecorder.Code, runRecorder.Body.String()) + } + if !strings.Contains(runRecorder.Body.String(), "扫描源配置未完成") { + t.Fatalf("expected missing scan config error, got %s", runRecorder.Body.String()) + } +} + func TestSettingsRejectsInvalidTimezone(t *testing.T) { dataStore := store.NewMemoryStore() handler := NewHandler(dataStore, nil) diff --git a/internal/config/config.go b/internal/config/config.go index 65ea2f67..b2e94a08 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "strconv" + "strings" ) type AppConfig struct { @@ -14,11 +15,11 @@ type AppConfig struct { MongoConnectRetries int MongoConnectRetryDelayMS int PollIntervalMS int - K8sEnabled bool K8sNamespace string K8sImagePullSecretName string + WorkspacePVCName string + WorkspaceMountPath string OperatorRunnerImage string - LocalOperatorRunnerCmd string AIAPIBaseURL string AIAPIKey string AIModel string @@ -33,10 +34,6 @@ func parseInt(value string, fallback int) int { return parsed } -func boolValue(value string) bool { - return value == "true" || value == "1" || value == "yes" -} - func Load() AppConfig { workingDirectory, _ := os.Getwd() return AppConfig{ @@ -47,11 +44,11 @@ func Load() AppConfig { MongoConnectRetries: parseInt(os.Getenv("MONGO_CONNECT_RETRIES"), 30), MongoConnectRetryDelayMS: parseInt(os.Getenv("MONGO_CONNECT_RETRY_DELAY_MS"), 2000), PollIntervalMS: parseInt(os.Getenv("WORKER_POLL_INTERVAL_MS"), 2000), - K8sEnabled: boolValue(os.Getenv("K8S_ENABLED")), K8sNamespace: getenv("K8S_NAMESPACE", "droboticflow"), K8sImagePullSecretName: getenv("K8S_IMAGE_PULL_SECRET_NAME", "droboticflow-registry"), + WorkspacePVCName: strings.TrimSpace(os.Getenv("WORKSPACE_PVC_NAME")), + WorkspaceMountPath: getenv("WORKSPACE_MOUNT_PATH", "/workspace"), OperatorRunnerImage: getenv("OPERATOR_RUNNER_IMAGE", "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:latest"), - LocalOperatorRunnerCmd: getenv("LOCAL_OPERATOR_RUNNER_CMD", "python3 backend/operator-runner/main.py"), AIAPIBaseURL: getenv("AI_API_BASE_URL", "https://coding.dashscope.aliyuncs.com/v1"), AIAPIKey: os.Getenv("AI_API_KEY"), AIModel: getenv("AI_MODEL", "glm-5"), diff --git a/internal/domain/data_platform.go b/internal/domain/data_platform.go new file mode 100644 index 00000000..ea4762ec --- /dev/null +++ b/internal/domain/data_platform.go @@ -0,0 +1,191 @@ +package domain + +type ScanNotificationSettings struct { + OnSuccess bool `json:"onSuccess" bson:"onSuccess"` + OnFailure bool `json:"onFailure" bson:"onFailure"` + Channel string `json:"channel,omitempty" bson:"channel,omitempty"` + Target string `json:"target,omitempty" bson:"target,omitempty"` +} + +type ScanConfig struct { + DataSource string `json:"dataSource,omitempty" bson:"dataSource,omitempty"` + SourcePath string `json:"sourcePath,omitempty" bson:"sourcePath,omitempty"` + ConnectionID string `json:"connectionId,omitempty" bson:"connectionId,omitempty"` + Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"` + Endpoint string `json:"endpoint,omitempty" bson:"endpoint,omitempty"` + AccessKeyID string `json:"accessKeyId,omitempty" bson:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty" bson:"secretAccessKey,omitempty"` + AccessKeyPreview string `json:"accessKeyPreview,omitempty" bson:"-"` + SecretAccessKeyPreview string `json:"secretAccessKeyPreview,omitempty" bson:"-"` + HasAccessKeyID bool `json:"hasAccessKeyId,omitempty" bson:"-"` + HasSecretAccessKey bool `json:"hasSecretAccessKey,omitempty" bson:"-"` + ScanMode string `json:"scanMode,omitempty" bson:"scanMode,omitempty"` + IncrementalUnit string `json:"incrementalUnit,omitempty" bson:"incrementalUnit,omitempty"` + DirectoryDepth int `json:"directoryDepth,omitempty" bson:"directoryDepth,omitempty"` + Pattern string `json:"pattern,omitempty" bson:"pattern,omitempty"` + Enabled bool `json:"enabled" bson:"enabled"` + Notifications ScanNotificationSettings `json:"notifications,omitempty" bson:"notifications,omitempty"` +} + +type SourceStateRecord struct { + ID string `json:"id" bson:"id"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + NodeID string `json:"nodeId" bson:"nodeId"` + SourceFingerprint string `json:"sourceFingerprint" bson:"sourceFingerprint"` + DataSource string `json:"dataSource,omitempty" bson:"dataSource,omitempty"` + SourcePath string `json:"sourcePath,omitempty" bson:"sourcePath,omitempty"` + Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"` + Pattern string `json:"pattern,omitempty" bson:"pattern,omitempty"` + ScanMode string `json:"scanMode,omitempty" bson:"scanMode,omitempty"` + IncrementalUnit string `json:"incrementalUnit,omitempty" bson:"incrementalUnit,omitempty"` + DirectoryDepth int `json:"directoryDepth,omitempty" bson:"directoryDepth,omitempty"` + SeenFingerprints []string `json:"seenFingerprints,omitempty" bson:"seenFingerprints,omitempty"` + LastRunID string `json:"lastRunId,omitempty" bson:"lastRunId,omitempty"` + LastListedCount int `json:"lastListedCount" bson:"lastListedCount"` + LastNewCount int `json:"lastNewCount" bson:"lastNewCount"` + LastScanStartedAt string `json:"lastScanStartedAt,omitempty" bson:"lastScanStartedAt,omitempty"` + LastScanCompletedAt string `json:"lastScanCompletedAt,omitempty" bson:"lastScanCompletedAt,omitempty"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type AssetStorageLocation struct { + Provider string `json:"provider,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Bucket string `json:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty"` + Path string `json:"path,omitempty"` +} + +type SourceCollectionRecord struct { + RunID string `json:"runId"` + WorkflowID string `json:"workflowId"` + WorkflowName string `json:"workflowName"` + ProjectID string `json:"projectId"` + ProjectName string `json:"projectName"` + CollectionID string `json:"collectionId"` + CollectionName string `json:"collectionName,omitempty"` + Path string `json:"path"` + RelativePath string `json:"relativePath"` + Meta map[string]any `json:"meta"` + HasIntrinsics bool `json:"hasIntrinsics"` + HasVideoMeta bool `json:"hasVideoMeta"` + RequiredFiles []string `json:"requiredFiles"` + MissingFiles []string `json:"missingRequiredFiles"` + VideoCount int `json:"videoCount"` + FileCount int `json:"fileCount"` + TotalBytes int64 `json:"totalBytes"` + Valid bool `json:"valid"` + CreatedAt string `json:"createdAt"` + Storage AssetStorageLocation `json:"storage"` +} + +type SourceCollectionEntry struct { + Path string `json:"path"` + Name string `json:"name"` + Kind string `json:"kind"` + CollectionName string `json:"collectionName,omitempty"` + SizeBytes int64 `json:"sizeBytes,omitempty"` + Codec string `json:"codec,omitempty"` + Resolution string `json:"resolution,omitempty"` + FrameRate float64 `json:"frameRate,omitempty"` + Duration float64 `json:"duration,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` + Artifact *ExecutionArtifact `json:"artifact,omitempty"` +} + +type SourceCollectionDetail struct { + Collection SourceCollectionRecord `json:"collection"` + Prefix string `json:"prefix,omitempty"` + Entries []SourceCollectionEntry `json:"entries"` +} + +type PackageRecord struct { + RunID string `json:"runId"` + WorkflowID string `json:"workflowId"` + WorkflowName string `json:"workflowName"` + ProjectID string `json:"projectId"` + ProjectName string `json:"projectName"` + PackageName string `json:"packageName"` + ClipID string `json:"clipId"` + SourceVideo string `json:"sourceVideo"` + ClipDuration float64 `json:"clipDuration"` + FileCount int `json:"fileCount"` + PipelineTags []string `json:"pipelineTags"` + StoragePath string `json:"storagePath"` + Quality map[string]any `json:"quality,omitempty"` + Annotation map[string]any `json:"annotation,omitempty"` + CreatedAt string `json:"createdAt"` + PreviewArtifact *ExecutionArtifact `json:"previewArtifact,omitempty"` + Storage AssetStorageLocation `json:"storage"` +} + +type PackageDetail struct { + Package PackageRecord `json:"package"` + Files []ExecutionArtifact `json:"files"` +} + +type SamplingTaskStatus string + +const ( + SamplingTaskStatusPending SamplingTaskStatus = "pending" + SamplingTaskStatusCompleted SamplingTaskStatus = "completed" +) + +type SamplingItemStatus string + +const ( + SamplingItemStatusPending SamplingItemStatus = "pending" + SamplingItemStatusCompleted SamplingItemStatus = "completed" +) + +type SamplingRuleSummary struct { + ID string `json:"id" bson:"id"` + Field string `json:"field" bson:"field"` + Operator string `json:"operator" bson:"operator"` + Value any `json:"value,omitempty" bson:"value,omitempty"` + SamplePercent float64 `json:"samplePercent" bson:"samplePercent"` + MatchedCount int `json:"matchedCount" bson:"matchedCount"` + SampledCount int `json:"sampledCount" bson:"sampledCount"` +} + +type SamplingReview struct { + VideoQuality string `json:"videoQuality" bson:"videoQuality"` + AnnotationAccuracy string `json:"annotationAccuracy" bson:"annotationAccuracy"` + AnnotationCompleteness string `json:"annotationCompleteness" bson:"annotationCompleteness"` + OverallGrade string `json:"overallGrade" bson:"overallGrade"` + Notes string `json:"notes,omitempty" bson:"notes,omitempty"` + ReviewedAt string `json:"reviewedAt" bson:"reviewedAt"` +} + +type SamplingItemRecord struct { + ID string `json:"id" bson:"id"` + TaskID string `json:"taskId" bson:"taskId"` + PackageName string `json:"packageName" bson:"packageName"` + ClipID string `json:"clipId" bson:"clipId"` + SourceVideo string `json:"sourceVideo" bson:"sourceVideo"` + SampleRuleID string `json:"sampleRuleId" bson:"sampleRuleId"` + StoragePath string `json:"storagePath,omitempty" bson:"storagePath,omitempty"` + Quality map[string]any `json:"quality,omitempty" bson:"quality,omitempty"` + Annotation map[string]any `json:"annotation,omitempty" bson:"annotation,omitempty"` + PreviewArtifact *ExecutionArtifact `json:"previewArtifact,omitempty" bson:"previewArtifact,omitempty"` + Status SamplingItemStatus `json:"status" bson:"status"` + Review *SamplingReview `json:"review,omitempty" bson:"review,omitempty"` +} + +type SamplingTaskRecord struct { + ID string `json:"id" bson:"id"` + RunID string `json:"runId" bson:"runId"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + WorkflowName string `json:"workflowName" bson:"workflowName"` + ProjectID string `json:"projectId" bson:"projectId"` + ProjectName string `json:"projectName" bson:"projectName"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Status SamplingTaskStatus `json:"status" bson:"status"` + Rules []SamplingRuleSummary `json:"rules" bson:"rules"` + Items []SamplingItemRecord `json:"items" bson:"items"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} diff --git a/internal/domain/database_browser.go b/internal/domain/database_browser.go new file mode 100644 index 00000000..6bc59c93 --- /dev/null +++ b/internal/domain/database_browser.go @@ -0,0 +1,108 @@ +package domain + +type DatabaseCollectionSummary struct { + Name string `json:"name"` + DocumentCount int64 `json:"documentCount"` + SampleFields []string `json:"sampleFields,omitempty"` +} + +type DatabaseIndexSummary struct { + Name string `json:"name"` + Keys map[string]any `json:"keys"` + Unique bool `json:"unique,omitempty"` +} + +type DatabaseOverview struct { + Target string `json:"target"` + ConnectionID string `json:"connectionId,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + Provider string `json:"provider"` + Database string `json:"database"` + ReadOnly bool `json:"readOnly"` + Configured bool `json:"configured"` + Collections []DatabaseCollectionSummary `json:"collections"` +} + +type DatabaseQueryInput struct { + Collection string `json:"collection"` + Filter map[string]any `json:"filter,omitempty"` + Limit int `json:"limit,omitempty"` + Skip int `json:"skip,omitempty"` +} + +type DatabaseQueryResult struct { + Target string `json:"target"` + ConnectionID string `json:"connectionId,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + Provider string `json:"provider"` + Database string `json:"database"` + Collection string `json:"collection"` + Configured bool `json:"configured"` + Filter map[string]any `json:"filter,omitempty"` + Limit int `json:"limit"` + Skip int `json:"skip"` + Total int64 `json:"total"` + SampleFields []string `json:"sampleFields,omitempty"` + Documents []map[string]any `json:"documents"` +} + +type DatabaseCollectionCreateInput struct { + Name string `json:"name"` +} + +type DatabaseCollectionRenameInput struct { + NewName string `json:"newName"` +} + +type DatabaseCollectionMutationResult struct { + Collection string `json:"collection"` + Summary DatabaseCollectionSummary `json:"summary"` +} + +type DatabaseCollectionDeleteResult struct { + Collection string `json:"collection"` + Deleted bool `json:"deleted"` +} + +type DatabaseIndexesResult struct { + Collection string `json:"collection"` + Indexes []DatabaseIndexSummary `json:"indexes"` +} + +type DatabaseIndexCreateInput struct { + Name string `json:"name,omitempty"` + Keys map[string]any `json:"keys"` + Unique bool `json:"unique,omitempty"` +} + +type DatabaseIndexMutationResult struct { + Collection string `json:"collection"` + Index DatabaseIndexSummary `json:"index"` +} + +type DatabaseIndexDeleteResult struct { + Collection string `json:"collection"` + IndexName string `json:"indexName"` + Deleted bool `json:"deleted"` +} + +type DatabaseDocumentMutationInput struct { + Document map[string]any `json:"document"` +} + +type DatabaseDocumentMutationResult struct { + Collection string `json:"collection"` + Document map[string]any `json:"document"` +} + +type DatabaseDocumentDeleteResult struct { + Collection string `json:"collection"` + DocumentID string `json:"documentId"` + Deleted bool `json:"deleted"` +} + +type MongoConnectionTestInput struct { + ID string `json:"id,omitempty"` + URI string `json:"uri,omitempty"` + Database string `json:"database,omitempty"` +} diff --git a/internal/domain/object_storage_browser.go b/internal/domain/object_storage_browser.go new file mode 100644 index 00000000..e095865b --- /dev/null +++ b/internal/domain/object_storage_browser.go @@ -0,0 +1,79 @@ +package domain + +type ObjectStorageBucketSummary struct { + Name string `json:"name"` + ObjectCount int64 `json:"objectCount,omitempty"` + TotalBytes int64 `json:"totalBytes,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + LastModified string `json:"lastModified,omitempty"` +} + +type ObjectStorageObjectSummary struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Name string `json:"name"` + Kind string `json:"kind"` + SizeBytes int64 `json:"sizeBytes,omitempty"` + ContentType string `json:"contentType,omitempty"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"lastModified,omitempty"` + StorageClass string `json:"storageClass,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +type ObjectStorageOverview struct { + ConnectionID string `json:"connectionId,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + Provider string `json:"provider"` + Endpoint string `json:"endpoint,omitempty"` + Bucket string `json:"bucket,omitempty"` + Configured bool `json:"configured"` + Buckets []ObjectStorageBucketSummary `json:"buckets"` +} + +type ObjectStorageBrowseResult struct { + ConnectionID string `json:"connectionId,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + Provider string `json:"provider"` + Endpoint string `json:"endpoint,omitempty"` + Configured bool `json:"configured"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix,omitempty"` + Items []ObjectStorageObjectSummary `json:"items"` +} + +type ObjectStorageBucketCreateInput struct { + Name string `json:"name"` +} + +type ObjectStorageBucketMutationResult struct { + Bucket ObjectStorageBucketSummary `json:"bucket"` +} + +type ObjectStorageBucketDeleteResult struct { + Bucket string `json:"bucket"` + Deleted bool `json:"deleted"` +} + +type ObjectStorageObjectPutInput struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Content string `json:"content"` + ContentType string `json:"contentType,omitempty"` +} + +type ObjectStorageObjectRenameInput struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + NewKey string `json:"newKey"` +} + +type ObjectStorageObjectMutationResult struct { + Object ObjectStorageObjectSummary `json:"object"` +} + +type ObjectStorageObjectDeleteResult struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Deleted bool `json:"deleted"` +} diff --git a/internal/domain/schedule.go b/internal/domain/schedule.go new file mode 100644 index 00000000..2eca4638 --- /dev/null +++ b/internal/domain/schedule.go @@ -0,0 +1,68 @@ +package domain + +import "strings" + +type WorkflowScheduleMode string + +const ( + WorkflowScheduleModeIntervalSeconds WorkflowScheduleMode = "interval_seconds" + WorkflowScheduleModeEveryHours WorkflowScheduleMode = "every_hours" + WorkflowScheduleModeEveryDaysAtTime WorkflowScheduleMode = "every_days_at_time" +) + +const DefaultWorkflowScheduleTimezone = "Asia/Shanghai" + +func NormalizeWorkflowScheduleMode(value WorkflowScheduleMode) WorkflowScheduleMode { + switch WorkflowScheduleMode(strings.TrimSpace(string(value))) { + case WorkflowScheduleModeEveryHours: + return WorkflowScheduleModeEveryHours + case WorkflowScheduleModeEveryDaysAtTime: + return WorkflowScheduleModeEveryDaysAtTime + case WorkflowScheduleModeIntervalSeconds: + return WorkflowScheduleModeIntervalSeconds + default: + return "" + } +} + +func WorkflowScheduleTimezone(value string) string { + if trimmed := strings.TrimSpace(value); trimmed != "" { + return trimmed + } + return DefaultWorkflowScheduleTimezone +} + +func EffectiveWorkflowScheduleMode(schedule *WorkflowSchedule) WorkflowScheduleMode { + if schedule == nil { + return "" + } + if mode := NormalizeWorkflowScheduleMode(schedule.Mode); mode != "" { + return mode + } + if schedule.Interval > 0 { + return WorkflowScheduleModeIntervalSeconds + } + if schedule.IntervalHours > 0 { + return WorkflowScheduleModeEveryHours + } + if schedule.DayInterval > 0 || strings.TrimSpace(schedule.TimeOfDay) != "" { + return WorkflowScheduleModeEveryDaysAtTime + } + return "" +} + +func WorkflowScheduleEnabled(schedule *WorkflowSchedule) bool { + if schedule == nil { + return false + } + switch EffectiveWorkflowScheduleMode(schedule) { + case WorkflowScheduleModeEveryHours: + return schedule.IntervalHours > 0 + case WorkflowScheduleModeEveryDaysAtTime: + return schedule.DayInterval > 0 && strings.TrimSpace(schedule.TimeOfDay) != "" + case WorkflowScheduleModeIntervalSeconds: + return schedule.Interval > 0 + default: + return false + } +} diff --git a/internal/domain/source_state.go b/internal/domain/source_state.go new file mode 100644 index 00000000..89cfdbb7 --- /dev/null +++ b/internal/domain/source_state.go @@ -0,0 +1,36 @@ +package domain + +import ( + "crypto/sha256" + "encoding/hex" + "strconv" + "strings" +) + +type SourceFingerprintInput struct { + ScanMode string + IncrementalUnit string + DirectoryDepth int + DataSource string + SourcePath string + Bucket string + Prefix string + Pattern string + Endpoint string +} + +func ComputeSourceFingerprint(input SourceFingerprintInput) string { + parts := []string{ + "scanMode=" + strings.TrimSpace(input.ScanMode), + "incrementalUnit=" + strings.TrimSpace(input.IncrementalUnit), + "directoryDepth=" + strconv.Itoa(input.DirectoryDepth), + "dataSource=" + strings.TrimSpace(input.DataSource), + "sourcePath=" + strings.TrimSpace(input.SourcePath), + "bucket=" + strings.TrimSpace(input.Bucket), + "prefix=" + strings.TrimSpace(input.Prefix), + "pattern=" + strings.TrimSpace(input.Pattern), + "endpoint=" + strings.TrimSpace(input.Endpoint), + } + sum := sha256.Sum256([]byte(strings.Join(parts, "\n"))) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/domain/types.go b/internal/domain/types.go index 4d7a9c76..15d29462 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -35,12 +35,26 @@ const ( WorkflowTypeStreaming WorkflowType = "streaming" ) +type WorkflowKind string + +const ( + WorkflowKindStandard WorkflowKind = "standard" + WorkflowKindScan WorkflowKind = "scan" +) + +type StorageProvider string + +const ( + StorageProviderBOS StorageProvider = "BOS" +) + type RunTrigger string const ( - RunTriggerManual RunTrigger = "manual" - RunTriggerScheduled RunTrigger = "scheduled" - RunTriggerRetry RunTrigger = "retry" + RunTriggerManual RunTrigger = "manual" + RunTriggerScheduled RunTrigger = "scheduled" + RunTriggerRetry RunTrigger = "retry" + RunTriggerRerunSubtree RunTrigger = "rerun_subtree" ) type RunStatus string @@ -48,12 +62,20 @@ type RunStatus string const ( RunStatusQueued RunStatus = "queued" RunStatusRunning RunStatus = "running" + RunStatusStopping RunStatus = "stopping" RunStatusWaitingForAudit RunStatus = "waiting_for_audit" RunStatusCompleted RunStatus = "completed" RunStatusFailed RunStatus = "failed" RunStatusCancelled RunStatus = "cancelled" ) +type RunStopMode string + +const ( + RunStopModeGraceful RunStopMode = "graceful" + RunStopModeForce RunStopMode = "force" +) + type AuditStatus string const ( @@ -70,6 +92,7 @@ const ( NodeExecutionStatusWaitingForAudit NodeExecutionStatus = "waiting_for_audit" NodeExecutionStatusCompleted NodeExecutionStatus = "completed" NodeExecutionStatusFailed NodeExecutionStatus = "failed" + NodeExecutionStatusCancelled NodeExecutionStatus = "cancelled" ) type Position struct { @@ -101,6 +124,7 @@ type WorkflowNodeData struct { Image string `json:"image,omitempty" bson:"image,omitempty"` SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"` Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` + ConnectionID string `json:"connectionId,omitempty" bson:"connectionId,omitempty"` Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"` Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"` Pattern string `json:"pattern,omitempty" bson:"pattern,omitempty"` @@ -125,13 +149,14 @@ type WorkflowEdge struct { } type ProjectRecord struct { - ID string `json:"id" bson:"id"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Status string `json:"status" bson:"status"` - WorkflowCount int `json:"workflowCount" bson:"-"` - CreatedAt string `json:"createdAt" bson:"createdAt"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Status string `json:"status" bson:"status"` + StoragePathFavorites []StoragePathFavoriteRecord `json:"storagePathFavorites,omitempty" bson:"storagePathFavorites,omitempty"` + WorkflowCount int `json:"workflowCount" bson:"-"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } type OperatorResources struct { @@ -158,10 +183,23 @@ type OperatorRecord struct { UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } +type BuiltinOperatorDeletionRecord struct { + Key string `json:"key" bson:"key"` + OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"` + OperatorName string `json:"operatorName,omitempty" bson:"operatorName,omitempty"` + DeletedAt string `json:"deletedAt" bson:"deletedAt"` +} + type WorkflowSchedule struct { - Interval int `json:"interval" bson:"interval"` - Cron string `json:"cron,omitempty" bson:"cron,omitempty"` - SourcePath string `json:"sourcePath,omitempty" bson:"sourcePath,omitempty"` + Mode WorkflowScheduleMode `json:"mode,omitempty" bson:"mode,omitempty"` + Interval int `json:"interval" bson:"interval"` + IntervalHours int `json:"intervalHours,omitempty" bson:"intervalHours,omitempty"` + DayInterval int `json:"dayInterval,omitempty" bson:"dayInterval,omitempty"` + TimeOfDay string `json:"timeOfDay,omitempty" bson:"timeOfDay,omitempty"` + AnchorAt string `json:"anchorAt,omitempty" bson:"anchorAt,omitempty"` + Timezone string `json:"timezone,omitempty" bson:"timezone,omitempty"` + Cron string `json:"cron,omitempty" bson:"cron,omitempty"` + SourcePath string `json:"sourcePath,omitempty" bson:"sourcePath,omitempty"` } type WorkflowRecord struct { @@ -170,6 +208,9 @@ type WorkflowRecord struct { Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` Type WorkflowType `json:"type" bson:"type"` + Kind WorkflowKind `json:"kind,omitempty" bson:"kind,omitempty"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty" bson:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` @@ -190,6 +231,9 @@ type TemplateRecord struct { Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` Type WorkflowType `json:"type" bson:"type"` + Kind WorkflowKind `json:"kind,omitempty" bson:"kind,omitempty"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty" bson:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` @@ -239,16 +283,31 @@ type ExecutionArtifact struct { Data map[string]any `json:"data,omitempty" bson:"data,omitempty"` } +type DataRefRecord struct { + ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"` + ArtifactName string `json:"artifactName,omitempty" bson:"artifactName,omitempty"` + PortID string `json:"portId,omitempty" bson:"portId,omitempty"` + Kind string `json:"kind,omitempty" bson:"kind,omitempty"` + DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"` + ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"` + PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"` + PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"` + PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"` + ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"` +} + type NodeExecutionRecord struct { - NodeID string `json:"nodeId" bson:"nodeId"` - NodeLabel string `json:"nodeLabel" bson:"nodeLabel"` - Status NodeExecutionStatus `json:"status" bson:"status"` - StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` - EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` - Logs []LogEntryRecord `json:"logs" bson:"logs"` - Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` - ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` - Error string `json:"error,omitempty" bson:"error,omitempty"` + NodeID string `json:"nodeId" bson:"nodeId"` + NodeLabel string `json:"nodeLabel" bson:"nodeLabel"` + Status NodeExecutionStatus `json:"status" bson:"status"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` + StageManifestPath string `json:"stageManifestPath,omitempty" bson:"stageManifestPath,omitempty"` + Logs []LogEntryRecord `json:"logs" bson:"logs"` + Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` + DataRefs []DataRefRecord `json:"dataRefs,omitempty" bson:"dataRefs,omitempty"` + ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` + Error string `json:"error,omitempty" bson:"error,omitempty"` } type K8sRef struct { @@ -258,28 +317,56 @@ type K8sRef struct { } type RunRecord struct { - ID string `json:"id" bson:"id"` - WorkflowID string `json:"workflowId" bson:"workflowId"` - WorkflowName string `json:"workflowName" bson:"workflowName"` - ProjectID string `json:"projectId" bson:"projectId"` - ProjectName string `json:"projectName" bson:"projectName"` - Trigger RunTrigger `json:"trigger" bson:"trigger"` - Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` - ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` - AvailableAt string `json:"availableAt,omitempty" bson:"availableAt,omitempty"` - Status RunStatus `json:"status" bson:"status"` - StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` - EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` - DurationMS int64 `json:"durationMs,omitempty" bson:"durationMs,omitempty"` - CurrentNodeID string `json:"currentNodeId,omitempty" bson:"currentNodeId,omitempty"` - NodeExecutions []NodeExecutionRecord `json:"nodeExecutions" bson:"nodeExecutions"` - Logs []LogEntryRecord `json:"logs" bson:"logs"` - ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` - Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` - K8sRefs []K8sRef `json:"k8sRefs" bson:"k8sRefs"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - CreatedAt string `json:"createdAt" bson:"createdAt"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + ID string `json:"id" bson:"id"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + WorkflowName string `json:"workflowName" bson:"workflowName"` + ProjectID string `json:"projectId" bson:"projectId"` + ProjectName string `json:"projectName" bson:"projectName"` + Trigger RunTrigger `json:"trigger" bson:"trigger"` + Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` + ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` + RestartFromNodeID string `json:"restartFromNodeId,omitempty" bson:"restartFromNodeId,omitempty"` + AvailableAt string `json:"availableAt,omitempty" bson:"availableAt,omitempty"` + Status RunStatus `json:"status" bson:"status"` + StopMode RunStopMode `json:"stopMode,omitempty" bson:"stopMode,omitempty"` + StopRequestedAt string `json:"stopRequestedAt,omitempty" bson:"stopRequestedAt,omitempty"` + PreviousRunAt string `json:"previousRunAt,omitempty" bson:"previousRunAt,omitempty"` + CurrentRunAt string `json:"currentRunAt,omitempty" bson:"currentRunAt,omitempty"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` + DurationMS int64 `json:"durationMs,omitempty" bson:"durationMs,omitempty"` + CurrentNodeID string `json:"currentNodeId,omitempty" bson:"currentNodeId,omitempty"` + NodeExecutions []NodeExecutionRecord `json:"nodeExecutions" bson:"nodeExecutions"` + Logs []LogEntryRecord `json:"logs" bson:"logs"` + ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` + Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` + K8sRefs []K8sRef `json:"k8sRefs" bson:"k8sRefs"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type RunSummary struct { + ID string `json:"id" bson:"id"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + WorkflowName string `json:"workflowName" bson:"workflowName"` + ProjectID string `json:"projectId" bson:"projectId"` + ProjectName string `json:"projectName" bson:"projectName"` + Trigger RunTrigger `json:"trigger" bson:"trigger"` + Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` + ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` + RestartFromNodeID string `json:"restartFromNodeId,omitempty" bson:"restartFromNodeId,omitempty"` + AvailableAt string `json:"availableAt,omitempty" bson:"availableAt,omitempty"` + Status RunStatus `json:"status" bson:"status"` + StopMode RunStopMode `json:"stopMode,omitempty" bson:"stopMode,omitempty"` + StopRequestedAt string `json:"stopRequestedAt,omitempty" bson:"stopRequestedAt,omitempty"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` + DurationMS int64 `json:"durationMs,omitempty" bson:"durationMs,omitempty"` + CurrentNodeID string `json:"currentNodeId,omitempty" bson:"currentNodeId,omitempty"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } type AuditTaskPayload struct { @@ -305,21 +392,30 @@ type AuditTaskRecord struct { } type UserRecord struct { - ID string `json:"id" bson:"id"` - Name string `json:"name" bson:"name"` - Email string `json:"email" bson:"email"` - Role UserRole `json:"role" bson:"role"` - CreatedAt string `json:"createdAt" bson:"createdAt"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Email string `json:"email" bson:"email"` + Role UserRole `json:"role" bson:"role"` + StoragePathFavorites []StoragePathFavoriteRecord `json:"storagePathFavorites,omitempty" bson:"storagePathFavorites,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type StoragePathFavoriteRecord struct { + ScopeKey string `json:"scopeKey" bson:"scopeKey"` + Bucket string `json:"bucket" bson:"bucket"` + Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"` + Label string `json:"label,omitempty" bson:"label,omitempty"` } type K8sSettings struct { - Enabled bool `json:"enabled" bson:"enabled"` - Endpoint string `json:"endpoint,omitempty" bson:"endpoint,omitempty"` - Namespace string `json:"namespace" bson:"namespace"` - Kubeconfig string `json:"kubeconfig,omitempty" bson:"kubeconfig,omitempty"` - KubeconfigPreview string `json:"kubeconfigPreview,omitempty" bson:"-"` - HasKubeconfig bool `json:"hasKubeconfig,omitempty" bson:"-"` + Endpoint string `json:"endpoint,omitempty" bson:"endpoint,omitempty"` + Namespace string `json:"namespace" bson:"namespace"` + WorkspacePVCName string `json:"workspacePVCName,omitempty" bson:"workspacePVCName,omitempty"` + WorkspaceMountPath string `json:"workspaceMountPath,omitempty" bson:"workspaceMountPath,omitempty"` + Kubeconfig string `json:"kubeconfig,omitempty" bson:"kubeconfig,omitempty"` + KubeconfigPreview string `json:"kubeconfigPreview,omitempty" bson:"-"` + HasKubeconfig bool `json:"hasKubeconfig,omitempty" bson:"-"` } type WorkerSettings struct { @@ -341,6 +437,13 @@ type AISettings struct { HasAPIKey bool `json:"hasAPIKey,omitempty" bson:"-"` } +type BusinessMongoSettings struct { + URI string `json:"uri,omitempty" bson:"uri,omitempty"` + Database string `json:"database,omitempty" bson:"database,omitempty"` + URIPreview string `json:"uriPreview,omitempty" bson:"-"` + HasURI bool `json:"hasUri,omitempty" bson:"-"` +} + type DefaultResources struct { CPURequest string `json:"cpuRequest" bson:"cpuRequest"` CPULimit string `json:"cpuLimit" bson:"cpuLimit"` @@ -348,25 +451,51 @@ type DefaultResources struct { MemoryLimit string `json:"memoryLimit" bson:"memoryLimit"` } +type ExternalBusinessWriteScope struct { + ConnectionID string `json:"connectionId" bson:"connectionId"` + Collections []string `json:"collections" bson:"collections"` +} + +type ExternalBusinessWriteTokenRecord struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + Enabled bool `json:"enabled" bson:"enabled"` + TokenValue string `json:"tokenValue,omitempty" bson:"tokenValue,omitempty"` + TokenHash string `json:"tokenHash,omitempty" bson:"tokenHash,omitempty"` + TokenPreview string `json:"tokenPreview,omitempty" bson:"tokenPreview,omitempty"` + Scopes []ExternalBusinessWriteScope `json:"scopes,omitempty" bson:"scopes,omitempty"` + LastUsedAt string `json:"lastUsedAt,omitempty" bson:"lastUsedAt,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type ExternalBusinessWriteSettings struct { + Tokens []ExternalBusinessWriteTokenRecord `json:"tokens,omitempty" bson:"tokens,omitempty"` +} + type SettingsRecord struct { - ID string `json:"id" bson:"id"` - K8s K8sSettings `json:"k8s" bson:"k8s"` - Worker WorkerSettings `json:"worker" bson:"worker"` - Storage StorageSettings `json:"storage" bson:"storage"` - AI AISettings `json:"ai" bson:"ai"` - Defaults DefaultResources `json:"defaults" bson:"defaults"` - Timezone string `json:"timezone" bson:"timezone"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + ID string `json:"id" bson:"id"` + K8s K8sSettings `json:"k8s" bson:"k8s"` + Worker WorkerSettings `json:"worker" bson:"worker"` + Storage StorageSettings `json:"storage" bson:"storage"` + AI AISettings `json:"ai" bson:"ai"` + BusinessMongo BusinessMongoSettings `json:"businessMongo" bson:"businessMongo"` + ExternalBusinessWrite ExternalBusinessWriteSettings `json:"externalBusinessWrite,omitempty" bson:"externalBusinessWrite,omitempty"` + Defaults DefaultResources `json:"defaults" bson:"defaults"` + Timezone string `json:"timezone" bson:"timezone"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } type SettingsView struct { - ID string `json:"id"` - K8s K8sSettings `json:"k8s"` - Worker WorkerSettings `json:"worker"` - AI AISettings `json:"ai"` - Defaults DefaultResources `json:"defaults"` - Timezone string `json:"timezone"` - UpdatedAt string `json:"updatedAt"` + ID string `json:"id"` + K8s K8sSettings `json:"k8s"` + Worker WorkerSettings `json:"worker"` + AI AISettings `json:"ai"` + BusinessMongo BusinessMongoSettings `json:"businessMongo"` + Defaults DefaultResources `json:"defaults"` + Timezone string `json:"timezone"` + UpdatedAt string `json:"updatedAt"` } type DashboardSummary struct { @@ -390,16 +519,65 @@ type DashboardSummary struct { } `json:"audits"` } +type StorageConnectionRecord struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + Provider StorageProvider `json:"provider" bson:"provider"` + Endpoint string `json:"endpoint,omitempty" bson:"endpoint,omitempty"` + Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"` + AccessKeyID string `json:"accessKeyId,omitempty" bson:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty" bson:"secretAccessKey,omitempty"` + AccessKeyPreview string `json:"accessKeyPreview,omitempty" bson:"-"` + SecretAccessKeyPreview string `json:"secretAccessKeyPreview,omitempty" bson:"-"` + HasAccessKeyID bool `json:"hasAccessKeyId,omitempty" bson:"-"` + HasSecretAccessKey bool `json:"hasSecretAccessKey,omitempty" bson:"-"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type MongoConnectionRecord struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + URI string `json:"uri,omitempty" bson:"uri,omitempty"` + Database string `json:"database,omitempty" bson:"database,omitempty"` + URIPreview string `json:"uriPreview,omitempty" bson:"-"` + HasURI bool `json:"hasUri,omitempty" bson:"-"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + type CreateProjectInput struct { Name string `json:"name"` Description string `json:"description"` } +type CreateStorageConnectionInput struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Provider StorageProvider `json:"provider"` + Endpoint string `json:"endpoint,omitempty"` + Bucket string `json:"bucket,omitempty"` + AccessKeyID string `json:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty"` +} + +type CreateMongoConnectionInput struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + URI string `json:"uri,omitempty"` + Database string `json:"database,omitempty"` +} + type CreateWorkflowInput struct { ProjectID string `json:"projectId"` Name string `json:"name"` Description string `json:"description"` Type WorkflowType `json:"type"` + Kind WorkflowKind `json:"kind,omitempty"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty"` MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty"` NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty"` RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty"` @@ -414,6 +592,9 @@ type CreateTemplateInput struct { Name string `json:"name"` Description string `json:"description"` Type WorkflowType `json:"type"` + Kind WorkflowKind `json:"kind,omitempty"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty"` MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty"` NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty"` RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty"` diff --git a/internal/domain/workflow.go b/internal/domain/workflow.go index b7cac647..b1ff16df 100644 --- a/internal/domain/workflow.go +++ b/internal/domain/workflow.go @@ -1,6 +1,23 @@ package domain -import "strings" +import ( + "sort" + "strings" +) + +func IsScanSourceNode(node WorkflowNode) bool { + label := strings.TrimSpace(node.Data.Label) + return node.ID == "scan-source" || label == "BOS 增量扫描" +} + +func FindScanSourceNode(nodes []WorkflowNode) *WorkflowNode { + for index := range nodes { + if IsScanSourceNode(nodes[index]) { + return &nodes[index] + } + } + return nil +} func GetNodeByID(nodes []WorkflowNode, nodeID string) *WorkflowNode { for index := range nodes { @@ -48,6 +65,98 @@ func GetStartNodeIDs(nodes []WorkflowNode, edges []WorkflowEdge) []string { return items } +type InputPortConflict struct { + TargetNodeID string `json:"targetNodeId"` + TargetNodeLabel string `json:"targetNodeLabel,omitempty"` + TargetHandle string `json:"targetHandle,omitempty"` + TargetPortName string `json:"targetPortName,omitempty"` + EdgeIDs []string `json:"edgeIds"` +} + +func normalizeWorkflowHandle(value string) string { + return strings.TrimSpace(value) +} + +func findInputPortName(node *WorkflowNode, handle string) string { + if node == nil { + if handle == "" { + return "默认输入" + } + return handle + } + normalizedHandle := normalizeWorkflowHandle(handle) + if normalizedHandle == "" { + return "默认输入" + } + for _, port := range node.Data.Inputs { + if normalizeWorkflowHandle(port.ID) == normalizedHandle { + if name := strings.TrimSpace(port.Name); name != "" { + return name + } + break + } + } + return normalizedHandle +} + +func FindInputPortConflicts(nodes []WorkflowNode, edges []WorkflowEdge) []InputPortConflict { + type key struct { + target string + handle string + } + + nodeByID := make(map[string]*WorkflowNode, len(nodes)) + for index := range nodes { + nodeByID[nodes[index].ID] = &nodes[index] + } + + grouped := make(map[key]*InputPortConflict) + for _, edge := range edges { + if strings.TrimSpace(edge.Target) == "" { + continue + } + conflictKey := key{ + target: strings.TrimSpace(edge.Target), + handle: normalizeWorkflowHandle(edge.TargetHandle), + } + conflict := grouped[conflictKey] + if conflict == nil { + targetNode := nodeByID[conflictKey.target] + conflict = &InputPortConflict{ + TargetNodeID: conflictKey.target, + TargetHandle: conflictKey.handle, + TargetPortName: findInputPortName(targetNode, conflictKey.handle), + } + if targetNode != nil { + conflict.TargetNodeLabel = strings.TrimSpace(targetNode.Data.Label) + } + grouped[conflictKey] = conflict + } + edgeID := strings.TrimSpace(edge.ID) + if edgeID == "" { + edgeID = edge.Source + "->" + edge.Target + } + conflict.EdgeIDs = append(conflict.EdgeIDs, edgeID) + } + + conflicts := make([]InputPortConflict, 0) + for _, conflict := range grouped { + if len(conflict.EdgeIDs) <= 1 { + continue + } + sort.Strings(conflict.EdgeIDs) + conflicts = append(conflicts, *conflict) + } + + sort.Slice(conflicts, func(i, j int) bool { + if conflicts[i].TargetNodeID != conflicts[j].TargetNodeID { + return conflicts[i].TargetNodeID < conflicts[j].TargetNodeID + } + return conflicts[i].TargetHandle < conflicts[j].TargetHandle + }) + return conflicts +} + func auditDecisionMatchesHandle(handle string, decision AuditStatus) bool { if handle == "" { return decision == AuditStatusApproved diff --git a/internal/k8s/launcher.go b/internal/k8s/launcher.go index 1a95089f..514fea4a 100644 --- a/internal/k8s/launcher.go +++ b/internal/k8s/launcher.go @@ -2,11 +2,13 @@ package k8s import ( "context" + "errors" "fmt" "io" "os" "path/filepath" "regexp" + "sort" "strings" "time" @@ -18,6 +20,8 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +const defaultJobWaitTimeout = 2 * time.Hour + type LaunchInput struct { Image string RunID string @@ -38,10 +42,27 @@ type LaunchResult struct { Output string } +type JobState string + +const ( + JobStateMissing JobState = "missing" + JobStatePending JobState = "pending" + JobStateActive JobState = "active" + JobStateSucceeded JobState = "succeeded" + JobStateFailed JobState = "failed" +) + +type JobInspection struct { + Reference *JobReference + State JobState + Output string +} + type Launcher struct { - enabled bool namespace string imagePullSecretName string + workspacePVCName string + workspaceMountPath string clientset kubernetes.Interface } @@ -58,7 +79,7 @@ func sanitizeName(value string) string { return normalized } -func CreateJobManifest(namespace string, imagePullSecretName string, input LaunchInput) *batchv1.Job { +func CreateJobManifest(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, input LaunchInput) *batchv1.Job { jobPrefix := sanitizeName(input.RunID + "-" + input.NodeID) if len(jobPrefix) > 52 { jobPrefix = jobPrefix[:52] @@ -121,6 +142,20 @@ func CreateJobManifest(namespace string, imagePullSecretName string, input Launc if imagePullSecretName != "" { job.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: imagePullSecretName}} } + if strings.TrimSpace(workspacePVCName) != "" { + job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "workspace", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: workspacePVCName, + }, + }, + }) + job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "workspace", + MountPath: workspaceMountPath, + }) + } return job } @@ -139,14 +174,23 @@ func loadRestConfig() (*rest.Config, error) { return clientcmd.BuildConfigFromFlags("", kubeconfig) } -func NewLauncher(enabled bool, namespace string, imagePullSecretName string) (*Launcher, error) { +func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string) (*Launcher, error) { + trimmedPVCName := strings.TrimSpace(workspacePVCName) + if trimmedPVCName == "" { + return nil, fmt.Errorf("workspace pvc name is required") + } + trimmedMountPath := strings.TrimSpace(workspaceMountPath) + if trimmedMountPath == "" { + trimmedMountPath = "/workspace" + } + if !strings.HasPrefix(trimmedMountPath, "/") { + return nil, fmt.Errorf("workspace mount path must be absolute") + } launcher := &Launcher{ - enabled: enabled, namespace: namespace, imagePullSecretName: imagePullSecretName, - } - if !enabled { - return launcher, nil + workspacePVCName: trimmedPVCName, + workspaceMountPath: trimmedMountPath, } config, err := loadRestConfig() if err != nil { @@ -160,8 +204,14 @@ func NewLauncher(enabled bool, namespace string, imagePullSecretName string) (*L return launcher, nil } -func (launcher *Launcher) Enabled() bool { - return launcher.enabled +func waitDeadlineForLaunch(ctx context.Context, input LaunchInput, now time.Time) time.Time { + if deadline, ok := ctx.Deadline(); ok { + return deadline + } + if input.TimeoutSeconds > 0 { + return now.Add(time.Duration(input.TimeoutSeconds)*time.Second + 30*time.Second) + } + return now.Add(defaultJobWaitTimeout) } func (launcher *Launcher) readJobOutput(ctx context.Context, jobName string) (string, error) { @@ -194,18 +244,59 @@ func (launcher *Launcher) readJobOutput(ctx context.Context, jobName string) (st return string(output), nil } -func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*LaunchResult, error) { - if !launcher.enabled { - return nil, nil +func (launcher *Launcher) TerminateJob(ctx context.Context, namespace string, jobName string) error { + targetNamespace := strings.TrimSpace(namespace) + if targetNamespace == "" { + targetNamespace = launcher.namespace } - job := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, input) + propagation := metav1.DeletePropagationBackground + err := launcher.clientset.BatchV1().Jobs(targetNamespace).Delete(ctx, jobName, metav1.DeleteOptions{ + PropagationPolicy: &propagation, + }) + if err != nil && strings.Contains(strings.ToLower(err.Error()), "not found") { + return nil + } + return err +} + +func (launcher *Launcher) TerminateRun(ctx context.Context, namespace string, runID string) error { + if launcher.clientset == nil || strings.TrimSpace(runID) == "" { + return nil + } + targetNamespace := strings.TrimSpace(namespace) + if targetNamespace == "" { + targetNamespace = launcher.namespace + } + jobs, err := launcher.clientset.BatchV1().Jobs(targetNamespace).List( + ctx, + metav1.ListOptions{LabelSelector: "runId=" + strings.TrimSpace(runID)}, + ) + if err != nil { + return err + } + for _, job := range jobs.Items { + if err := launcher.TerminateJob(ctx, targetNamespace, job.Name); err != nil { + return err + } + } + return nil +} + +func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*LaunchResult, error) { + job := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, launcher.workspacePVCName, launcher.workspaceMountPath, input) created, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Create(ctx, job, metav1.CreateOptions{}) if err != nil { return nil, err } jobName := created.Name reference := &JobReference{JobName: jobName, Namespace: launcher.namespace} - for attempt := 0; attempt < 300; attempt++ { + + waitCtx, cancel := context.WithDeadline(ctx, waitDeadlineForLaunch(ctx, input, time.Now())) + defer cancel() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + checkStatus := func() (*LaunchResult, error) { status, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Get(ctx, jobName, metav1.GetOptions{}) if err != nil { return nil, err @@ -224,7 +315,60 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc } return nil, fmt.Errorf("kubernetes job %s failed: %s", jobName, strings.TrimSpace(output)) } - time.Sleep(time.Second) + return nil, nil + } + + for { + result, err := checkStatus() + if result != nil || err != nil { + return result, err + } + select { + case <-waitCtx.Done(): + if errors.Is(waitCtx.Err(), context.Canceled) { + _ = launcher.TerminateJob(context.Background(), launcher.namespace, jobName) + } + if errors.Is(waitCtx.Err(), context.DeadlineExceeded) { + return nil, fmt.Errorf("timed out waiting for kubernetes job %s", jobName) + } + return nil, waitCtx.Err() + case <-ticker.C: + } + } +} + +func (launcher *Launcher) Inspect(ctx context.Context, runID string, nodeID string) (*JobInspection, error) { + selectorParts := []string{"runId=" + strings.TrimSpace(runID)} + if trimmedNodeID := strings.TrimSpace(nodeID); trimmedNodeID != "" { + selectorParts = append(selectorParts, "nodeId="+trimmedNodeID) + } + jobs, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).List(ctx, metav1.ListOptions{ + LabelSelector: strings.Join(selectorParts, ","), + }) + if err != nil { + return nil, err + } + if len(jobs.Items) == 0 { + return &JobInspection{State: JobStateMissing}, nil + } + sort.Slice(jobs.Items, func(i int, j int) bool { + return jobs.Items[i].CreationTimestamp.Time.Before(jobs.Items[j].CreationTimestamp.Time) + }) + job := jobs.Items[len(jobs.Items)-1] + reference := &JobReference{JobName: job.Name, Namespace: launcher.namespace} + switch { + case job.Status.Succeeded > 0: + output, err := launcher.readJobOutput(ctx, job.Name) + if err != nil { + return nil, err + } + return &JobInspection{Reference: reference, State: JobStateSucceeded, Output: output}, nil + case job.Status.Failed > 0: + output, _ := launcher.readJobOutput(ctx, job.Name) + return &JobInspection{Reference: reference, State: JobStateFailed, Output: output}, nil + case job.Status.Active > 0: + return &JobInspection{Reference: reference, State: JobStateActive}, nil + default: + return &JobInspection{Reference: reference, State: JobStatePending}, nil } - return nil, fmt.Errorf("timed out waiting for kubernetes job %s", jobName) } diff --git a/internal/k8s/launcher_test.go b/internal/k8s/launcher_test.go index 863bf19c..1c0332d5 100644 --- a/internal/k8s/launcher_test.go +++ b/internal/k8s/launcher_test.go @@ -1,9 +1,17 @@ package k8s -import "testing" +import ( + "context" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) func TestCreateJobManifest(t *testing.T) { - manifest := CreateJobManifest("droboticflow", "droboticflow-registry", LaunchInput{ + manifest := CreateJobManifest("droboticflow", "droboticflow-registry", "droboticflow-workspace", "/workspace", LaunchInput{ Image: "ccr.example.com/dataflow/operator-runner:latest", RunID: "run-1", WorkflowID: "wf-1", @@ -24,6 +32,12 @@ func TestCreateJobManifest(t *testing.T) { if len(manifest.Spec.Template.Spec.ImagePullSecrets) == 0 || manifest.Spec.Template.Spec.ImagePullSecrets[0].Name != "droboticflow-registry" { t.Fatalf("expected image pull secret") } + if len(manifest.Spec.Template.Spec.Volumes) != 1 || manifest.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim == nil || manifest.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != "droboticflow-workspace" { + t.Fatalf("expected workspace pvc mount") + } + if len(manifest.Spec.Template.Spec.Containers[0].VolumeMounts) != 1 || manifest.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath != "/workspace" { + t.Fatalf("expected workspace volume mount") + } foundNodeLabel := false for _, item := range manifest.Spec.Template.Spec.Containers[0].Env { if item.Name == "NODE_LABEL" && item.Value == "模型推理" { @@ -34,3 +48,80 @@ func TestCreateJobManifest(t *testing.T) { t.Fatalf("expected NODE_LABEL env var") } } + +func TestNewLauncherRequiresWorkspacePVCName(t *testing.T) { + if _, err := NewLauncher("droboticflow", "registry", "", "/workspace"); err == nil { + t.Fatalf("expected workspace pvc validation error") + } +} + +func TestWaitDeadlineForLaunchUsesInputTimeoutWhenContextHasNoDeadline(t *testing.T) { + now := time.Date(2026, time.April, 12, 0, 0, 0, 0, time.UTC) + deadline := waitDeadlineForLaunch(context.Background(), LaunchInput{TimeoutSeconds: 600}, now) + expected := now.Add(10*time.Minute + 30*time.Second) + if !deadline.Equal(expected) { + t.Fatalf("expected %s, got %s", expected, deadline) + } +} + +func TestWaitDeadlineForLaunchUsesDefaultForLongRunningJobs(t *testing.T) { + now := time.Date(2026, time.April, 12, 0, 0, 0, 0, time.UTC) + deadline := waitDeadlineForLaunch(context.Background(), LaunchInput{}, now) + expected := now.Add(defaultJobWaitTimeout) + if !deadline.Equal(expected) { + t.Fatalf("expected %s, got %s", expected, deadline) + } +} + +func TestWaitDeadlineForLaunchRespectsExistingContextDeadline(t *testing.T) { + now := time.Date(2026, time.April, 12, 0, 0, 0, 0, time.UTC) + ctx, cancel := context.WithDeadline(context.Background(), now.Add(3*time.Minute)) + defer cancel() + deadline := waitDeadlineForLaunch(ctx, LaunchInput{TimeoutSeconds: 600}, now) + expected := now.Add(3 * time.Minute) + if !deadline.Equal(expected) { + t.Fatalf("expected %s, got %s", expected, deadline) + } +} + +func TestTerminateRunDeletesJobsOneByOne(t *testing.T) { + clientset := fake.NewSimpleClientset( + &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "run-1-node-a", + Namespace: "droboticflow", + Labels: map[string]string{"runId": "run-1"}, + }, + }, + &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "run-1-node-b", + Namespace: "droboticflow", + Labels: map[string]string{"runId": "run-1"}, + }, + }, + &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "run-2-node-a", + Namespace: "droboticflow", + Labels: map[string]string{"runId": "run-2"}, + }, + }, + ) + launcher := &Launcher{ + namespace: "droboticflow", + clientset: clientset, + } + + if err := launcher.TerminateRun(context.Background(), "droboticflow", "run-1"); err != nil { + t.Fatalf("TerminateRun returned error: %v", err) + } + + list, err := clientset.BatchV1().Jobs("droboticflow").List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing jobs: %v", err) + } + if len(list.Items) != 1 || list.Items[0].Name != "run-2-node-a" { + t.Fatalf("expected only unrelated job to remain, got %#v", list.Items) + } +} diff --git a/internal/store/database_browser_helpers.go b/internal/store/database_browser_helpers.go new file mode 100644 index 00000000..470f2a41 --- /dev/null +++ b/internal/store/database_browser_helpers.go @@ -0,0 +1,141 @@ +package store + +import ( + "encoding/json" + "fmt" + "sort" + "strings" +) + +var sensitiveDatabaseKeys = map[string]struct{}{ + "accesskeyid": {}, + "secretaccesskey": {}, + "apikey": {}, + "tokenhash": {}, + "kubeconfig": {}, + "mongouri": {}, + "password": {}, + "token": {}, +} + +func databaseBrowserMaskedValue(value any) string { + compact := strings.Join(strings.Fields(strings.TrimSpace(fmt.Sprint(value))), "") + if compact == "" { + compact = strings.TrimSpace(fmt.Sprint(value)) + } + if compact == "" { + return "" + } + runes := []rune(compact) + switch length := len(runes); { + case length <= 2: + return strings.Repeat("*", length) + case length <= 6: + return string(runes[:1]) + "****" + string(runes[length-1:]) + case length <= 10: + return string(runes[:2]) + "****" + string(runes[length-2:]) + default: + return string(runes[:4]) + "****" + string(runes[length-4:]) + } +} + +func sanitizeDatabaseBrowserValue(key string, value any) any { + normalizedKey := strings.ToLower(strings.TrimSpace(key)) + if _, sensitive := sensitiveDatabaseKeys[normalizedKey]; sensitive { + return databaseBrowserMaskedValue(value) + } + if normalizedKey == "uri" { + if text := strings.TrimSpace(fmt.Sprint(value)); strings.HasPrefix(strings.ToLower(text), "mongodb://") || strings.HasPrefix(strings.ToLower(text), "mongodb+srv://") { + return databaseBrowserMaskedValue(value) + } + } + switch typed := value.(type) { + case map[string]any: + return sanitizeDatabaseBrowserDocument(typed) + case []any: + items := make([]any, 0, len(typed)) + for _, item := range typed { + items = append(items, sanitizeDatabaseBrowserValue("", item)) + } + return items + default: + return typed + } +} + +func sanitizeDatabaseBrowserDocument(document map[string]any) map[string]any { + if len(document) == 0 { + return map[string]any{} + } + sanitized := make(map[string]any, len(document)) + for key, value := range document { + sanitized[key] = sanitizeDatabaseBrowserValue(key, value) + } + return sanitized +} + +func databaseBrowserSampleFields(document map[string]any) []string { + if len(document) == 0 { + return []string{} + } + fields := make([]string, 0, len(document)) + for key := range document { + fields = append(fields, key) + } + sort.Strings(fields) + if len(fields) > 8 { + fields = fields[:8] + } + return fields +} + +func documentFromValue(value any) map[string]any { + if value == nil { + return map[string]any{} + } + if typed, ok := value.(map[string]any); ok { + return sanitizeDatabaseBrowserDocument(typed) + } + data, _ := json.Marshal(value) + document := map[string]any{} + _ = json.Unmarshal(data, &document) + return sanitizeDatabaseBrowserDocument(document) +} + +func databaseFieldValue(document map[string]any, field string) any { + current := any(document) + for _, token := range strings.Split(field, ".") { + token = strings.TrimSpace(token) + if token == "" { + continue + } + next, ok := current.(map[string]any) + if !ok { + return nil + } + current = next[token] + } + return current +} + +func matchesDatabaseBrowserFilter(document map[string]any, filter map[string]any) bool { + if len(filter) == 0 { + return true + } + for key, expected := range filter { + actual := databaseFieldValue(document, key) + if nested, ok := expected.(map[string]any); ok { + if operator, ok := nested["$exists"]; ok { + expectedExists, _ := operator.(bool) + if expectedExists != (actual != nil) { + return false + } + continue + } + } + if fmt.Sprint(actual) != fmt.Sprint(expected) { + return false + } + } + return true +} diff --git a/internal/store/helpers.go b/internal/store/helpers.go index a2543f01..327f320b 100644 --- a/internal/store/helpers.go +++ b/internal/store/helpers.go @@ -31,6 +31,57 @@ func computeRunDuration(run *domain.RunRecord) { run.DurationMS = endedAt.Sub(startedAt).Milliseconds() } +func hydrateRunDerivedFields(run *domain.RunRecord) { + if run == nil { + return + } + run.Logs = make([]domain.LogEntryRecord, 0) + run.ResourceSeries = make([]domain.ResourceSample, 0) + run.Artifacts = make([]domain.ExecutionArtifact, 0) + for _, execution := range run.NodeExecutions { + if len(execution.Logs) > 0 { + run.Logs = append(run.Logs, execution.Logs...) + } + if len(execution.ResourceSeries) > 0 { + run.ResourceSeries = append(run.ResourceSeries, execution.ResourceSeries...) + } + if len(execution.Artifacts) > 0 { + run.Artifacts = append(run.Artifacts, execution.Artifacts...) + } + } +} + +func compactRunForStorage(run domain.RunRecord) domain.RunRecord { + stored := clone(run) + stored.Logs = nil + stored.ResourceSeries = nil + stored.Artifacts = nil + return stored +} + +func finalizeNodeExecutionsForStop(run *domain.RunRecord, mode domain.RunStopMode, endedAt string) { + if run == nil { + return + } + message := "用户已停止工作流" + if mode == domain.RunStopModeForce { + message = "用户已强制停止工作流" + } + for index := range run.NodeExecutions { + execution := &run.NodeExecutions[index] + switch execution.Status { + case domain.NodeExecutionStatusQueued, domain.NodeExecutionStatusRunning, domain.NodeExecutionStatusWaitingForAudit: + execution.Status = domain.NodeExecutionStatusCancelled + if strings.TrimSpace(execution.EndedAt) == "" { + execution.EndedAt = endedAt + } + if strings.TrimSpace(execution.Error) == "" { + execution.Error = message + } + } + } +} + func parseISOTime(value string) (time.Time, bool) { if value == "" { return time.Time{}, false @@ -74,8 +125,19 @@ func defaultSettings() domain.SettingsRecord { return domain.SettingsRecord{ ID: "system", K8s: domain.K8sSettings{ - Enabled: false, - Namespace: "droboticflow", + Namespace: func() string { + if value := strings.TrimSpace(os.Getenv("K8S_NAMESPACE")); value != "" { + return value + } + return "droboticflow" + }(), + WorkspacePVCName: strings.TrimSpace(os.Getenv("WORKSPACE_PVC_NAME")), + WorkspaceMountPath: func() string { + if value := strings.TrimSpace(os.Getenv("WORKSPACE_MOUNT_PATH")); value != "" { + return value + } + return "/workspace" + }(), }, Worker: domain.WorkerSettings{ PollIntervalMS: 2000, @@ -90,6 +152,13 @@ func defaultSettings() domain.SettingsRecord { BaseURL: "https://coding.dashscope.aliyuncs.com/v1", Model: "glm-5", }, + BusinessMongo: domain.BusinessMongoSettings{ + URI: strings.TrimSpace(os.Getenv("BUSINESS_MONGO_URI")), + Database: strings.TrimSpace(os.Getenv("BUSINESS_MONGO_DB")), + }, + ExternalBusinessWrite: domain.ExternalBusinessWriteSettings{ + Tokens: []domain.ExternalBusinessWriteTokenRecord{}, + }, Timezone: "Asia/Shanghai", Defaults: domain.DefaultResources{ CPURequest: "500m", @@ -109,6 +178,12 @@ func ensureSettingsDefaults(item domain.SettingsRecord) domain.SettingsRecord { if item.K8s.Namespace == "" { item.K8s.Namespace = defaults.K8s.Namespace } + if item.K8s.WorkspacePVCName == "" { + item.K8s.WorkspacePVCName = defaults.K8s.WorkspacePVCName + } + if item.K8s.WorkspaceMountPath == "" { + item.K8s.WorkspaceMountPath = defaults.K8s.WorkspaceMountPath + } if item.Worker.PollIntervalMS <= 0 { item.Worker.PollIntervalMS = defaults.Worker.PollIntervalMS } @@ -127,6 +202,15 @@ func ensureSettingsDefaults(item domain.SettingsRecord) domain.SettingsRecord { if item.Timezone == "" { item.Timezone = defaults.Timezone } + if item.BusinessMongo.Database == "" { + item.BusinessMongo.Database = defaults.BusinessMongo.Database + } + if item.BusinessMongo.URI == "" { + item.BusinessMongo.URI = defaults.BusinessMongo.URI + } + if item.ExternalBusinessWrite.Tokens == nil { + item.ExternalBusinessWrite.Tokens = defaults.ExternalBusinessWrite.Tokens + } if item.Defaults.CPURequest == "" { item.Defaults.CPURequest = defaults.Defaults.CPURequest } diff --git a/internal/store/helpers_test.go b/internal/store/helpers_test.go new file mode 100644 index 00000000..e02027b1 --- /dev/null +++ b/internal/store/helpers_test.go @@ -0,0 +1,66 @@ +package store + +import ( + "testing" + + "droboticflow/internal/domain" +) + +func TestCompactRunForStorageDropsTopLevelExecutionCopies(t *testing.T) { + run := domain.RunRecord{ + ID: "run-1", + NodeExecutions: []domain.NodeExecutionRecord{ + { + NodeID: "node-1", + NodeLabel: "目录扫描", + Status: domain.NodeExecutionStatusCompleted, + Logs: []domain.LogEntryRecord{ + {Timestamp: "2026-04-21T00:00:00Z", Level: "INFO", Message: "scan started"}, + }, + ResourceSeries: []domain.ResourceSample{ + {Timestamp: "2026-04-21T00:00:01Z", CPU: 10, Memory: 128}, + }, + Artifacts: []domain.ExecutionArtifact{ + {ID: "artifact-1", RunID: "run-1", NodeID: "node-1", Name: "manifest.json", Type: domain.PortTypeJSON, Size: "1 KB", Path: "/tmp/manifest.json"}, + }, + }, + }, + Logs: []domain.LogEntryRecord{ + {Timestamp: "2026-04-21T00:00:00Z", Level: "INFO", Message: "duplicated"}, + }, + ResourceSeries: []domain.ResourceSample{ + {Timestamp: "2026-04-21T00:00:01Z", CPU: 99, Memory: 2048}, + }, + Artifacts: []domain.ExecutionArtifact{ + {ID: "artifact-duplicate", RunID: "run-1", NodeID: "node-1", Name: "duplicate.json", Type: domain.PortTypeJSON, Size: "1 KB", Path: "/tmp/duplicate.json"}, + }, + } + + compacted := compactRunForStorage(run) + if compacted.ID != run.ID { + t.Fatalf("compacted run id = %q, want %q", compacted.ID, run.ID) + } + if len(compacted.NodeExecutions) != 1 { + t.Fatalf("compacted node executions = %d, want 1", len(compacted.NodeExecutions)) + } + if compacted.Logs != nil { + t.Fatalf("compacted logs should be nil, got %#v", compacted.Logs) + } + if compacted.ResourceSeries != nil { + t.Fatalf("compacted resource series should be nil, got %#v", compacted.ResourceSeries) + } + if compacted.Artifacts != nil { + t.Fatalf("compacted artifacts should be nil, got %#v", compacted.Artifacts) + } + + hydrateRunDerivedFields(&compacted) + if len(compacted.Logs) != 1 || compacted.Logs[0].Message != "scan started" { + t.Fatalf("hydrated logs = %#v", compacted.Logs) + } + if len(compacted.ResourceSeries) != 1 || compacted.ResourceSeries[0].CPU != 10 { + t.Fatalf("hydrated resource series = %#v", compacted.ResourceSeries) + } + if len(compacted.Artifacts) != 1 || compacted.Artifacts[0].ID != "artifact-1" { + t.Fatalf("hydrated artifacts = %#v", compacted.Artifacts) + } +} diff --git a/internal/store/memory.go b/internal/store/memory.go index 41ad9cd5..2dc49640 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -12,26 +12,40 @@ import ( ) type MemoryStore struct { - mu sync.Mutex - projects map[string]domain.ProjectRecord - operators map[string]domain.OperatorRecord - workflows map[string]domain.WorkflowRecord - templates map[string]domain.TemplateRecord - runs map[string]domain.RunRecord - auditTasks map[string]domain.AuditTaskRecord - users map[string]domain.UserRecord - settings *domain.SettingsRecord + mu sync.Mutex + projects map[string]domain.ProjectRecord + operators map[string]domain.OperatorRecord + builtinOperatorDeletions map[string]domain.BuiltinOperatorDeletionRecord + workflows map[string]domain.WorkflowRecord + templates map[string]domain.TemplateRecord + runs map[string]domain.RunRecord + sourceStates map[string]domain.SourceStateRecord + auditTasks map[string]domain.AuditTaskRecord + samplingTasks map[string]domain.SamplingTaskRecord + users map[string]domain.UserRecord + storageConnections map[string]domain.StorageConnectionRecord + mongoConnections map[string]domain.MongoConnectionRecord + databaseCollections map[string][]map[string]any + databaseIndexes map[string][]domain.DatabaseIndexSummary + settings *domain.SettingsRecord } func NewMemoryStore() *MemoryStore { return &MemoryStore{ - projects: map[string]domain.ProjectRecord{}, - operators: map[string]domain.OperatorRecord{}, - workflows: map[string]domain.WorkflowRecord{}, - templates: map[string]domain.TemplateRecord{}, - runs: map[string]domain.RunRecord{}, - auditTasks: map[string]domain.AuditTaskRecord{}, - users: map[string]domain.UserRecord{}, + projects: map[string]domain.ProjectRecord{}, + operators: map[string]domain.OperatorRecord{}, + builtinOperatorDeletions: map[string]domain.BuiltinOperatorDeletionRecord{}, + workflows: map[string]domain.WorkflowRecord{}, + templates: map[string]domain.TemplateRecord{}, + runs: map[string]domain.RunRecord{}, + sourceStates: map[string]domain.SourceStateRecord{}, + auditTasks: map[string]domain.AuditTaskRecord{}, + samplingTasks: map[string]domain.SamplingTaskRecord{}, + users: map[string]domain.UserRecord{}, + storageConnections: map[string]domain.StorageConnectionRecord{}, + mongoConnections: map[string]domain.MongoConnectionRecord{}, + databaseCollections: map[string][]map[string]any{}, + databaseIndexes: map[string][]domain.DatabaseIndexSummary{}, } } @@ -43,6 +57,10 @@ func sortOperators(items []domain.OperatorRecord) { sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) } +func sortBuiltinOperatorDeletions(items []domain.BuiltinOperatorDeletionRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].DeletedAt > items[right].DeletedAt }) +} + func sortWorkflows(items []domain.WorkflowRecord) { sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) } @@ -59,10 +77,22 @@ func sortAuditTasks(items []domain.AuditTaskRecord) { sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) } +func sortSamplingTasks(items []domain.SamplingTaskRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + func sortUsers(items []domain.UserRecord) { sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt < items[right].CreatedAt }) } +func sortStorageConnections(items []domain.StorageConnectionRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + +func sortMongoConnections(items []domain.MongoConnectionRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + func (store *MemoryStore) ListProjects(context.Context) ([]domain.ProjectRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -141,6 +171,26 @@ func (store *MemoryStore) DeleteProject(_ context.Context, id string) (bool, err } } } + for taskID, task := range store.samplingTasks { + if task.ProjectID == id { + delete(store.samplingTasks, taskID) + continue + } + for _, workflowID := range workflowIDs { + if task.WorkflowID == workflowID { + delete(store.samplingTasks, taskID) + break + } + } + } + for stateID, state := range store.sourceStates { + for _, workflowID := range workflowIDs { + if state.WorkflowID == workflowID { + delete(store.sourceStates, stateID) + break + } + } + } return true, nil } @@ -215,6 +265,24 @@ func (store *MemoryStore) UpsertOperator(_ context.Context, item domain.Operator return nil } +func (store *MemoryStore) ListBuiltinOperatorDeletions(context.Context) ([]domain.BuiltinOperatorDeletionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.BuiltinOperatorDeletionRecord, 0, len(store.builtinOperatorDeletions)) + for _, item := range store.builtinOperatorDeletions { + items = append(items, clone(item)) + } + sortBuiltinOperatorDeletions(items) + return items, nil +} + +func (store *MemoryStore) UpsertBuiltinOperatorDeletion(_ context.Context, item domain.BuiltinOperatorDeletionRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.builtinOperatorDeletions[item.Key] = clone(item) + return nil +} + func (store *MemoryStore) ListWorkflowsByProject(_ context.Context, projectID string) ([]domain.WorkflowRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -260,6 +328,9 @@ func (store *MemoryStore) CreateWorkflow(_ context.Context, input domain.CreateW Name: input.Name, Description: input.Description, Type: input.Type, + Kind: input.Kind, + ScanConfig: clone(input.ScanConfig), + Hyperparams: clone(input.Hyperparams), MaxConcurrentRuns: input.MaxConcurrentRuns, NodeTimeoutSeconds: input.NodeTimeoutSeconds, RunTimeoutSeconds: input.RunTimeoutSeconds, @@ -294,6 +365,16 @@ func (store *MemoryStore) DeleteWorkflow(_ context.Context, id string) (bool, er delete(store.auditTasks, taskID) } } + for taskID, task := range store.samplingTasks { + if task.WorkflowID == id { + delete(store.samplingTasks, taskID) + } + } + for stateID, state := range store.sourceStates { + if state.WorkflowID == id { + delete(store.sourceStates, stateID) + } + } return true, nil } @@ -339,6 +420,9 @@ func (store *MemoryStore) CreateTemplate(_ context.Context, input domain.CreateT Name: input.Name, Description: input.Description, Type: templateType, + Kind: input.Kind, + ScanConfig: clone(input.ScanConfig), + Hyperparams: clone(input.Hyperparams), MaxConcurrentRuns: input.MaxConcurrentRuns, NodeTimeoutSeconds: input.NodeTimeoutSeconds, RunTimeoutSeconds: input.RunTimeoutSeconds, @@ -385,6 +469,53 @@ func (store *MemoryStore) ListRuns(_ context.Context, status string) ([]domain.R return items, nil } +func summarizeRun(run domain.RunRecord) domain.RunSummary { + return domain.RunSummary{ + ID: run.ID, + WorkflowID: run.WorkflowID, + WorkflowName: run.WorkflowName, + ProjectID: run.ProjectID, + ProjectName: run.ProjectName, + Trigger: run.Trigger, + Attempt: run.Attempt, + ParentRunID: run.ParentRunID, + RestartFromNodeID: run.RestartFromNodeID, + AvailableAt: run.AvailableAt, + Status: run.Status, + StopMode: run.StopMode, + StopRequestedAt: run.StopRequestedAt, + StartedAt: run.StartedAt, + EndedAt: run.EndedAt, + DurationMS: run.DurationMS, + CurrentNodeID: run.CurrentNodeID, + Error: run.Error, + CreatedAt: run.CreatedAt, + UpdatedAt: run.UpdatedAt, + } +} + +func sortRunSummaries(items []domain.RunSummary) { + sort.Slice(items, func(i, j int) bool { + if items[i].CreatedAt == items[j].CreatedAt { + return items[i].ID > items[j].ID + } + return items[i].CreatedAt > items[j].CreatedAt + }) +} + +func (store *MemoryStore) ListRunSummaries(_ context.Context, status string) ([]domain.RunSummary, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.RunSummary, 0) + for _, run := range store.runs { + if status == "" || string(run.Status) == status { + items = append(items, summarizeRun(run)) + } + } + sortRunSummaries(items) + return items, nil +} + func (store *MemoryStore) GetRun(_ context.Context, id string) (*domain.RunRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -403,25 +534,32 @@ func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigge if !ok { return nil, fmt.Errorf("workflow %s not found", workflowID) } - if workflow.Status == domain.WorkflowStatusCompleted && trigger != domain.RunTriggerScheduled { + if workflow.Status == domain.WorkflowStatusCompleted && workflow.Kind != domain.WorkflowKindScan && trigger != domain.RunTriggerScheduled { return nil, ErrWorkflowAlreadyCompleted } - project, ok := store.projects[workflow.ProjectID] - if !ok { - return nil, fmt.Errorf("project %s not found", workflow.ProjectID) + projectName := "" + if workflow.ProjectID != "" { + project, ok := store.projects[workflow.ProjectID] + if !ok { + return nil, fmt.Errorf("project %s not found", workflow.ProjectID) + } + projectName = project.Name } now := nowISO() + previousRunAt := workflow.LastRunAt run := domain.RunRecord{ ID: domain.NewID("run"), WorkflowID: workflow.ID, WorkflowName: workflow.Name, - ProjectID: project.ID, - ProjectName: project.Name, + ProjectID: workflow.ProjectID, + ProjectName: projectName, Trigger: trigger, Attempt: 1, AvailableAt: now, Status: domain.RunStatusQueued, + PreviousRunAt: previousRunAt, + CurrentRunAt: now, NodeExecutions: []domain.NodeExecutionRecord{}, Logs: []domain.LogEntryRecord{}, ResourceSeries: []domain.ResourceSample{}, @@ -490,6 +628,54 @@ func (store *MemoryStore) ClaimQueuedRun(_ context.Context, runID string) (*doma return &cloned, nil } +func (store *MemoryStore) RequestRunStop(_ context.Context, runID string, mode domain.RunStopMode) (*domain.RunRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + + run, ok := store.runs[runID] + if !ok { + return nil, nil + } + if run.Status == domain.RunStatusCompleted || run.Status == domain.RunStatusFailed || run.Status == domain.RunStatusCancelled { + cloned := clone(run) + return &cloned, nil + } + + now := nowISO() + run.StopMode = mode + run.StopRequestedAt = now + run.UpdatedAt = now + + switch mode { + case domain.RunStopModeGraceful: + if run.Status == domain.RunStatusQueued || run.Status == domain.RunStatusWaitingForAudit { + run.Status = domain.RunStatusCancelled + run.EndedAt = now + finalizeNodeExecutionsForStop(&run, mode, now) + if strings.TrimSpace(run.Error) == "" { + run.Error = "用户已停止工作流" + } + } else { + run.Status = domain.RunStatusStopping + } + case domain.RunStopModeForce: + run.Status = domain.RunStatusCancelled + run.EndedAt = now + finalizeNodeExecutionsForStop(&run, mode, now) + if strings.TrimSpace(run.Error) == "" { + run.Error = "用户已强制停止工作流" + } + default: + cloned := clone(run) + return &cloned, nil + } + + computeRunDuration(&run) + store.runs[run.ID] = clone(run) + cloned := clone(run) + return &cloned, nil +} + func (store *MemoryStore) UpsertRun(_ context.Context, run domain.RunRecord) error { store.mu.Lock() defer store.mu.Unlock() @@ -498,6 +684,65 @@ func (store *MemoryStore) UpsertRun(_ context.Context, run domain.RunRecord) err return nil } +func sourceStateKey(workflowID string, nodeID string, sourceFingerprint string) string { + return workflowID + ":" + nodeID + ":" + sourceFingerprint +} + +func (store *MemoryStore) GetSourceState(_ context.Context, workflowID string, nodeID string, sourceFingerprint string) (*domain.SourceStateRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.sourceStates[sourceStateKey(workflowID, nodeID, sourceFingerprint)] + if !ok { + return nil, nil + } + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) ListSourceStates(_ context.Context, workflowID string, nodeID string) ([]domain.SourceStateRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.SourceStateRecord, 0) + for _, item := range store.sourceStates { + if item.WorkflowID != workflowID { + continue + } + if nodeID != "" && item.NodeID != nodeID { + continue + } + items = append(items, clone(item)) + } + return items, nil +} + +func (store *MemoryStore) UpsertSourceState(_ context.Context, item domain.SourceStateRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + if item.ID == "" { + item.ID = sourceStateKey(item.WorkflowID, item.NodeID, item.SourceFingerprint) + } + store.sourceStates[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) DeleteSourceState(_ context.Context, workflowID string, nodeID string, sourceFingerprint string) error { + store.mu.Lock() + defer store.mu.Unlock() + delete(store.sourceStates, sourceStateKey(workflowID, nodeID, sourceFingerprint)) + return nil +} + +func (store *MemoryStore) DeleteSourceStatesByWorkflow(_ context.Context, workflowID string) error { + store.mu.Lock() + defer store.mu.Unlock() + for stateID, state := range store.sourceStates { + if state.WorkflowID == workflowID { + delete(store.sourceStates, stateID) + } + } + return nil +} + func (store *MemoryStore) ListAuditTasks(_ context.Context, status string) ([]domain.AuditTaskRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -537,6 +782,48 @@ func (store *MemoryStore) UpsertAuditTask(_ context.Context, task domain.AuditTa return nil } +func (store *MemoryStore) ListSamplingTasks(_ context.Context, status string) ([]domain.SamplingTaskRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.SamplingTaskRecord, 0, len(store.samplingTasks)) + for _, task := range store.samplingTasks { + if status == "" || string(task.Status) == status { + items = append(items, clone(task)) + } + } + sortSamplingTasks(items) + return items, nil +} + +func (store *MemoryStore) GetSamplingTask(_ context.Context, id string) (*domain.SamplingTaskRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + task, ok := store.samplingTasks[id] + if !ok { + return nil, nil + } + cloned := clone(task) + return &cloned, nil +} + +func (store *MemoryStore) CreateSamplingTask(_ context.Context, task domain.SamplingTaskRecord) (*domain.SamplingTaskRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + if strings.TrimSpace(task.ID) == "" { + task.ID = domain.NewID("sampling") + } + store.samplingTasks[task.ID] = clone(task) + cloned := clone(task) + return &cloned, nil +} + +func (store *MemoryStore) UpsertSamplingTask(_ context.Context, task domain.SamplingTaskRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.samplingTasks[task.ID] = clone(task) + return nil +} + func (store *MemoryStore) ListUsers(context.Context) ([]domain.UserRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -593,6 +880,123 @@ func (store *MemoryStore) UpsertUser(_ context.Context, item domain.UserRecord) return nil } +func (store *MemoryStore) ListStorageConnections(context.Context) ([]domain.StorageConnectionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.StorageConnectionRecord, 0, len(store.storageConnections)) + for _, item := range store.storageConnections { + items = append(items, clone(item)) + } + sortStorageConnections(items) + return items, nil +} + +func (store *MemoryStore) GetStorageConnection(_ context.Context, id string) (*domain.StorageConnectionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.storageConnections[id] + if !ok { + return nil, nil + } + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) CreateStorageConnection(_ context.Context, input domain.CreateStorageConnectionInput) (*domain.StorageConnectionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + now := nowISO() + item := domain.StorageConnectionRecord{ + ID: domain.NewID("storage"), + Name: input.Name, + Description: input.Description, + Provider: input.Provider, + Endpoint: input.Endpoint, + Bucket: input.Bucket, + AccessKeyID: input.AccessKeyID, + SecretAccessKey: input.SecretAccessKey, + CreatedAt: now, + UpdatedAt: now, + } + store.storageConnections[item.ID] = clone(item) + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) DeleteStorageConnection(_ context.Context, id string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + if _, ok := store.storageConnections[id]; !ok { + return false, nil + } + delete(store.storageConnections, id) + return true, nil +} + +func (store *MemoryStore) UpsertStorageConnection(_ context.Context, item domain.StorageConnectionRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.storageConnections[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) ListMongoConnections(context.Context) ([]domain.MongoConnectionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.MongoConnectionRecord, 0, len(store.mongoConnections)) + for _, item := range store.mongoConnections { + items = append(items, clone(item)) + } + sortMongoConnections(items) + return items, nil +} + +func (store *MemoryStore) GetMongoConnection(_ context.Context, id string) (*domain.MongoConnectionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.mongoConnections[id] + if !ok { + return nil, nil + } + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) CreateMongoConnection(_ context.Context, input domain.CreateMongoConnectionInput) (*domain.MongoConnectionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + now := nowISO() + item := domain.MongoConnectionRecord{ + ID: domain.NewID("mongo"), + Name: input.Name, + Description: input.Description, + URI: input.URI, + Database: input.Database, + CreatedAt: now, + UpdatedAt: now, + } + store.mongoConnections[item.ID] = clone(item) + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) DeleteMongoConnection(_ context.Context, id string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + if _, ok := store.mongoConnections[id]; !ok { + return false, nil + } + delete(store.mongoConnections, id) + return true, nil +} + +func (store *MemoryStore) UpsertMongoConnection(_ context.Context, item domain.MongoConnectionRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.mongoConnections[item.ID] = clone(item) + return nil +} + func (store *MemoryStore) GetSettings(_ context.Context) (*domain.SettingsRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -621,7 +1025,7 @@ func (store *MemoryStore) GetDashboardSummary(ctx context.Context) (domain.Dashb if err != nil { return domain.DashboardSummary{}, err } - runs, err := store.ListRuns(ctx, "") + runs, err := store.ListRunSummaries(ctx, "") if err != nil { return domain.DashboardSummary{}, err } @@ -636,12 +1040,12 @@ func (store *MemoryStore) GetDashboardSummary(ctx context.Context) (domain.Dashb func buildDashboardSummary( projects []domain.ProjectRecord, operators []domain.OperatorRecord, - runs []domain.RunRecord, + runs []domain.RunSummary, auditTasks []domain.AuditTaskRecord, ) domain.DashboardSummary { - completedRuns := make([]domain.RunRecord, 0) - failedRuns := make([]domain.RunRecord, 0) - runningRuns := make([]domain.RunRecord, 0) + completedRuns := make([]domain.RunSummary, 0) + failedRuns := make([]domain.RunSummary, 0) + runningRuns := make([]domain.RunSummary, 0) for _, run := range runs { switch run.Status { case domain.RunStatusCompleted: @@ -709,6 +1113,493 @@ func (store *MemoryStore) ListStorageObjects(ctx context.Context, sourcePath str return listObjectsFromSourcePath(sourcePath, bucket, prefix) } +func (store *MemoryStore) databaseCollectionDocumentsLocked() map[string][]map[string]any { + collections := map[string][]map[string]any{ + "projects": {}, + "operators": {}, + "operatorTombstones": {}, + "workflows": {}, + "templates": {}, + "runs": {}, + "sourceStates": {}, + "auditTasks": {}, + "samplingTasks": {}, + "users": {}, + "storageConnections": {}, + "mongoConnections": {}, + "settings": {}, + } + for name, documents := range store.databaseCollections { + items := make([]map[string]any, 0, len(documents)) + for _, document := range documents { + items = append(items, clone(document)) + } + collections[name] = items + } + for _, item := range store.projects { + collections["projects"] = append(collections["projects"], documentFromValue(item)) + } + for _, item := range store.operators { + collections["operators"] = append(collections["operators"], documentFromValue(item)) + } + for _, item := range store.builtinOperatorDeletions { + collections["operatorTombstones"] = append(collections["operatorTombstones"], documentFromValue(item)) + } + for _, item := range store.workflows { + collections["workflows"] = append(collections["workflows"], documentFromValue(item)) + } + for _, item := range store.templates { + collections["templates"] = append(collections["templates"], documentFromValue(item)) + } + for _, item := range store.runs { + collections["runs"] = append(collections["runs"], documentFromValue(item)) + } + for _, item := range store.sourceStates { + collections["sourceStates"] = append(collections["sourceStates"], documentFromValue(item)) + } + for _, item := range store.auditTasks { + collections["auditTasks"] = append(collections["auditTasks"], documentFromValue(item)) + } + for _, item := range store.samplingTasks { + collections["samplingTasks"] = append(collections["samplingTasks"], documentFromValue(item)) + } + for _, item := range store.users { + collections["users"] = append(collections["users"], documentFromValue(item)) + } + for _, item := range store.storageConnections { + collections["storageConnections"] = append(collections["storageConnections"], documentFromValue(item)) + } + for _, item := range store.mongoConnections { + collections["mongoConnections"] = append(collections["mongoConnections"], documentFromValue(item)) + } + if store.settings != nil { + collections["settings"] = append(collections["settings"], documentFromValue(*store.settings)) + } + return collections +} + +func memoryBrowserDocumentID(document map[string]any) string { + value, ok := document["_id"] + if !ok || value == nil { + return "" + } + switch typed := value.(type) { + case string: + return typed + case map[string]any: + if oid, ok := typed["$oid"].(string); ok { + return oid + } + } + return fmt.Sprintf("%v", value) +} + +func defaultMemoryDatabaseIndexes() []domain.DatabaseIndexSummary { + return []domain.DatabaseIndexSummary{ + { + Name: "_id_", + Keys: map[string]any{"_id": 1}, + }, + } +} + +func (store *MemoryStore) CreateDatabaseCollection(_ context.Context, name string) (domain.DatabaseCollectionSummary, error) { + store.mu.Lock() + defer store.mu.Unlock() + + collectionName := strings.TrimSpace(name) + if collectionName == "" { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required") + } + if _, exists := store.databaseCollectionDocumentsLocked()[collectionName]; exists { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection %s already exists", collectionName) + } + store.databaseCollections[collectionName] = []map[string]any{} + store.databaseIndexes[collectionName] = defaultMemoryDatabaseIndexes() + return domain.DatabaseCollectionSummary{Name: collectionName, DocumentCount: 0}, nil +} + +func (store *MemoryStore) RenameDatabaseCollection(_ context.Context, name string, newName string) (domain.DatabaseCollectionSummary, error) { + store.mu.Lock() + defer store.mu.Unlock() + + currentName := strings.TrimSpace(name) + nextName := strings.TrimSpace(newName) + if currentName == "" { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required") + } + if nextName == "" { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("new collection name is required") + } + documents, ok := store.databaseCollections[currentName] + if !ok { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection %s not found", currentName) + } + if currentName != nextName { + if _, exists := store.databaseCollectionDocumentsLocked()[nextName]; exists { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection %s already exists", nextName) + } + delete(store.databaseCollections, currentName) + store.databaseCollections[nextName] = documents + store.databaseIndexes[nextName] = clone(store.databaseIndexes[currentName]) + delete(store.databaseIndexes, currentName) + } + summary := domain.DatabaseCollectionSummary{ + Name: nextName, + DocumentCount: int64(len(store.databaseCollections[nextName])), + } + if len(store.databaseCollections[nextName]) > 0 { + summary.SampleFields = databaseBrowserSampleFields(store.databaseCollections[nextName][0]) + } + return summary, nil +} + +func (store *MemoryStore) DeleteDatabaseCollection(_ context.Context, name string) error { + store.mu.Lock() + defer store.mu.Unlock() + + collectionName := strings.TrimSpace(name) + if collectionName == "" { + return fmt.Errorf("collection is required") + } + if _, ok := store.databaseCollections[collectionName]; !ok { + return fmt.Errorf("collection %s not found", collectionName) + } + delete(store.databaseCollections, collectionName) + delete(store.databaseIndexes, collectionName) + return nil +} + +func (store *MemoryStore) ListDatabaseIndexes(_ context.Context, collectionName string) ([]domain.DatabaseIndexSummary, error) { + store.mu.Lock() + defer store.mu.Unlock() + + name := strings.TrimSpace(collectionName) + if name == "" { + return nil, fmt.Errorf("collection is required") + } + indexes, ok := store.databaseIndexes[name] + if !ok { + return nil, fmt.Errorf("collection %s not found", name) + } + return clone(indexes), nil +} + +func (store *MemoryStore) CreateDatabaseIndex(_ context.Context, collectionName string, input domain.DatabaseIndexCreateInput) (domain.DatabaseIndexSummary, error) { + store.mu.Lock() + defer store.mu.Unlock() + + name := strings.TrimSpace(collectionName) + if name == "" { + return domain.DatabaseIndexSummary{}, fmt.Errorf("collection is required") + } + if _, ok := store.databaseCollections[name]; !ok { + return domain.DatabaseIndexSummary{}, fmt.Errorf("collection %s not found", name) + } + if len(input.Keys) == 0 { + return domain.DatabaseIndexSummary{}, fmt.Errorf("index keys are required") + } + indexName := strings.TrimSpace(input.Name) + if indexName == "" { + indexName = domain.NewID("idx") + } + for _, item := range store.databaseIndexes[name] { + if item.Name == indexName { + return domain.DatabaseIndexSummary{}, fmt.Errorf("index %s already exists", indexName) + } + } + summary := domain.DatabaseIndexSummary{ + Name: indexName, + Keys: clone(input.Keys), + Unique: input.Unique, + } + store.databaseIndexes[name] = append(store.databaseIndexes[name], summary) + return clone(summary), nil +} + +func (store *MemoryStore) DeleteDatabaseIndex(_ context.Context, collectionName string, indexName string) error { + store.mu.Lock() + defer store.mu.Unlock() + + name := strings.TrimSpace(collectionName) + target := strings.TrimSpace(indexName) + if name == "" { + return fmt.Errorf("collection is required") + } + if target == "" { + return fmt.Errorf("index name is required") + } + indexes, ok := store.databaseIndexes[name] + if !ok { + return fmt.Errorf("collection %s not found", name) + } + filtered := make([]domain.DatabaseIndexSummary, 0, len(indexes)) + deleted := false + for _, item := range indexes { + if item.Name == target { + if item.Name == "_id_" { + return fmt.Errorf("index %s cannot be deleted", target) + } + deleted = true + continue + } + filtered = append(filtered, item) + } + if !deleted { + return fmt.Errorf("index %s not found", target) + } + store.databaseIndexes[name] = filtered + return nil +} + +func (store *MemoryStore) InsertDatabaseDocument(_ context.Context, collectionName string, document map[string]any) (map[string]any, error) { + store.mu.Lock() + defer store.mu.Unlock() + + name := strings.TrimSpace(collectionName) + if name == "" { + return nil, fmt.Errorf("collection is required") + } + documents, ok := store.databaseCollections[name] + if !ok { + return nil, fmt.Errorf("collection %s not found", name) + } + inserted := clone(document) + if memoryBrowserDocumentID(inserted) == "" { + inserted["_id"] = domain.NewID("doc") + } + store.databaseCollections[name] = append(documents, inserted) + return clone(inserted), nil +} + +func (store *MemoryStore) ReplaceDatabaseDocument(_ context.Context, collectionName string, documentID string, document map[string]any) (map[string]any, error) { + store.mu.Lock() + defer store.mu.Unlock() + + name := strings.TrimSpace(collectionName) + id := strings.TrimSpace(documentID) + if name == "" { + return nil, fmt.Errorf("collection is required") + } + if id == "" { + return nil, fmt.Errorf("document id is required") + } + documents, ok := store.databaseCollections[name] + if !ok { + return nil, fmt.Errorf("collection %s not found", name) + } + for index, item := range documents { + if memoryBrowserDocumentID(item) != id { + continue + } + replaced := clone(document) + replaced["_id"] = item["_id"] + store.databaseCollections[name][index] = replaced + return clone(replaced), nil + } + return nil, fmt.Errorf("document %s not found", id) +} + +func (store *MemoryStore) PatchDatabaseDocument(_ context.Context, collectionName string, documentID string, patch map[string]any) (map[string]any, error) { + store.mu.Lock() + defer store.mu.Unlock() + + name := strings.TrimSpace(collectionName) + id := strings.TrimSpace(documentID) + if name == "" { + return nil, fmt.Errorf("collection is required") + } + if id == "" { + return nil, fmt.Errorf("document id is required") + } + documents, ok := store.databaseCollections[name] + if !ok { + return nil, fmt.Errorf("collection %s not found", name) + } + for index, item := range documents { + if memoryBrowserDocumentID(item) != id { + continue + } + updated := clone(item) + for key, value := range clone(patch) { + if key == "_id" { + continue + } + updated[key] = value + } + store.databaseCollections[name][index] = updated + return clone(updated), nil + } + return nil, fmt.Errorf("document %s not found", id) +} + +func (store *MemoryStore) UpsertDatabaseDocument(_ context.Context, collectionName string, match map[string]any, patch map[string]any, setOnInsert map[string]any) (map[string]any, bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + + name := strings.TrimSpace(collectionName) + if name == "" { + return nil, false, fmt.Errorf("collection is required") + } + documents, ok := store.databaseCollections[name] + if !ok { + return nil, false, fmt.Errorf("collection %s not found", name) + } + for index, item := range documents { + if !matchesDatabaseBrowserFilter(item, match) { + continue + } + updated := clone(item) + for key, value := range clone(patch) { + if key == "_id" { + continue + } + updated[key] = value + } + store.databaseCollections[name][index] = updated + return clone(updated), false, nil + } + + inserted := map[string]any{} + for key, value := range clone(match) { + if key == "_id" { + continue + } + inserted[key] = value + } + for key, value := range clone(setOnInsert) { + if key == "_id" { + continue + } + inserted[key] = value + } + for key, value := range clone(patch) { + if key == "_id" { + continue + } + inserted[key] = value + } + if memoryBrowserDocumentID(inserted) == "" { + inserted["_id"] = domain.NewID("doc") + } + store.databaseCollections[name] = append(documents, inserted) + return clone(inserted), true, nil +} + +func (store *MemoryStore) DeleteDatabaseDocument(_ context.Context, collectionName string, documentID string) error { + store.mu.Lock() + defer store.mu.Unlock() + + name := strings.TrimSpace(collectionName) + id := strings.TrimSpace(documentID) + if name == "" { + return fmt.Errorf("collection is required") + } + if id == "" { + return fmt.Errorf("document id is required") + } + documents, ok := store.databaseCollections[name] + if !ok { + return fmt.Errorf("collection %s not found", name) + } + filtered := make([]map[string]any, 0, len(documents)) + deleted := false + for _, item := range documents { + if !deleted && memoryBrowserDocumentID(item) == id { + deleted = true + continue + } + filtered = append(filtered, item) + } + if !deleted { + return fmt.Errorf("document %s not found", id) + } + store.databaseCollections[name] = filtered + return nil +} + +func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseOverview, error) { + store.mu.Lock() + defer store.mu.Unlock() + + collectionsMap := store.databaseCollectionDocumentsLocked() + names := make([]string, 0, len(collectionsMap)) + for name := range collectionsMap { + names = append(names, name) + } + sort.Strings(names) + + overview := domain.DatabaseOverview{ + Provider: "memory", + Database: "droboticflow-memory", + ReadOnly: true, + Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)), + } + for _, name := range names { + documents := collectionsMap[name] + summary := domain.DatabaseCollectionSummary{ + Name: name, + DocumentCount: int64(len(documents)), + } + if len(documents) > 0 { + summary.SampleFields = databaseBrowserSampleFields(documents[0]) + } + overview.Collections = append(overview.Collections, summary) + } + return overview, nil +} + +func (store *MemoryStore) QueryDatabaseCollection(_ context.Context, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) { + store.mu.Lock() + defer store.mu.Unlock() + + collectionsMap := store.databaseCollectionDocumentsLocked() + documents, ok := collectionsMap[input.Collection] + if !ok { + return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", input.Collection) + } + + limit := input.Limit + if limit <= 0 { + limit = 50 + } + skip := input.Skip + if skip < 0 { + skip = 0 + } + + filtered := make([]map[string]any, 0, len(documents)) + for _, item := range documents { + if matchesDatabaseBrowserFilter(item, input.Filter) { + filtered = append(filtered, item) + } + } + total := int64(len(filtered)) + if skip > len(filtered) { + skip = len(filtered) + } + end := skip + limit + if end > len(filtered) { + end = len(filtered) + } + page := filtered[skip:end] + + result := domain.DatabaseQueryResult{ + Provider: "memory", + Database: "droboticflow-memory", + Collection: input.Collection, + Filter: clone(input.Filter), + Limit: limit, + Skip: skip, + Total: total, + Documents: clone(page), + } + if len(page) > 0 { + result.SampleFields = databaseBrowserSampleFields(page[0]) + } + return result, nil +} + func MergeSettings(current domain.SettingsRecord, patch domain.SettingsRecord) domain.SettingsRecord { merged := current if patch.K8s.Namespace != "" { @@ -720,9 +1611,6 @@ func MergeSettings(current domain.SettingsRecord, patch domain.SettingsRecord) d if patch.K8s.Kubeconfig != "" { merged.K8s.Kubeconfig = patch.K8s.Kubeconfig } - if patch.K8s.Enabled != current.K8s.Enabled { - merged.K8s.Enabled = patch.K8s.Enabled - } if patch.Worker.PollIntervalMS > 0 { merged.Worker.PollIntervalMS = patch.Worker.PollIntervalMS } diff --git a/internal/store/memory_test.go b/internal/store/memory_test.go index 41dccda2..c33f48a1 100644 --- a/internal/store/memory_test.go +++ b/internal/store/memory_test.go @@ -221,6 +221,146 @@ func TestMemoryStoreCRUD(t *testing.T) { } } +func TestRequestRunStopCancelsActiveNodeExecutions(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + run := domain.RunRecord{ + ID: "run-stop-1", + WorkflowID: "workflow-1", + Status: domain.RunStatusRunning, + StartedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano), + NodeExecutions: []domain.NodeExecutionRecord{ + { + NodeID: "sleep", + NodeLabel: "Shell 命令执行", + Status: domain.NodeExecutionStatusRunning, + StartedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano), + }, + { + NodeID: "upload", + NodeLabel: "结果上传", + Status: domain.NodeExecutionStatusQueued, + }, + }, + Logs: []domain.LogEntryRecord{}, + ResourceSeries: []domain.ResourceSample{}, + Artifacts: []domain.ExecutionArtifact{}, + K8sRefs: []domain.K8sRef{}, + CreatedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano), + UpdatedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano), + } + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("UpsertRun: %v", err) + } + + stopped, err := dataStore.RequestRunStop(ctx, run.ID, domain.RunStopModeForce) + if err != nil { + t.Fatalf("RequestRunStop: %v", err) + } + if stopped.Status != domain.RunStatusCancelled { + t.Fatalf("expected cancelled run, got %s", stopped.Status) + } + if len(stopped.NodeExecutions) != 2 { + t.Fatalf("expected 2 node executions, got %d", len(stopped.NodeExecutions)) + } + for _, execution := range stopped.NodeExecutions { + if execution.Status != domain.NodeExecutionStatusCancelled { + t.Fatalf("expected cancelled execution, got %#v", execution) + } + if execution.EndedAt == "" { + t.Fatalf("expected endedAt for cancelled execution %#v", execution) + } + } +} + +func TestMemoryStoreSourceStateCRUD(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + state := domain.SourceStateRecord{ + ID: "workflow-1:node-1:fingerprint-1", + WorkflowID: "workflow-1", + NodeID: "node-1", + SourceFingerprint: "fingerprint-1", + DataSource: "BOS", + SourcePath: "bos://datastorage/algo/X-platform-test", + Bucket: "datastorage", + Prefix: "algo/X-platform-test", + ScanMode: "INCREMENTAL", + SeenFingerprints: []string{"entry-a", "entry-b"}, + LastRunID: "run-1", + LastListedCount: 2, + LastNewCount: 1, + UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano), + } + if err := dataStore.UpsertSourceState(ctx, state); err != nil { + t.Fatalf("upsert source state: %v", err) + } + + stored, err := dataStore.GetSourceState(ctx, "workflow-1", "node-1", "fingerprint-1") + if err != nil { + t.Fatalf("get source state: %v", err) + } + if stored == nil { + t.Fatalf("expected source state") + } + if stored.LastRunID != "run-1" || len(stored.SeenFingerprints) != 2 { + t.Fatalf("unexpected source state payload: %#v", stored) + } +} + +func TestMemoryStoreStorageConnectionCRUD(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 主连接", + Description: "用于扫描", + Provider: domain.StorageProviderBOS, + Endpoint: "https://bj.bcebos.com", + AccessKeyID: "ak-memory-1234", + SecretAccessKey: "sk-memory-5678", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + if connection.ID == "" { + t.Fatalf("expected storage connection id") + } + + stored, err := dataStore.GetStorageConnection(ctx, connection.ID) + if err != nil { + t.Fatalf("get storage connection: %v", err) + } + if stored == nil { + t.Fatalf("expected stored connection") + } + if stored.AccessKeyID != "ak-memory-1234" || stored.SecretAccessKey != "sk-memory-5678" { + t.Fatalf("unexpected stored secrets: %#v", stored) + } + + items, err := dataStore.ListStorageConnections(ctx) + if err != nil { + t.Fatalf("list storage connections: %v", err) + } + if len(items) != 1 || items[0].ID != connection.ID { + t.Fatalf("unexpected storage connection list: %#v", items) + } + + stored.Description = "已更新" + if err := dataStore.UpsertStorageConnection(ctx, *stored); err != nil { + t.Fatalf("upsert storage connection: %v", err) + } + + updated, err := dataStore.GetStorageConnection(ctx, connection.ID) + if err != nil { + t.Fatalf("get updated storage connection: %v", err) + } + if updated == nil || updated.Description != "已更新" { + t.Fatalf("expected updated description, got %#v", updated) + } +} + func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -275,3 +415,50 @@ func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { t.Fatalf("expected scheduled run queued, got %#v", scheduledRun) } } + +func TestCreateRunTracksPreviousAndCurrentRunTimestamps(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "时间戳项目", + Description: "run timestamp tracking", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "时间戳工作流", + Description: "timestamps", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{}, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + firstRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create first run: %v", err) + } + if firstRun.PreviousRunAt != "" { + t.Fatalf("expected first previousRunAt empty, got %q", firstRun.PreviousRunAt) + } + if firstRun.CurrentRunAt == "" { + t.Fatalf("expected first currentRunAt populated") + } + + secondRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled) + if err != nil { + t.Fatalf("create second run: %v", err) + } + if secondRun.PreviousRunAt != firstRun.CurrentRunAt { + t.Fatalf("expected second previousRunAt=%q, got %q", firstRun.CurrentRunAt, secondRun.PreviousRunAt) + } + if secondRun.CurrentRunAt == "" { + t.Fatalf("expected second currentRunAt populated") + } +} diff --git a/internal/store/mongo.go b/internal/store/mongo.go index a00a8f5f..9d8cecfa 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -2,6 +2,7 @@ package store import ( "context" + "encoding/json" "fmt" "sort" "strings" @@ -9,33 +10,44 @@ import ( "droboticflow/internal/domain" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) type MongoStore struct { - db *mongo.Database - projects *mongo.Collection - operators *mongo.Collection - workflows *mongo.Collection - templates *mongo.Collection - runs *mongo.Collection - auditTasks *mongo.Collection - users *mongo.Collection - settings *mongo.Collection + db *mongo.Database + projects *mongo.Collection + operators *mongo.Collection + builtinOperatorDeletions *mongo.Collection + workflows *mongo.Collection + templates *mongo.Collection + runs *mongo.Collection + sourceStates *mongo.Collection + auditTasks *mongo.Collection + samplingTasks *mongo.Collection + users *mongo.Collection + storageConnections *mongo.Collection + mongoConnections *mongo.Collection + settings *mongo.Collection } func NewMongoStore(db *mongo.Database) *MongoStore { return &MongoStore{ - db: db, - projects: db.Collection("projects"), - operators: db.Collection("operators"), - workflows: db.Collection("workflows"), - templates: db.Collection("templates"), - runs: db.Collection("runs"), - auditTasks: db.Collection("auditTasks"), - users: db.Collection("users"), - settings: db.Collection("settings"), + db: db, + projects: db.Collection("projects"), + operators: db.Collection("operators"), + builtinOperatorDeletions: db.Collection("operatorTombstones"), + workflows: db.Collection("workflows"), + templates: db.Collection("templates"), + runs: db.Collection("runs"), + sourceStates: db.Collection("sourceStates"), + auditTasks: db.Collection("auditTasks"), + samplingTasks: db.Collection("samplingTasks"), + users: db.Collection("users"), + storageConnections: db.Collection("storageConnections"), + mongoConnections: db.Collection("mongoConnections"), + settings: db.Collection("settings"), } } @@ -113,10 +125,19 @@ func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, er if _, err := store.auditTasks.DeleteMany(ctx, bson.M{"projectId": id}); err != nil { return false, err } + if _, err := store.samplingTasks.DeleteMany(ctx, bson.M{"projectId": id}); err != nil { + return false, err + } if len(workflowIDs) > 0 { + if _, err := store.sourceStates.DeleteMany(ctx, bson.M{"workflowId": bson.M{"$in": workflowIDs}}); err != nil { + return false, err + } if _, err := store.runs.DeleteMany(ctx, bson.M{"workflowId": bson.M{"$in": workflowIDs}}); err != nil { return false, err } + if _, err := store.samplingTasks.DeleteMany(ctx, bson.M{"workflowId": bson.M{"$in": workflowIDs}}); err != nil { + return false, err + } } result, err := store.projects.DeleteOne(ctx, bson.M{"id": id}) if err != nil { @@ -187,6 +208,19 @@ func (store *MongoStore) UpsertOperator(ctx context.Context, item domain.Operato return err } +func (store *MongoStore) ListBuiltinOperatorDeletions(ctx context.Context) ([]domain.BuiltinOperatorDeletionRecord, error) { + cursor, err := store.builtinOperatorDeletions.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "deletedAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.BuiltinOperatorDeletionRecord](ctx, cursor) +} + +func (store *MongoStore) UpsertBuiltinOperatorDeletion(ctx context.Context, item domain.BuiltinOperatorDeletionRecord) error { + _, err := store.builtinOperatorDeletions.UpdateOne(ctx, bson.M{"key": item.Key}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + func (store *MongoStore) ListWorkflowsByProject(ctx context.Context, projectID string) ([]domain.WorkflowRecord, error) { cursor, err := store.workflows.Find(ctx, bson.M{"projectId": projectID}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) if err != nil { @@ -223,6 +257,9 @@ func (store *MongoStore) CreateWorkflow(ctx context.Context, input domain.Create Name: input.Name, Description: input.Description, Type: input.Type, + Kind: input.Kind, + ScanConfig: input.ScanConfig, + Hyperparams: input.Hyperparams, MaxConcurrentRuns: input.MaxConcurrentRuns, NodeTimeoutSeconds: input.NodeTimeoutSeconds, RunTimeoutSeconds: input.RunTimeoutSeconds, @@ -242,12 +279,18 @@ func (store *MongoStore) CreateWorkflow(ctx context.Context, input domain.Create } func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, error) { + if _, err := store.sourceStates.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil { + return false, err + } if _, err := store.runs.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil { return false, err } if _, err := store.auditTasks.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil { return false, err } + if _, err := store.samplingTasks.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil { + return false, err + } result, err := store.workflows.DeleteOne(ctx, bson.M{"id": id}) if err != nil { return false, err @@ -291,6 +334,9 @@ func (store *MongoStore) CreateTemplate(ctx context.Context, input domain.Create Name: input.Name, Description: input.Description, Type: templateType, + Kind: input.Kind, + ScanConfig: input.ScanConfig, + Hyperparams: input.Hyperparams, MaxConcurrentRuns: input.MaxConcurrentRuns, NodeTimeoutSeconds: input.NodeTimeoutSeconds, RunTimeoutSeconds: input.RunTimeoutSeconds, @@ -330,7 +376,54 @@ func (store *MongoStore) ListRuns(ctx context.Context, status string) ([]domain. if err != nil { return nil, err } - return decodeMany[domain.RunRecord](ctx, cursor) + items, err := decodeMany[domain.RunRecord](ctx, cursor) + if err != nil { + return nil, err + } + for index := range items { + hydrateRunDerivedFields(&items[index]) + } + return items, nil +} + +func (store *MongoStore) ListRunSummaries(ctx context.Context, status string) ([]domain.RunSummary, error) { + filter := bson.M{} + if status != "" { + filter["status"] = status + } + projection := bson.M{ + "id": 1, + "workflowId": 1, + "workflowName": 1, + "projectId": 1, + "projectName": 1, + "trigger": 1, + "attempt": 1, + "parentRunId": 1, + "restartFromNodeId": 1, + "availableAt": 1, + "status": 1, + "stopMode": 1, + "stopRequestedAt": 1, + "startedAt": 1, + "endedAt": 1, + "durationMs": 1, + "currentNodeId": 1, + "error": 1, + "createdAt": 1, + "updatedAt": 1, + } + cursor, err := store.runs.Find( + ctx, + filter, + options.Find(). + SetProjection(projection). + SetSort(bson.D{{Key: "createdAt", Value: -1}}), + ) + if err != nil { + return nil, err + } + return decodeMany[domain.RunSummary](ctx, cursor) } func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunRecord, error) { @@ -342,6 +435,7 @@ func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunReco if err != nil { return nil, err } + hydrateRunDerivedFields(&item) return &item, nil } @@ -353,27 +447,34 @@ func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigg if workflow == nil { return nil, fmt.Errorf("workflow %s not found", workflowID) } - if workflow.Status == domain.WorkflowStatusCompleted && trigger != domain.RunTriggerScheduled { + if workflow.Status == domain.WorkflowStatusCompleted && workflow.Kind != domain.WorkflowKindScan && trigger != domain.RunTriggerScheduled { return nil, ErrWorkflowAlreadyCompleted } - project, err := store.GetProject(ctx, workflow.ProjectID) - if err != nil { - return nil, err - } - if project == nil { - return nil, fmt.Errorf("project %s not found", workflow.ProjectID) + projectName := "" + if workflow.ProjectID != "" { + project, err := store.GetProject(ctx, workflow.ProjectID) + if err != nil { + return nil, err + } + if project == nil { + return nil, fmt.Errorf("project %s not found", workflow.ProjectID) + } + projectName = project.Name } now := nowISO() + previousRunAt := workflow.LastRunAt run := domain.RunRecord{ ID: domain.NewID("run"), WorkflowID: workflow.ID, WorkflowName: workflow.Name, - ProjectID: project.ID, - ProjectName: project.Name, + ProjectID: workflow.ProjectID, + ProjectName: projectName, Trigger: trigger, Attempt: 1, AvailableAt: now, Status: domain.RunStatusQueued, + PreviousRunAt: previousRunAt, + CurrentRunAt: now, NodeExecutions: []domain.NodeExecutionRecord{}, Logs: []domain.LogEntryRecord{}, ResourceSeries: []domain.ResourceSample{}, @@ -422,6 +523,7 @@ func (store *MongoStore) ClaimNextQueuedRun(ctx context.Context) (*domain.RunRec if err != nil { return nil, err } + hydrateRunDerivedFields(&item) return &item, nil } @@ -453,12 +555,121 @@ func (store *MongoStore) ClaimQueuedRun(ctx context.Context, runID string) (*dom if err != nil { return nil, err } + hydrateRunDerivedFields(&item) return &item, nil } +func (store *MongoStore) RequestRunStop(ctx context.Context, runID string, mode domain.RunStopMode) (*domain.RunRecord, error) { + run, err := store.GetRun(ctx, runID) + if err != nil || run == nil { + return run, err + } + if run.Status == domain.RunStatusCompleted || run.Status == domain.RunStatusFailed || run.Status == domain.RunStatusCancelled { + return run, nil + } + + now := nowISO() + set := bson.M{ + "stopMode": string(mode), + "stopRequestedAt": now, + "updatedAt": now, + } + switch mode { + case domain.RunStopModeGraceful: + if run.Status == domain.RunStatusQueued || run.Status == domain.RunStatusWaitingForAudit { + set["status"] = string(domain.RunStatusCancelled) + set["endedAt"] = now + run.EndedAt = now + finalizeNodeExecutionsForStop(run, mode, now) + set["nodeExecutions"] = run.NodeExecutions + if strings.TrimSpace(run.Error) == "" { + set["error"] = "用户已停止工作流" + } + } else { + set["status"] = string(domain.RunStatusStopping) + } + case domain.RunStopModeForce: + set["status"] = string(domain.RunStatusCancelled) + set["endedAt"] = now + run.EndedAt = now + finalizeNodeExecutionsForStop(run, mode, now) + set["nodeExecutions"] = run.NodeExecutions + if strings.TrimSpace(run.Error) == "" { + set["error"] = "用户已强制停止工作流" + } + default: + return run, nil + } + computeRunDuration(run) + if run.DurationMS > 0 { + set["durationMs"] = run.DurationMS + } + + if _, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, bson.M{"$set": set}); err != nil { + return nil, err + } + return store.GetRun(ctx, runID) +} + func (store *MongoStore) UpsertRun(ctx context.Context, item domain.RunRecord) error { computeRunDuration(&item) - _, err := store.runs.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + stored := compactRunForStorage(item) + _, err := store.runs.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": stored}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) GetSourceState(ctx context.Context, workflowID string, nodeID string, sourceFingerprint string) (*domain.SourceStateRecord, error) { + var item domain.SourceStateRecord + err := store.sourceStates.FindOne(ctx, bson.M{ + "workflowId": workflowID, + "nodeId": nodeID, + "sourceFingerprint": sourceFingerprint, + }).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) ListSourceStates(ctx context.Context, workflowID string, nodeID string) ([]domain.SourceStateRecord, error) { + filter := bson.M{"workflowId": workflowID} + if nodeID != "" { + filter["nodeId"] = nodeID + } + cursor, err := store.sourceStates.Find(ctx, filter) + if err != nil { + return nil, err + } + return decodeMany[domain.SourceStateRecord](ctx, cursor) +} + +func (store *MongoStore) UpsertSourceState(ctx context.Context, item domain.SourceStateRecord) error { + if item.ID == "" { + item.ID = item.WorkflowID + ":" + item.NodeID + ":" + item.SourceFingerprint + } + _, err := store.sourceStates.UpdateOne( + ctx, + bson.M{"id": item.ID}, + bson.M{"$set": item}, + options.Update().SetUpsert(true), + ) + return err +} + +func (store *MongoStore) DeleteSourceState(ctx context.Context, workflowID string, nodeID string, sourceFingerprint string) error { + _, err := store.sourceStates.DeleteOne(ctx, bson.M{ + "workflowId": workflowID, + "nodeId": nodeID, + "sourceFingerprint": sourceFingerprint, + }) + return err +} + +func (store *MongoStore) DeleteSourceStatesByWorkflow(ctx context.Context, workflowID string) error { + _, err := store.sourceStates.DeleteMany(ctx, bson.M{"workflowId": workflowID}) return err } @@ -498,6 +709,45 @@ func (store *MongoStore) UpsertAuditTask(ctx context.Context, item domain.AuditT return err } +func (store *MongoStore) ListSamplingTasks(ctx context.Context, status string) ([]domain.SamplingTaskRecord, error) { + filter := bson.M{} + if status != "" { + filter["status"] = status + } + cursor, err := store.samplingTasks.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.SamplingTaskRecord](ctx, cursor) +} + +func (store *MongoStore) GetSamplingTask(ctx context.Context, id string) (*domain.SamplingTaskRecord, error) { + var item domain.SamplingTaskRecord + err := store.samplingTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateSamplingTask(ctx context.Context, item domain.SamplingTaskRecord) (*domain.SamplingTaskRecord, error) { + if strings.TrimSpace(item.ID) == "" { + item.ID = domain.NewID("sampling") + } + if err := store.UpsertSamplingTask(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) UpsertSamplingTask(ctx context.Context, item domain.SamplingTaskRecord) error { + _, err := store.samplingTasks.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + func (store *MongoStore) ListUsers(ctx context.Context) ([]domain.UserRecord, error) { cursor, err := store.users.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}})) if err != nil { @@ -547,6 +797,59 @@ func (store *MongoStore) UpsertUser(ctx context.Context, item domain.UserRecord) return err } +func (store *MongoStore) ListStorageConnections(ctx context.Context) ([]domain.StorageConnectionRecord, error) { + cursor, err := store.storageConnections.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.StorageConnectionRecord](ctx, cursor) +} + +func (store *MongoStore) GetStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) { + var item domain.StorageConnectionRecord + err := store.storageConnections.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateStorageConnection(ctx context.Context, input domain.CreateStorageConnectionInput) (*domain.StorageConnectionRecord, error) { + now := nowISO() + item := domain.StorageConnectionRecord{ + ID: domain.NewID("storage"), + Name: input.Name, + Description: input.Description, + Provider: input.Provider, + Endpoint: input.Endpoint, + Bucket: input.Bucket, + AccessKeyID: input.AccessKeyID, + SecretAccessKey: input.SecretAccessKey, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.UpsertStorageConnection(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) DeleteStorageConnection(ctx context.Context, id string) (bool, error) { + result, err := store.storageConnections.DeleteOne(ctx, bson.M{"id": id}) + if err != nil { + return false, err + } + return result.DeletedCount > 0, nil +} + +func (store *MongoStore) UpsertStorageConnection(ctx context.Context, item domain.StorageConnectionRecord) error { + _, err := store.storageConnections.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + func (store *MongoStore) GetSettings(ctx context.Context) (*domain.SettingsRecord, error) { var item domain.SettingsRecord err := store.settings.FindOne(ctx, bson.M{"id": "system"}).Decode(&item) @@ -570,6 +873,56 @@ func (store *MongoStore) UpsertSettings(ctx context.Context, item domain.Setting return err } +func (store *MongoStore) ListMongoConnections(ctx context.Context) ([]domain.MongoConnectionRecord, error) { + cursor, err := store.mongoConnections.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.MongoConnectionRecord](ctx, cursor) +} + +func (store *MongoStore) GetMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) { + var item domain.MongoConnectionRecord + err := store.mongoConnections.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateMongoConnection(ctx context.Context, input domain.CreateMongoConnectionInput) (*domain.MongoConnectionRecord, error) { + now := nowISO() + item := domain.MongoConnectionRecord{ + ID: domain.NewID("mongo"), + Name: input.Name, + Description: input.Description, + URI: input.URI, + Database: input.Database, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := store.mongoConnections.InsertOne(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) DeleteMongoConnection(ctx context.Context, id string) (bool, error) { + result, err := store.mongoConnections.DeleteOne(ctx, bson.M{"id": id}) + if err != nil { + return false, err + } + return result.DeletedCount > 0, nil +} + +func (store *MongoStore) UpsertMongoConnection(ctx context.Context, item domain.MongoConnectionRecord) error { + _, err := store.mongoConnections.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + func (store *MongoStore) GetDashboardSummary(ctx context.Context) (domain.DashboardSummary, error) { projects, err := store.ListProjects(ctx) if err != nil { @@ -579,7 +932,7 @@ func (store *MongoStore) GetDashboardSummary(ctx context.Context) (domain.Dashbo if err != nil { return domain.DashboardSummary{}, err } - runs, err := store.ListRuns(ctx, "") + runs, err := store.ListRunSummaries(ctx, "") if err != nil { return domain.DashboardSummary{}, err } @@ -604,3 +957,428 @@ func (store *MongoStore) ListStorageObjects(ctx context.Context, sourcePath stri sort.Strings(items) return items, nil } + +func mongoBrowserFilter(value any) any { + switch typed := value.(type) { + case map[string]any: + filter := bson.M{} + for key, item := range typed { + filter[key] = mongoBrowserFilter(item) + } + return filter + case []any: + items := make([]any, 0, len(typed)) + for _, item := range typed { + items = append(items, mongoBrowserFilter(item)) + } + return items + default: + return typed + } +} + +func mongoBrowserDocument(document bson.M) map[string]any { + if len(document) == 0 { + return map[string]any{} + } + payload, err := bson.MarshalExtJSON(document, false, false) + if err != nil { + return sanitizeDatabaseBrowserDocument(map[string]any{}) + } + result := map[string]any{} + if err := json.Unmarshal(payload, &result); err != nil { + return sanitizeDatabaseBrowserDocument(map[string]any{}) + } + return sanitizeDatabaseBrowserDocument(result) +} + +func mongoBrowserSampleDocument(ctx context.Context, collection *mongo.Collection) (map[string]any, error) { + var sample bson.M + err := collection.FindOne(ctx, bson.D{}, options.FindOne().SetSort(bson.D{{Key: "_id", Value: -1}})).Decode(&sample) + if err == mongo.ErrNoDocuments { + return map[string]any{}, nil + } + if err != nil { + return nil, err + } + return mongoBrowserDocument(sample), nil +} + +func mongoBrowserCollectionSummary(ctx context.Context, collection *mongo.Collection, name string) (domain.DatabaseCollectionSummary, error) { + count, err := collection.EstimatedDocumentCount(ctx) + if err != nil { + count, err = collection.CountDocuments(ctx, bson.D{}) + if err != nil { + return domain.DatabaseCollectionSummary{}, err + } + } + sample, err := mongoBrowserSampleDocument(ctx, collection) + if err != nil { + return domain.DatabaseCollectionSummary{}, err + } + return domain.DatabaseCollectionSummary{ + Name: name, + DocumentCount: count, + SampleFields: databaseBrowserSampleFields(sample), + }, nil +} + +func mongoBrowserDocumentIDFilter(documentID string) (bson.M, error) { + trimmed := strings.TrimSpace(documentID) + if trimmed == "" { + return nil, fmt.Errorf("document id is required") + } + if oid, err := primitive.ObjectIDFromHex(trimmed); err == nil { + return bson.M{"_id": oid}, nil + } + return bson.M{"_id": trimmed}, nil +} + +func mongoBrowserWritableDocument(document map[string]any) bson.M { + writable := clone(document) + delete(writable, "_id") + return mongoBrowserFilter(writable).(bson.M) +} + +func mongoBrowserWritablePatch(document map[string]any) bson.M { + writable := clone(document) + delete(writable, "_id") + return mongoBrowserFilter(writable).(bson.M) +} + +func mongoBrowserFindDocumentByFilter(ctx context.Context, collection *mongo.Collection, filter bson.M) (map[string]any, error) { + document := bson.M{} + if err := collection.FindOne(ctx, filter).Decode(&document); err != nil { + if err == mongo.ErrNoDocuments { + return nil, fmt.Errorf("document not found") + } + return nil, err + } + return mongoBrowserDocument(document), nil +} + +func mongoBrowserIndexSummary(document bson.M) domain.DatabaseIndexSummary { + summary := domain.DatabaseIndexSummary{ + Name: strings.TrimSpace(fmt.Sprintf("%v", document["name"])), + Keys: map[string]any{}, + } + if key, ok := document["key"].(bson.M); ok { + summary.Keys = mongoBrowserDocument(key) + } + if unique, ok := document["unique"].(bool); ok { + summary.Unique = unique + } + return summary +} + +func (store *MongoStore) GetDatabaseOverview(ctx context.Context) (domain.DatabaseOverview, error) { + names, err := store.db.ListCollectionNames(ctx, bson.D{}) + if err != nil { + return domain.DatabaseOverview{}, err + } + sort.Strings(names) + + overview := domain.DatabaseOverview{ + Provider: "mongodb", + Database: store.db.Name(), + ReadOnly: true, + Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)), + } + for _, name := range names { + summary, err := mongoBrowserCollectionSummary(ctx, store.db.Collection(name), name) + if err != nil { + return domain.DatabaseOverview{}, err + } + overview.Collections = append(overview.Collections, summary) + } + return overview, nil +} + +func (store *MongoStore) QueryDatabaseCollection(ctx context.Context, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) { + collectionName := strings.TrimSpace(input.Collection) + if collectionName == "" { + return domain.DatabaseQueryResult{}, fmt.Errorf("collection is required") + } + limit := input.Limit + if limit <= 0 { + limit = 50 + } + skip := input.Skip + if skip < 0 { + skip = 0 + } + + collection := store.db.Collection(collectionName) + filter, _ := mongoBrowserFilter(input.Filter).(bson.M) + if filter == nil { + filter = bson.M{} + } + total, err := collection.CountDocuments(ctx, filter) + if err != nil { + return domain.DatabaseQueryResult{}, err + } + cursor, err := collection.Find( + ctx, + filter, + options.Find(). + SetLimit(int64(limit)). + SetSkip(int64(skip)). + SetSort(bson.D{{Key: "_id", Value: -1}}), + ) + if err != nil { + return domain.DatabaseQueryResult{}, err + } + defer cursor.Close(ctx) + + documents := make([]map[string]any, 0) + for cursor.Next(ctx) { + document := bson.M{} + if err := cursor.Decode(&document); err != nil { + return domain.DatabaseQueryResult{}, err + } + documents = append(documents, mongoBrowserDocument(document)) + } + if err := cursor.Err(); err != nil { + return domain.DatabaseQueryResult{}, err + } + + result := domain.DatabaseQueryResult{ + Provider: "mongodb", + Database: store.db.Name(), + Collection: collectionName, + Filter: clone(input.Filter), + Limit: limit, + Skip: skip, + Total: total, + Documents: documents, + } + if len(documents) > 0 { + result.SampleFields = databaseBrowserSampleFields(documents[0]) + } + return result, nil +} + +func (store *MongoStore) CreateDatabaseCollection(ctx context.Context, name string) (domain.DatabaseCollectionSummary, error) { + collectionName := strings.TrimSpace(name) + if collectionName == "" { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required") + } + if err := store.db.CreateCollection(ctx, collectionName); err != nil { + return domain.DatabaseCollectionSummary{}, err + } + return mongoBrowserCollectionSummary(ctx, store.db.Collection(collectionName), collectionName) +} + +func (store *MongoStore) ListDatabaseIndexes(ctx context.Context, collectionName string) ([]domain.DatabaseIndexSummary, error) { + name := strings.TrimSpace(collectionName) + if name == "" { + return nil, fmt.Errorf("collection is required") + } + cursor, err := store.db.Collection(name).Indexes().List(ctx) + if err != nil { + return nil, err + } + defer cursor.Close(ctx) + items := make([]domain.DatabaseIndexSummary, 0) + for cursor.Next(ctx) { + document := bson.M{} + if err := cursor.Decode(&document); err != nil { + return nil, err + } + items = append(items, mongoBrowserIndexSummary(document)) + } + if err := cursor.Err(); err != nil { + return nil, err + } + sort.Slice(items, func(left int, right int) bool { + return items[left].Name < items[right].Name + }) + return items, nil +} + +func (store *MongoStore) CreateDatabaseIndex(ctx context.Context, collectionName string, input domain.DatabaseIndexCreateInput) (domain.DatabaseIndexSummary, error) { + name := strings.TrimSpace(collectionName) + if name == "" { + return domain.DatabaseIndexSummary{}, fmt.Errorf("collection is required") + } + if len(input.Keys) == 0 { + return domain.DatabaseIndexSummary{}, fmt.Errorf("index keys are required") + } + model := mongo.IndexModel{ + Keys: mongoBrowserFilter(input.Keys), + } + indexName := strings.TrimSpace(input.Name) + if indexName != "" || input.Unique { + opts := options.Index() + if indexName != "" { + opts.SetName(indexName) + } + if input.Unique { + opts.SetUnique(true) + } + model.Options = opts + } + createdName, err := store.db.Collection(name).Indexes().CreateOne(ctx, model) + if err != nil { + return domain.DatabaseIndexSummary{}, err + } + indexes, err := store.ListDatabaseIndexes(ctx, name) + if err != nil { + return domain.DatabaseIndexSummary{}, err + } + for _, item := range indexes { + if item.Name == createdName { + return item, nil + } + } + return domain.DatabaseIndexSummary{Name: createdName, Keys: clone(input.Keys), Unique: input.Unique}, nil +} + +func (store *MongoStore) DeleteDatabaseIndex(ctx context.Context, collectionName string, indexName string) error { + name := strings.TrimSpace(collectionName) + target := strings.TrimSpace(indexName) + if name == "" { + return fmt.Errorf("collection is required") + } + if target == "" { + return fmt.Errorf("index name is required") + } + if target == "_id_" { + return fmt.Errorf("index %s cannot be deleted", target) + } + _, err := store.db.Collection(name).Indexes().DropOne(ctx, target) + return err +} + +func (store *MongoStore) RenameDatabaseCollection(ctx context.Context, name string, newName string) (domain.DatabaseCollectionSummary, error) { + collectionName := strings.TrimSpace(name) + nextName := strings.TrimSpace(newName) + if collectionName == "" { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required") + } + if nextName == "" { + return domain.DatabaseCollectionSummary{}, fmt.Errorf("new collection name is required") + } + command := bson.D{ + {Key: "renameCollection", Value: store.db.Name() + "." + collectionName}, + {Key: "to", Value: store.db.Name() + "." + nextName}, + } + if err := store.db.Client().Database("admin").RunCommand(ctx, command).Err(); err != nil { + return domain.DatabaseCollectionSummary{}, err + } + return mongoBrowserCollectionSummary(ctx, store.db.Collection(nextName), nextName) +} + +func (store *MongoStore) DeleteDatabaseCollection(ctx context.Context, name string) error { + collectionName := strings.TrimSpace(name) + if collectionName == "" { + return fmt.Errorf("collection is required") + } + if err := store.db.Collection(collectionName).Drop(ctx); err != nil { + return err + } + return nil +} + +func (store *MongoStore) InsertDatabaseDocument(ctx context.Context, collectionName string, document map[string]any) (map[string]any, error) { + name := strings.TrimSpace(collectionName) + if name == "" { + return nil, fmt.Errorf("collection is required") + } + collection := store.db.Collection(name) + payload := mongoBrowserWritableDocument(document) + result, err := collection.InsertOne(ctx, payload) + if err != nil { + return nil, err + } + return mongoBrowserFindDocumentByFilter(ctx, collection, bson.M{"_id": result.InsertedID}) +} + +func (store *MongoStore) ReplaceDatabaseDocument(ctx context.Context, collectionName string, documentID string, document map[string]any) (map[string]any, error) { + name := strings.TrimSpace(collectionName) + if name == "" { + return nil, fmt.Errorf("collection is required") + } + filter, err := mongoBrowserDocumentIDFilter(documentID) + if err != nil { + return nil, err + } + collection := store.db.Collection(name) + result, err := collection.ReplaceOne(ctx, filter, mongoBrowserWritableDocument(document)) + if err != nil { + return nil, err + } + if result.MatchedCount == 0 { + return nil, fmt.Errorf("document %s not found", documentID) + } + return mongoBrowserFindDocumentByFilter(ctx, collection, filter) +} + +func (store *MongoStore) PatchDatabaseDocument(ctx context.Context, collectionName string, documentID string, patch map[string]any) (map[string]any, error) { + name := strings.TrimSpace(collectionName) + if name == "" { + return nil, fmt.Errorf("collection is required") + } + filter, err := mongoBrowserDocumentIDFilter(documentID) + if err != nil { + return nil, err + } + collection := store.db.Collection(name) + result, err := collection.UpdateOne(ctx, filter, bson.M{"$set": mongoBrowserWritablePatch(patch)}) + if err != nil { + return nil, err + } + if result.MatchedCount == 0 { + return nil, fmt.Errorf("document %s not found", documentID) + } + return mongoBrowserFindDocumentByFilter(ctx, collection, filter) +} + +func (store *MongoStore) UpsertDatabaseDocument(ctx context.Context, collectionName string, match map[string]any, patch map[string]any, setOnInsert map[string]any) (map[string]any, bool, error) { + name := strings.TrimSpace(collectionName) + if name == "" { + return nil, false, fmt.Errorf("collection is required") + } + filter, _ := mongoBrowserFilter(match).(bson.M) + if filter == nil { + filter = bson.M{} + } + update := bson.M{"$set": mongoBrowserWritablePatch(patch)} + if len(setOnInsert) > 0 { + update["$setOnInsert"] = mongoBrowserWritablePatch(setOnInsert) + } + collection := store.db.Collection(name) + result, err := collection.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true)) + if err != nil { + return nil, false, err + } + created := result.UpsertedCount > 0 + if created && result.UpsertedID != nil { + document, err := mongoBrowserFindDocumentByFilter(ctx, collection, bson.M{"_id": result.UpsertedID}) + return document, true, err + } + document, err := mongoBrowserFindDocumentByFilter(ctx, collection, filter) + if err != nil { + return nil, false, err + } + return document, false, nil +} + +func (store *MongoStore) DeleteDatabaseDocument(ctx context.Context, collectionName string, documentID string) error { + name := strings.TrimSpace(collectionName) + if name == "" { + return fmt.Errorf("collection is required") + } + filter, err := mongoBrowserDocumentIDFilter(documentID) + if err != nil { + return err + } + result, err := store.db.Collection(name).DeleteOne(ctx, filter) + if err != nil { + return err + } + if result.DeletedCount == 0 { + return fmt.Errorf("document %s not found", documentID) + } + return nil +} diff --git a/internal/store/store.go b/internal/store/store.go index 55a11497..166800e5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -18,6 +18,8 @@ type Store interface { CreateOperator(context.Context, domain.CreateOperatorInput) (*domain.OperatorRecord, error) DeleteOperator(context.Context, string) (bool, error) UpsertOperator(context.Context, domain.OperatorRecord) error + ListBuiltinOperatorDeletions(context.Context) ([]domain.BuiltinOperatorDeletionRecord, error) + UpsertBuiltinOperatorDeletion(context.Context, domain.BuiltinOperatorDeletionRecord) error ListWorkflowsByProject(context.Context, string) ([]domain.WorkflowRecord, error) ListAllWorkflows(context.Context) ([]domain.WorkflowRecord, error) @@ -33,27 +35,55 @@ type Store interface { UpsertTemplate(context.Context, domain.TemplateRecord) error ListRuns(context.Context, string) ([]domain.RunRecord, error) + ListRunSummaries(context.Context, string) ([]domain.RunSummary, error) GetRun(context.Context, string) (*domain.RunRecord, error) CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error) ClaimNextQueuedRun(context.Context) (*domain.RunRecord, error) ClaimQueuedRun(context.Context, string) (*domain.RunRecord, error) + RequestRunStop(context.Context, string, domain.RunStopMode) (*domain.RunRecord, error) UpsertRun(context.Context, domain.RunRecord) error + GetSourceState(context.Context, string, string, string) (*domain.SourceStateRecord, error) + ListSourceStates(context.Context, string, string) ([]domain.SourceStateRecord, error) + UpsertSourceState(context.Context, domain.SourceStateRecord) error + DeleteSourceState(context.Context, string, string, string) error + DeleteSourceStatesByWorkflow(context.Context, string) error ListAuditTasks(context.Context, string) ([]domain.AuditTaskRecord, error) GetAuditTask(context.Context, string) (*domain.AuditTaskRecord, error) CreateAuditTask(context.Context, domain.AuditTaskRecord) (*domain.AuditTaskRecord, error) UpsertAuditTask(context.Context, domain.AuditTaskRecord) error + ListSamplingTasks(context.Context, string) ([]domain.SamplingTaskRecord, error) + GetSamplingTask(context.Context, string) (*domain.SamplingTaskRecord, error) + CreateSamplingTask(context.Context, domain.SamplingTaskRecord) (*domain.SamplingTaskRecord, error) + UpsertSamplingTask(context.Context, domain.SamplingTaskRecord) error + ListUsers(context.Context) ([]domain.UserRecord, error) GetUser(context.Context, string) (*domain.UserRecord, error) CreateUser(context.Context, domain.CreateUserInput) (*domain.UserRecord, error) DeleteUser(context.Context, string) (bool, error) UpsertUser(context.Context, domain.UserRecord) error + ListStorageConnections(context.Context) ([]domain.StorageConnectionRecord, error) + GetStorageConnection(context.Context, string) (*domain.StorageConnectionRecord, error) + CreateStorageConnection(context.Context, domain.CreateStorageConnectionInput) (*domain.StorageConnectionRecord, error) + DeleteStorageConnection(context.Context, string) (bool, error) + UpsertStorageConnection(context.Context, domain.StorageConnectionRecord) error + + ListMongoConnections(context.Context) ([]domain.MongoConnectionRecord, error) + GetMongoConnection(context.Context, string) (*domain.MongoConnectionRecord, error) + CreateMongoConnection(context.Context, domain.CreateMongoConnectionInput) (*domain.MongoConnectionRecord, error) + DeleteMongoConnection(context.Context, string) (bool, error) + UpsertMongoConnection(context.Context, domain.MongoConnectionRecord) error + GetSettings(context.Context) (*domain.SettingsRecord, error) UpsertSettings(context.Context, domain.SettingsRecord) error GetDashboardSummary(context.Context) (domain.DashboardSummary, error) ListStorageBuckets(context.Context, string) ([]string, error) ListStorageObjects(context.Context, string, string, string) ([]string, error) + GetDatabaseOverview(context.Context) (domain.DatabaseOverview, error) + QueryDatabaseCollection(context.Context, domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) + PatchDatabaseDocument(context.Context, string, string, map[string]any) (map[string]any, error) + UpsertDatabaseDocument(context.Context, string, map[string]any, map[string]any, map[string]any) (map[string]any, bool, error) } diff --git a/internal/worker/executor.go b/internal/worker/executor.go index f44bcbe9..be4638c8 100644 --- a/internal/worker/executor.go +++ b/internal/worker/executor.go @@ -5,26 +5,57 @@ import ( "encoding/json" "fmt" "os" - "os/exec" "path/filepath" "regexp" - "slices" "strings" + "time" "droboticflow/internal/domain" "droboticflow/internal/k8s" ) const resultPrefix = "DROBOTICFLOW_RESULT=" +const resultPathPrefix = "DROBOTICFLOW_RESULT_PATH=" +const inputArtifactsInlineLimit = 64 * 1024 -type executionResult struct { - Logs []domain.LogEntryRecord `json:"logs"` - Artifacts []domain.ExecutionArtifact `json:"artifacts"` - ResourceSeries []domain.ResourceSample `json:"resourceSeries"` +type nodeWorkspace struct { + RunRoot string + Root string + OutputDir string + InputArtifactsPath string + ResultPath string + StageManifestPath string } +type executionResult struct { + Logs []domain.LogEntryRecord `json:"logs"` + Artifacts []domain.ExecutionArtifact `json:"artifacts"` + DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"` + StageManifestPath string `json:"stageManifestPath,omitempty"` + ResourceSeries []domain.ResourceSample `json:"resourceSeries"` +} + +type stageManifestRecord struct { + DataRefs []domain.DataRefRecord `json:"dataRefs"` +} + +var protectedSystemEnv = map[string]struct{}{ + "WORKFLOW_PREVIOUS_RUN_TIMESTAMP": {}, + "WORKFLOW_CURRENT_RUN_TIMESTAMP": {}, +} + +var workflowEnvLocation = loadWorkflowEnvLocation() + var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`) +func loadWorkflowEnvLocation() *time.Location { + location, err := time.LoadLocation("Asia/Shanghai") + if err != nil { + return time.FixedZone("Asia/Shanghai", 8*60*60) + } + return location +} + func sanitizeEnvKey(value string) string { normalized := strings.ToUpper(strings.TrimSpace(value)) if normalized == "" { @@ -49,6 +80,27 @@ func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact return cloned } +func runTimestampEnvValue(value string) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "" + } + parsed, err := time.Parse(time.RFC3339Nano, trimmed) + if err != nil { + return "" + } + return parsed.In(workflowEnvLocation).Format(time.RFC3339) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact { items := make([]domain.ExecutionArtifact, 0) seen := map[string]struct{}{} @@ -158,28 +210,60 @@ func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.Execut return payload } -func prepareOutputDir(runID string, nodeID string) (string, error) { - outputDir := filepath.Join(os.TempDir(), "droboticflow", runID, nodeID) +func prepareNodeWorkspace(workspaceRoot string, runID string, nodeID string) (nodeWorkspace, error) { + root := strings.TrimSpace(workspaceRoot) + if root == "" { + root = filepath.Join(os.TempDir(), "droboticflow") + } + runRoot := filepath.Join(root, "runs", runID) + nodeRoot := filepath.Join(runRoot, "nodes", nodeID) + outputDir := filepath.Join(nodeRoot, "outputs") if err := os.MkdirAll(outputDir, 0o755); err != nil { - return "", err + return nodeWorkspace{}, err } - return outputDir, nil + return nodeWorkspace{ + RunRoot: runRoot, + Root: nodeRoot, + OutputDir: outputDir, + InputArtifactsPath: filepath.Join(nodeRoot, "input-artifacts.json"), + ResultPath: filepath.Join(nodeRoot, "result.json"), + StageManifestPath: filepath.Join(nodeRoot, "stage-manifest.json"), + }, nil } -func envMapToSlice(values map[string]string) []string { - keys := make([]string, 0, len(values)) - for key := range values { - keys = append(keys, key) - } - slices.Sort(keys) - items := make([]string, 0, len(keys)) - for _, key := range keys { - items = append(items, key+"="+values[key]) - } - return items +func shouldInjectPlatformMongoEnv(node domain.WorkflowNode) bool { + label := strings.ToLower(strings.TrimSpace(node.Data.Label)) + return strings.Contains(label, "mongodb") || strings.Contains(label, "mongo") } -func buildNodeEnv(run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, outputDir string) (map[string]string, error) { +func writeJSONFile(path string, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} + +func applyHyperparamsToEnv(env map[string]string, items []domain.HyperParam) { + for _, item := range items { + key := sanitizeEnvKey(item.Key) + if key == "" { + continue + } + if _, blocked := protectedSystemEnv[key]; blocked { + continue + } + env[key] = item.Value + } +} + +func buildNodeEnv(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, settings *domain.SettingsRecord) (map[string]string, error) { + if inputArtifacts == nil { + inputArtifacts = []domain.ExecutionArtifact{} + } nodeDataJSON, err := json.Marshal(node.Data) if err != nil { return nil, err @@ -188,17 +272,30 @@ func buildNodeEnv(run *domain.RunRecord, node domain.WorkflowNode, inputArtifact if err != nil { return nil, err } + if err := writeJSONFile(workspace.InputArtifactsPath, inputArtifacts); err != nil { + return nil, err + } env := map[string]string{ - "RUN_ID": run.ID, - "WORKFLOW_ID": run.WorkflowID, - "NODE_ID": node.ID, - "NODE_LABEL": node.Data.Label, - "NODE_TYPE": node.Type, - "NODE_CATEGORY": node.Data.Category, - "NODE_DESCRIPTION": node.Data.Description, - "NODE_DATA_JSON": string(nodeDataJSON), - "INPUT_ARTIFACTS_JSON": string(inputArtifactsJSON), - "OUTPUT_DIR": outputDir, + "RUN_ID": run.ID, + "WORKFLOW_ID": run.WorkflowID, + "NODE_ID": node.ID, + "NODE_LABEL": node.Data.Label, + "NODE_TYPE": node.Type, + "NODE_CATEGORY": node.Data.Category, + "NODE_DESCRIPTION": node.Data.Description, + "NODE_DATA_JSON": string(nodeDataJSON), + "INPUT_ARTIFACTS_PATH": workspace.InputArtifactsPath, + "RESULT_OUTPUT_PATH": workspace.ResultPath, + "STAGE_MANIFEST_PATH": workspace.StageManifestPath, + "OUTPUT_DIR": workspace.OutputDir, + "WORKSPACE_ROOT": workspace.RunRoot, + } + if run.Trigger == domain.RunTriggerScheduled { + env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] = runTimestampEnvValue(firstNonEmpty(run.CurrentRunAt, run.CreatedAt)) + env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] = runTimestampEnvValue(run.PreviousRunAt) + } + if len(inputArtifactsJSON) <= inputArtifactsInlineLimit { + env["INPUT_ARTIFACTS_JSON"] = string(inputArtifactsJSON) } if node.Data.Bucket != "" { env["STORAGE_BUCKET"] = node.Data.Bucket @@ -218,13 +315,18 @@ func buildNodeEnv(run *domain.RunRecord, node domain.WorkflowNode, inputArtifact if node.Data.Image != "" { env["NODE_IMAGE"] = node.Data.Image } - for _, item := range node.Data.Hyperparams { - key := sanitizeEnvKey(item.Key) - if key == "" { - continue + if shouldInjectPlatformMongoEnv(node) { + if value := strings.TrimSpace(os.Getenv("MONGO_URI")); value != "" { + env["PLATFORM_MONGO_URI"] = value + } + if value := strings.TrimSpace(os.Getenv("MONGO_DB")); value != "" { + env["PLATFORM_MONGO_DB"] = value } - env[key] = item.Value } + if workflow != nil { + applyHyperparamsToEnv(env, workflow.Hyperparams) + } + applyHyperparamsToEnv(env, node.Data.Hyperparams) return env, nil } @@ -236,6 +338,7 @@ func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, err } lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n") structuredPayload := "" + structuredPayloadPath := "" rawLogs := make([]domain.LogEntryRecord, 0) for _, line := range lines { trimmed := strings.TrimSpace(line) @@ -246,11 +349,23 @@ func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, err structuredPayload = strings.TrimPrefix(trimmed, resultPrefix) continue } + if strings.HasPrefix(trimmed, resultPathPrefix) { + structuredPayloadPath = strings.TrimSpace(strings.TrimPrefix(trimmed, resultPathPrefix)) + continue + } rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel)) } + if structuredPayloadPath != "" { + payloadBytes, err := os.ReadFile(structuredPayloadPath) + if err != nil { + result.Logs = rawLogs + return result, fmt.Errorf("read operator result path: %w", err) + } + structuredPayload = strings.TrimSpace(string(payloadBytes)) + } if structuredPayload == "" { result.Logs = rawLogs - return result, fmt.Errorf("operator execution did not emit %s payload", resultPrefix) + return result, fmt.Errorf("operator execution did not emit %s or %s payload", resultPrefix, resultPathPrefix) } var parsed executionResult if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil { @@ -266,64 +381,62 @@ func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, err return result, nil } -func (runtime *Runtime) executeLocalProcess(ctx context.Context, command string, env map[string]string) (string, error) { - cmd := exec.CommandContext(ctx, "sh", "-lc", command) - cmd.Env = append(os.Environ(), envMapToSlice(env)...) - output, err := cmd.CombinedOutput() - return string(output), err -} - -func (runtime *Runtime) executeLocalDocker(ctx context.Context, image string, env map[string]string, outputDir string) (string, error) { - args := []string{"run", "--rm", "-v", outputDir + ":" + outputDir} - for key, value := range env { - args = append(args, "-e", key+"="+value) +func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) { + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return nil, nil } - args = append(args, image) - cmd := exec.CommandContext(ctx, "docker", args...) - output, err := cmd.CombinedOutput() - return string(output), err + payload, err := os.ReadFile(trimmed) + if err != nil { + return nil, err + } + var manifest stageManifestRecord + if err := json.Unmarshal(payload, &manifest); err != nil { + return nil, err + } + if manifest.DataRefs == nil { + return []domain.DataRefRecord{}, nil + } + return manifest.DataRefs, nil } func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) { - outputDir, err := prepareOutputDir(run.ID, node.ID) + workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID) if err != nil { return executionResult{}, nil, err } - env, err := buildNodeEnv(run, node, inputArtifacts, outputDir) + workflow, workflowErr := runtime.store.GetWorkflow(ctx, run.WorkflowID) + if workflowErr != nil { + workflow = nil + } + settings, err := runtime.store.GetSettings(ctx) if err != nil { return executionResult{}, nil, err } - if runtime.launcher.Enabled() { - nodeTimeoutSeconds := 0 - if workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID); err == nil && workflow != nil { - nodeTimeoutSeconds = workflow.NodeTimeoutSeconds - } - launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{ - Image: chooseImage(node, runtime.operatorRunnerImage), - RunID: run.ID, - WorkflowID: run.WorkflowID, - NodeID: node.ID, - NodeLabel: node.Data.Label, - TimeoutSeconds: nodeTimeoutSeconds, - Env: env, - }) - if err != nil { - return executionResult{}, nil, err - } - parsed, err := parseExecutionOutput(launched.Output, node.Data.Label) - return parsed, launched.Reference, err - } - - image := chooseImage(node, runtime.operatorRunnerImage) - var stdout string - if node.Data.Image != "" && node.Data.Image != runtime.operatorRunnerImage { - stdout, err = runtime.executeLocalDocker(ctx, image, env, outputDir) - } else { - stdout, err = runtime.executeLocalProcess(ctx, runtime.localRunnerCommand, env) - } + env, err := buildNodeEnv(run, workflow, node, inputArtifacts, workspace, settings) if err != nil { - return executionResult{}, nil, fmt.Errorf("%w: %s", err, stdout) + return executionResult{}, nil, err } - parsed, err := parseExecutionOutput(stdout, node.Data.Label) - return parsed, nil, err + nodeTimeoutSeconds := 0 + if workflow != nil { + nodeTimeoutSeconds = workflow.NodeTimeoutSeconds + } + launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{ + Image: chooseImage(node, runtime.operatorRunnerImage), + RunID: run.ID, + WorkflowID: run.WorkflowID, + NodeID: node.ID, + NodeLabel: node.Data.Label, + TimeoutSeconds: nodeTimeoutSeconds, + Env: env, + }) + if err != nil { + return executionResult{}, nil, err + } + parsed, err := parseExecutionOutput(launched.Output, node.Data.Label) + if err == nil { + parsed.StageManifestPath = workspace.StageManifestPath + parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath) + } + return parsed, launched.Reference, err } diff --git a/internal/worker/executor_test.go b/internal/worker/executor_test.go new file mode 100644 index 00000000..5bb40530 --- /dev/null +++ b/internal/worker/executor_test.go @@ -0,0 +1,256 @@ +package worker + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "droboticflow/internal/domain" +) + +func TestBuildNodeEnvInjectsPlatformMongoEnvForMongoNode(t *testing.T) { + t.Setenv("MONGO_URI", "mongodb://platform-mongo:27017") + t.Setenv("MONGO_DB", "droboticflow") + + run := &domain.RunRecord{ + ID: "run-1", + WorkflowID: "workflow-1", + Trigger: domain.RunTriggerScheduled, + PreviousRunAt: "2026-04-21T10:00:00Z", + CurrentRunAt: "2026-04-21T11:30:00Z", + } + workflow := &domain.WorkflowRecord{ + ID: "workflow-1", + Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "from-workflow"}}, + } + node := domain.WorkflowNode{ + ID: "node-1", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "MongoDB 写入", + Category: "DATA", + Description: "write docs", + Hyperparams: []domain.HyperParam{{Key: "MONGO_COLLECTION", Value: "qualityReports"}}, + }, + } + + workspace := nodeWorkspace{ + RunRoot: "/tmp/workspace/runs/run-1", + Root: "/tmp/workspace/runs/run-1/nodes/node-1", + OutputDir: "/tmp/workspace/runs/run-1/nodes/node-1/outputs", + InputArtifactsPath: "/tmp/workspace/runs/run-1/nodes/node-1/input-artifacts.json", + ResultPath: "/tmp/workspace/runs/run-1/nodes/node-1/result.json", + StageManifestPath: "/tmp/workspace/runs/run-1/nodes/node-1/stage-manifest.json", + } + + env, err := buildNodeEnv(run, workflow, node, nil, workspace, &domain.SettingsRecord{}) + if err != nil { + t.Fatalf("buildNodeEnv returned error: %v", err) + } + if env["PLATFORM_MONGO_URI"] != "mongodb://platform-mongo:27017" { + t.Fatalf("expected PLATFORM_MONGO_URI to be injected, got %q", env["PLATFORM_MONGO_URI"]) + } + if env["PLATFORM_MONGO_DB"] != "droboticflow" { + t.Fatalf("expected PLATFORM_MONGO_DB to be injected, got %q", env["PLATFORM_MONGO_DB"]) + } + if _, ok := env["BUSINESS_MONGO_URI"]; ok { + t.Fatalf("did not expect BUSINESS_MONGO_URI to be injected") + } + if _, ok := env["BUSINESS_MONGO_DB"]; ok { + t.Fatalf("did not expect BUSINESS_MONGO_DB to be injected") + } + if env["MONGO_COLLECTION"] != "qualityReports" { + t.Fatalf("expected node hyperparam preserved, got %q", env["MONGO_COLLECTION"]) + } + if env["GLOBAL_ENV"] != "from-workflow" { + t.Fatalf("expected workflow hyperparam injected, got %q", env["GLOBAL_ENV"]) + } + if env["INPUT_ARTIFACTS_PATH"] != workspace.InputArtifactsPath { + t.Fatalf("expected INPUT_ARTIFACTS_PATH to be populated, got %q", env["INPUT_ARTIFACTS_PATH"]) + } + if env["RESULT_OUTPUT_PATH"] != workspace.ResultPath { + t.Fatalf("expected RESULT_OUTPUT_PATH to be populated, got %q", env["RESULT_OUTPUT_PATH"]) + } + if env["STAGE_MANIFEST_PATH"] != workspace.StageManifestPath { + t.Fatalf("expected STAGE_MANIFEST_PATH to be populated, got %q", env["STAGE_MANIFEST_PATH"]) + } + if env["WORKSPACE_ROOT"] != workspace.RunRoot { + t.Fatalf("expected WORKSPACE_ROOT to point at run root, got %q", env["WORKSPACE_ROOT"]) + } + expectedPrevious := "2026-04-21T18:00:00+08:00" + if env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] != expectedPrevious { + t.Fatalf("expected previous run timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"]) + } + expectedCurrent := "2026-04-21T19:30:00+08:00" + if env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] != expectedCurrent { + t.Fatalf("expected current run timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"]) + } +} + +func TestBuildNodeEnvWritesInputArtifactsManifest(t *testing.T) { + run := &domain.RunRecord{ + ID: "run-1", + WorkflowID: "workflow-1", + Trigger: domain.RunTriggerManual, + CurrentRunAt: "2026-04-21T11:30:00Z", + } + workflow := &domain.WorkflowRecord{ + ID: "workflow-1", + Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "workflow-default"}}, + } + node := domain.WorkflowNode{ + ID: "node-1", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "数据过滤", + Category: "DATA", + Description: "filter docs", + Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "node-override"}}, + }, + } + root := t.TempDir() + workspace := nodeWorkspace{ + RunRoot: filepath.Join(root, "runs", "run-1"), + Root: filepath.Join(root, "runs", "run-1", "nodes", "node-1"), + OutputDir: filepath.Join(root, "runs", "run-1", "nodes", "node-1", "outputs"), + InputArtifactsPath: filepath.Join(root, "runs", "run-1", "nodes", "node-1", "input-artifacts.json"), + ResultPath: filepath.Join(root, "runs", "run-1", "nodes", "node-1", "result.json"), + StageManifestPath: filepath.Join(root, "runs", "run-1", "nodes", "node-1", "stage-manifest.json"), + } + inputs := []domain.ExecutionArtifact{{ID: "artifact-1", Name: "manifest.json", Type: domain.PortTypeJSON, Path: "bos://bucket/demo.json"}} + + env, err := buildNodeEnv(run, workflow, node, inputs, workspace, &domain.SettingsRecord{}) + if err != nil { + t.Fatalf("buildNodeEnv returned error: %v", err) + } + + payload, err := os.ReadFile(workspace.InputArtifactsPath) + if err != nil { + t.Fatalf("read input manifest: %v", err) + } + var decoded []domain.ExecutionArtifact + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("decode input manifest: %v", err) + } + if len(decoded) != 1 || decoded[0].ID != "artifact-1" { + t.Fatalf("unexpected input manifest payload: %#v", decoded) + } + if env["INPUT_ARTIFACTS_JSON"] == "" { + t.Fatalf("expected small input manifest to remain inline for convenience") + } + if _, ok := env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"]; ok { + t.Fatalf("expected manual run to omit previous run timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"]) + } + if _, ok := env["WORKFLOW_CURRENT_RUN_TIMESTAMP"]; ok { + t.Fatalf("expected manual run to omit current run timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"]) + } + if env["GLOBAL_ENV"] != "node-override" { + t.Fatalf("expected node hyperparams to override workflow-level env, got %q", env["GLOBAL_ENV"]) + } +} + +func TestBuildNodeEnvDoesNotAllowHyperparamsToOverrideRunTimestampEnv(t *testing.T) { + run := &domain.RunRecord{ + ID: "run-2", + WorkflowID: "workflow-2", + Trigger: domain.RunTriggerScheduled, + PreviousRunAt: "2026-04-21T10:00:00Z", + CurrentRunAt: "2026-04-21T11:30:00Z", + } + workflow := &domain.WorkflowRecord{ + ID: "workflow-2", + Hyperparams: []domain.HyperParam{ + {Key: "WORKFLOW_PREVIOUS_RUN_TIMESTAMP", Value: "workflow-999"}, + {Key: "WORKFLOW_CURRENT_RUN_TIMESTAMP", Value: "workflow-888"}, + }, + } + node := domain.WorkflowNode{ + ID: "node-2", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "FLOW", + Description: "exec", + Hyperparams: []domain.HyperParam{ + {Key: "WORKFLOW_PREVIOUS_RUN_TIMESTAMP", Value: "999"}, + {Key: "WORKFLOW_CURRENT_RUN_TIMESTAMP", Value: "888"}, + }, + }, + } + workspace := nodeWorkspace{ + RunRoot: "/tmp/workspace/runs/run-2", + Root: "/tmp/workspace/runs/run-2/nodes/node-2", + OutputDir: "/tmp/workspace/runs/run-2/nodes/node-2/outputs", + InputArtifactsPath: "/tmp/workspace/runs/run-2/nodes/node-2/input-artifacts.json", + ResultPath: "/tmp/workspace/runs/run-2/nodes/node-2/result.json", + StageManifestPath: "/tmp/workspace/runs/run-2/nodes/node-2/stage-manifest.json", + } + + env, err := buildNodeEnv(run, workflow, node, nil, workspace, &domain.SettingsRecord{}) + if err != nil { + t.Fatalf("buildNodeEnv returned error: %v", err) + } + if env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] != "2026-04-21T18:00:00+08:00" { + t.Fatalf("expected protected previous timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"]) + } + if env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] != "2026-04-21T19:30:00+08:00" { + t.Fatalf("expected protected current timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"]) + } +} + +func TestParseExecutionOutputReadsResultFilePayload(t *testing.T) { + resultPath := filepath.Join(t.TempDir(), "result.json") + payload := executionResult{ + Logs: []domain.LogEntryRecord{{Level: "INFO", Message: "done", Operator: "测试节点"}}, + Artifacts: []domain.ExecutionArtifact{{ID: "artifact-1", Name: "report.json"}}, + } + data, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + if err := os.WriteFile(resultPath, data, 0o644); err != nil { + t.Fatalf("write payload: %v", err) + } + + parsed, err := parseExecutionOutput("some log\n"+resultPathPrefix+resultPath+"\n", "测试节点") + if err != nil { + t.Fatalf("parseExecutionOutput returned error: %v", err) + } + if len(parsed.Artifacts) != 1 || parsed.Artifacts[0].ID != "artifact-1" { + t.Fatalf("unexpected parsed artifacts: %#v", parsed.Artifacts) + } +} + +func TestLoadStageManifestDataRefs(t *testing.T) { + stageManifestPath := filepath.Join(t.TempDir(), "stage-manifest.json") + payload := map[string]any{ + "dataRefs": []map[string]any{ + { + "artifactId": "artifact-1", + "artifactName": "dataset-index.json", + "kind": "dataset", + "datasetType": "x-platform-index", + "manifestPath": "/workspace/run/stage/manifest.json", + "payloadPath": "/workspace/run/stage/payload.json", + "partitionDir": "/workspace/run/stage/partitions", + "partitionCount": 3, + "itemCount": 12, + }, + }, + } + data, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + if err := os.WriteFile(stageManifestPath, data, 0o644); err != nil { + t.Fatalf("write stage manifest: %v", err) + } + dataRefs, err := loadStageManifestDataRefs(stageManifestPath) + if err != nil { + t.Fatalf("loadStageManifestDataRefs returned error: %v", err) + } + if len(dataRefs) != 1 || dataRefs[0].DatasetType != "x-platform-index" { + t.Fatalf("unexpected data refs: %#v", dataRefs) + } +} diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 2b0a3d66..472f070d 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -2,9 +2,14 @@ package worker import ( "context" + "encoding/json" "errors" "fmt" + "os" + "reflect" "sort" + "strconv" + "strings" "time" "droboticflow/internal/domain" @@ -12,39 +17,44 @@ import ( "droboticflow/internal/store" ) +type jobLauncher interface { + Launch(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error) + Inspect(context.Context, string, string) (*k8s.JobInspection, error) +} + type Runtime struct { store store.Store - mode string - namespace string - imagePullSecretName string operatorRunnerImage string - localRunnerCommand string - launcher *k8s.Launcher + workspaceRoot string + launcher jobLauncher } var activeDispatchStatuses = map[domain.RunStatus]struct{}{ domain.RunStatusRunning: {}, + domain.RunStatusStopping: {}, domain.RunStatusWaitingForAudit: {}, } var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{ domain.RunStatusQueued: {}, domain.RunStatusRunning: {}, + domain.RunStatusStopping: {}, domain.RunStatusWaitingForAudit: {}, } -func NewRuntime(dataStore store.Store, mode string, namespace string, imagePullSecretName string, operatorRunnerImage string, localRunnerCommand string) (*Runtime, error) { - launcher, err := k8s.NewLauncher(mode == "k8s", namespace, imagePullSecretName) +func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string) (*Runtime, error) { + launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath) if err != nil { return nil, err } + trimmedWorkspaceRoot := strings.TrimSpace(workspaceMountPath) + if trimmedWorkspaceRoot == "" { + trimmedWorkspaceRoot = "/workspace" + } return &Runtime{ store: dataStore, - mode: mode, - namespace: namespace, - imagePullSecretName: imagePullSecretName, operatorRunnerImage: operatorRunnerImage, - localRunnerCommand: localRunnerCommand, + workspaceRoot: trimmedWorkspaceRoot, launcher: launcher, }, nil } @@ -112,6 +122,118 @@ func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings return globalConcurrencyLimit(settings) } +func parseRFC3339Nano(value string) (time.Time, bool) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return time.Time{}, false + } + parsed, err := time.Parse(time.RFC3339Nano, trimmed) + if err != nil { + return time.Time{}, false + } + return parsed, true +} + +func firstNonEmptyString(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func nextHourlyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) { + intervalHours := schedule.IntervalHours + if intervalHours <= 0 { + return time.Time{}, false + } + anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt) + if !ok { + if !lastRunAt.IsZero() { + anchorAt = lastRunAt + } else { + anchorAt = now + } + } + interval := time.Duration(intervalHours) * time.Hour + if lastRunAt.IsZero() || lastRunAt.Before(anchorAt) { + return anchorAt, true + } + elapsed := lastRunAt.Sub(anchorAt) + slotsElapsed := int(elapsed/interval) + 1 + return anchorAt.Add(time.Duration(slotsElapsed) * interval), true +} + +func nextDailyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) { + if schedule.DayInterval <= 0 { + return time.Time{}, false + } + parts := strings.Split(strings.TrimSpace(schedule.TimeOfDay), ":") + if len(parts) != 2 { + return time.Time{}, false + } + hour, err := strconv.Atoi(parts[0]) + if err != nil { + return time.Time{}, false + } + minute, err := strconv.Atoi(parts[1]) + if err != nil { + return time.Time{}, false + } + location, err := time.LoadLocation(domain.WorkflowScheduleTimezone(schedule.Timezone)) + if err != nil { + return time.Time{}, false + } + anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt) + if !ok { + if !lastRunAt.IsZero() { + anchorAt = lastRunAt + } else { + anchorAt = now + } + } + anchorLocal := anchorAt.In(location) + firstDue := time.Date(anchorLocal.Year(), anchorLocal.Month(), anchorLocal.Day(), hour, minute, 0, 0, location) + if anchorLocal.After(firstDue) { + firstDue = firstDue.AddDate(0, 0, schedule.DayInterval) + } + if lastRunAt.IsZero() { + return firstDue, true + } + lastLocal := lastRunAt.In(location) + nextDue := firstDue + for !nextDue.After(lastLocal) { + nextDue = nextDue.AddDate(0, 0, schedule.DayInterval) + } + return nextDue, true +} + +func shouldEnqueueScheduledRun(workflow domain.WorkflowRecord, now time.Time) bool { + if workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule) { + return false + } + lastRunAt, _ := parseRFC3339Nano(workflow.LastRunAt) + switch domain.EffectiveWorkflowScheduleMode(workflow.Schedule) { + case domain.WorkflowScheduleModeEveryHours: + nextDue, ok := nextHourlyScheduleTime(workflow.Schedule, lastRunAt, now) + return ok && !now.Before(nextDue) + case domain.WorkflowScheduleModeEveryDaysAtTime: + nextDue, ok := nextDailyScheduleTime(workflow.Schedule, lastRunAt, now) + return ok && !now.Before(nextDue) + case domain.WorkflowScheduleModeIntervalSeconds: + if workflow.Schedule.Interval <= 0 { + return false + } + if workflow.LastRunAt == "" { + return true + } + return !lastRunAt.IsZero() && now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second + default: + return false + } +} + func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string { globalLimit := globalConcurrencyLimit(settings) workflowByID := make(map[string]domain.WorkflowRecord, len(workflows)) @@ -183,9 +305,19 @@ func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error { } now := time.Now().UTC() for _, workflow := range workflows { - if workflow.Type != domain.WorkflowTypeStreaming || workflow.Schedule == nil || workflow.Schedule.Interval <= 0 { + if !shouldEnqueueScheduledRun(workflow, now) { continue } + if strings.TrimSpace(workflow.ProjectID) != "" { + project, err := runtime.store.GetProject(ctx, workflow.ProjectID) + if err != nil { + return err + } + if project == nil { + fmt.Printf("[worker] skipping streaming workflow %s because project %s is missing\n", workflow.ID, workflow.ProjectID) + continue + } + } inflightCount := 0 for _, run := range runs { if run.WorkflowID == workflow.ID { @@ -197,17 +329,8 @@ func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error { if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) { continue } - lastRunAt := time.Time{} - if workflow.LastRunAt != "" { - parsed, err := time.Parse(time.RFC3339Nano, workflow.LastRunAt) - if err == nil { - lastRunAt = parsed - } - } - if workflow.LastRunAt == "" || now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second { - if _, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled); err != nil { - return err - } + if _, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled); err != nil { + return err } } return nil @@ -220,6 +343,9 @@ func (runtime *Runtime) processClaimedRun(claimed *domain.RunRecord) { } func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { + if err := runtime.reconcileActiveRuns(ctx); err != nil { + return err + } if err := runtime.enqueueStreamingWorkflows(ctx); err != nil { return err } @@ -250,6 +376,9 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { func chooseImage(node domain.WorkflowNode, fallback string) string { if node.Data.Image != "" { + if strings.Contains(node.Data.Image, "droboticflow-operator-runner") && fallback != "" { + return fallback + } return node.Data.Image } return fallback @@ -262,6 +391,514 @@ func chooseValue(value string, fallback string) string { return value } +func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error { + runs, err := runtime.store.ListRuns(ctx, "") + if err != nil { + return err + } + for _, candidate := range runs { + if candidate.Status != domain.RunStatusRunning { + continue + } + run, err := runtime.store.GetRun(ctx, candidate.ID) + if err != nil { + return err + } + if run == nil { + continue + } + if strings.TrimSpace(run.CurrentNodeID) == "" { + run.Status = domain.RunStatusQueued + run.StartedAt = "" + run.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *run); err != nil { + return err + } + continue + } + inspection, err := runtime.launcher.Inspect(ctx, run.ID, run.CurrentNodeID) + if err != nil { + return err + } + switch inspection.State { + case k8s.JobStateMissing: + run.Status = domain.RunStatusQueued + run.StartedAt = "" + run.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *run); err != nil { + return err + } + case k8s.JobStateSucceeded: + workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) + if err != nil { + return err + } + if workflow == nil { + continue + } + node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID) + if node == nil { + continue + } + _, existing := findNodeExecution(run, node.ID) + execution := domain.NodeExecutionRecord{ + NodeID: node.ID, + NodeLabel: node.Data.Label, + Status: domain.NodeExecutionStatusRunning, + StartedAt: firstNonEmptyString(run.StartedAt, run.CurrentRunAt, run.CreatedAt), + Logs: []domain.LogEntryRecord{}, + Artifacts: []domain.ExecutionArtifact{}, + DataRefs: []domain.DataRefRecord{}, + ResourceSeries: []domain.ResourceSample{}, + } + if existing != nil { + execution = *existing + } + parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label) + if err != nil { + return err + } + execution.Status = domain.NodeExecutionStatusCompleted + execution.EndedAt = nowISO() + execution.Logs = parsed.Logs + execution.Artifacts = parsed.Artifacts + execution.DataRefs = parsed.DataRefs + execution.ResourceSeries = parsed.ResourceSeries + execution.Error = "" + applyExecution(run, execution) + run.Status = domain.RunStatusQueued + run.StartedAt = "" + run.CurrentNodeID = "" + run.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *run); err != nil { + return err + } + } + } + return nil +} + +func upsertNodeHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam { + for index := range items { + if items[index].Key == key { + items[index].Value = value + return items + } + } + return append(items, domain.HyperParam{Key: key, Value: value}) +} + +func bosSourcePath(bucket string, prefix string) string { + normalizedBucket := strings.TrimSpace(bucket) + if normalizedBucket == "" { + return "" + } + sourcePath := "bos://" + normalizedBucket + if normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); normalizedPrefix != "" { + sourcePath += "/" + normalizedPrefix + } + return sourcePath +} + +func storageConnectionEndpoint(connection *domain.StorageConnectionRecord) string { + if connection == nil { + return "https://bj.bcebos.com" + } + if endpoint := strings.TrimSpace(connection.Endpoint); endpoint != "" { + return endpoint + } + return "https://bj.bcebos.com" +} + +func nodeHyperparamsMap(node domain.WorkflowNode) map[string]string { + values := make(map[string]string, len(node.Data.Hyperparams)) + for _, item := range node.Data.Hyperparams { + key := strings.TrimSpace(item.Key) + if key == "" { + continue + } + values[key] = item.Value + } + return values +} + +func chooseScanMode(node domain.WorkflowNode, params map[string]string) string { + mode := strings.ToUpper(strings.TrimSpace(params["SCAN_MODE"])) + if mode != "" { + return mode + } + if node.Data.Label == "BOS 增量扫描" { + return "INCREMENTAL" + } + return "" +} + +func chooseIncrementalUnit(params map[string]string) string { + switch strings.ToUpper(strings.TrimSpace(params["INCREMENTAL_UNIT"])) { + case "DIRECTORY": + return "DIRECTORY" + default: + return "OBJECT" + } +} + +func chooseDirectoryDepth(params map[string]string) int { + if parsed, err := strconv.Atoi(strings.TrimSpace(params["INCREMENTAL_DIRECTORY_DEPTH"])); err == nil && parsed > 0 { + return parsed + } + return 1 +} + +func scanWorkflowSourceNode(node domain.WorkflowNode) bool { + label := strings.TrimSpace(node.Data.Label) + return node.ID == "scan-source" || label == "BOS 增量扫描" +} + +func sourceFingerprint(node domain.WorkflowNode, params map[string]string, dataSource string, scanMode string) string { + return domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{ + ScanMode: scanMode, + IncrementalUnit: chooseIncrementalUnit(params), + DirectoryDepth: chooseDirectoryDepth(params), + DataSource: dataSource, + SourcePath: params["SOURCE_PATH"], + Bucket: node.Data.Bucket, + Prefix: node.Data.Prefix, + Pattern: node.Data.Pattern, + Endpoint: params["BOS_ENDPOINT"], + }) +} + +func stringList(value any) []string { + if value == nil { + return nil + } + if items, ok := value.([]string); ok { + result := make([]string, 0, len(items)) + for _, item := range items { + trimmed := strings.TrimSpace(item) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result + } + items, ok := value.([]any) + if ok { + result := make([]string, 0, len(items)) + for _, item := range items { + trimmed := strings.TrimSpace(fmt.Sprint(item)) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result + } + reflected := reflect.ValueOf(value) + if reflected.Kind() != reflect.Slice && reflected.Kind() != reflect.Array { + return nil + } + result := make([]string, 0, reflected.Len()) + for index := 0; index < reflected.Len(); index++ { + trimmed := strings.TrimSpace(fmt.Sprint(reflected.Index(index).Interface())) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + +func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) { + effectiveWorkflow, err := runtime.resolveScanWorkflow(ctx, workflow) + if err != nil { + return node, err + } + copy := scanExecutionNode(effectiveWorkflow, node) + copy, err = runtime.resolveNodeStorageConnection(ctx, copy) + if err != nil { + return node, err + } + copy, err = runtime.resolveNodeMongoConnection(ctx, copy) + if err != nil { + return node, err + } + params := nodeHyperparamsMap(copy) + dataSource := chooseValue(strings.ToUpper(strings.TrimSpace(params["DATA_SOURCE"])), "BOS") + scanMode := chooseScanMode(copy, params) + incrementalUnit := chooseIncrementalUnit(params) + directoryDepth := chooseDirectoryDepth(params) + if scanMode != "INCREMENTAL" || workflow == nil { + return copy, nil + } + fingerprint := sourceFingerprint(copy, params, dataSource, scanMode) + state, err := runtime.store.GetSourceState(ctx, workflow.ID, copy.ID, fingerprint) + if err != nil { + return copy, err + } + payload := map[string]any{ + "sourceFingerprint": fingerprint, + "scanMode": scanMode, + "incrementalUnit": incrementalUnit, + "directoryDepth": directoryDepth, + "dataSource": dataSource, + "sourcePath": strings.TrimSpace(params["SOURCE_PATH"]), + "bucket": strings.TrimSpace(copy.Data.Bucket), + "prefix": strings.TrimSpace(copy.Data.Prefix), + "pattern": strings.TrimSpace(copy.Data.Pattern), + "seenFingerprints": []string{}, + } + if state != nil { + payload["seenFingerprints"] = append([]string(nil), state.SeenFingerprints...) + payload["lastRunId"] = state.LastRunID + payload["lastListedCount"] = state.LastListedCount + payload["lastNewCount"] = state.LastNewCount + payload["lastScanCompletedAt"] = state.LastScanCompletedAt + } + encoded, err := json.Marshal(payload) + if err != nil { + return copy, err + } + copy.Data.Hyperparams = append([]domain.HyperParam(nil), copy.Data.Hyperparams...) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SCAN_MODE", scanMode) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_UNIT", incrementalUnit) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_DIRECTORY_DEPTH", strconv.Itoa(directoryDepth)) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_FINGERPRINT", fingerprint) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_STATE_JSON", string(encoded)) + return copy, nil +} + +func (runtime *Runtime) resolveScanWorkflow(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.WorkflowRecord, error) { + if workflow == nil || workflow.ScanConfig == nil || strings.TrimSpace(workflow.ScanConfig.ConnectionID) == "" { + return workflow, nil + } + connection, err := runtime.store.GetStorageConnection(ctx, workflow.ScanConfig.ConnectionID) + if err != nil { + return nil, err + } + if connection == nil { + return nil, fmt.Errorf("storage connection %s not found", workflow.ScanConfig.ConnectionID) + } + copy := *workflow + scanConfig := *workflow.ScanConfig + scanConfig.Endpoint = strings.TrimSpace(connection.Endpoint) + if scanConfig.Endpoint == "" { + scanConfig.Endpoint = "https://bj.bcebos.com" + } + if strings.TrimSpace(scanConfig.AccessKeyID) == "" { + scanConfig.AccessKeyID = connection.AccessKeyID + } + if strings.TrimSpace(scanConfig.SecretAccessKey) == "" { + scanConfig.SecretAccessKey = connection.SecretAccessKey + } + if strings.TrimSpace(scanConfig.SourcePath) == "" { + dataSource := chooseValue(strings.ToUpper(strings.TrimSpace(scanConfig.DataSource)), "BOS") + if dataSource == "BOS" { + scanConfig.SourcePath = bosSourcePath(scanConfig.Bucket, scanConfig.Prefix) + } + } + copy.ScanConfig = &scanConfig + return ©, nil +} + +func (runtime *Runtime) resolveNodeStorageConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) { + connectionID := strings.TrimSpace(node.Data.ConnectionID) + if connectionID == "" { + return node, nil + } + connection, err := runtime.store.GetStorageConnection(ctx, connectionID) + if err != nil { + return node, err + } + if connection == nil { + return node, fmt.Errorf("storage connection %s not found", connectionID) + } + copy := node + copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...) + sourcePath := strings.TrimSpace(nodeHyperparamsMap(copy)["SOURCE_PATH"]) + if sourcePath == "" { + sourcePath = bosSourcePath(copy.Data.Bucket, copy.Data.Prefix) + } + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", "BOS") + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", storageConnectionEndpoint(connection)) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", connection.AccessKeyID) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", connection.SecretAccessKey) + return copy, nil +} + +func (runtime *Runtime) resolveNodeMongoConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) { + connectionID := strings.TrimSpace(nodeHyperparamsMap(node)["MONGO_CONNECTION_ID"]) + if connectionID == "" { + return node, nil + } + connection, err := runtime.store.GetMongoConnection(ctx, connectionID) + if err != nil { + return node, err + } + if connection == nil { + return node, fmt.Errorf("mongo connection %s not found", connectionID) + } + copy := node + copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_URI", strings.TrimSpace(connection.URI)) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_DB", strings.TrimSpace(connection.Database)) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_TARGET", "business") + return copy, nil +} + +func checkpointArtifact(execution domain.NodeExecutionRecord) *domain.ExecutionArtifact { + for index := range execution.Artifacts { + if execution.Artifacts[index].Name == "source-checkpoint.json" { + return &execution.Artifacts[index] + } + } + return nil +} + +func sourceCheckpointPayload(artifact *domain.ExecutionArtifact) map[string]any { + if artifact == nil { + return nil + } + path := strings.TrimSpace(artifact.Path) + if path != "" { + data, err := os.ReadFile(path) + if err == nil { + var payload map[string]any + if err := json.Unmarshal(data, &payload); err == nil && len(payload) > 0 { + return payload + } + } + } + if artifact.Data == nil { + return nil + } + return artifact.Data +} + +func sourceStateFromCheckpoint(workflow *domain.WorkflowRecord, run *domain.RunRecord, execution domain.NodeExecutionRecord, artifact *domain.ExecutionArtifact) *domain.SourceStateRecord { + payload := sourceCheckpointPayload(artifact) + if workflow == nil || run == nil || payload == nil { + return nil + } + fingerprint := strings.TrimSpace(fmt.Sprint(payload["sourceFingerprint"])) + if fingerprint == "" { + return nil + } + return &domain.SourceStateRecord{ + ID: workflow.ID + ":" + execution.NodeID + ":" + fingerprint, + WorkflowID: workflow.ID, + NodeID: execution.NodeID, + SourceFingerprint: fingerprint, + DataSource: strings.TrimSpace(fmt.Sprint(payload["dataSource"])), + SourcePath: strings.TrimSpace(fmt.Sprint(payload["sourcePath"])), + Bucket: strings.TrimSpace(fmt.Sprint(payload["bucket"])), + Prefix: strings.TrimSpace(fmt.Sprint(payload["prefix"])), + Pattern: strings.TrimSpace(fmt.Sprint(payload["pattern"])), + ScanMode: strings.TrimSpace(fmt.Sprint(payload["scanMode"])), + IncrementalUnit: strings.TrimSpace(fmt.Sprint(payload["incrementalUnit"])), + DirectoryDepth: intValue(payload["directoryDepth"]), + SeenFingerprints: stringList(payload["seenFingerprints"]), + LastRunID: run.ID, + LastListedCount: intValue(payload["listedCount"]), + LastNewCount: intValue(payload["newCount"]), + LastScanStartedAt: strings.TrimSpace(fmt.Sprint(payload["scanStartedAt"])), + LastScanCompletedAt: strings.TrimSpace(fmt.Sprint(payload["scanCompletedAt"])), + UpdatedAt: nowISO(), + } +} + +func intValue(value any) int { + switch typed := value.(type) { + case int: + return typed + case int32: + return int(typed) + case int64: + return int(typed) + case float32: + return int(typed) + case float64: + return int(typed) + case json.Number: + if parsed, err := typed.Int64(); err == nil { + return int(parsed) + } + if parsed, err := typed.Float64(); err == nil { + return int(parsed) + } + case string: + if parsed, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil { + return parsed + } + default: + return 0 + } + return 0 +} + +func (runtime *Runtime) commitSourceStates(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord) error { + if workflow == nil || run == nil { + return nil + } + sourceRun := run + persistedRun, err := runtime.store.GetRun(ctx, run.ID) + if err != nil { + return err + } + if persistedRun != nil { + sourceRun = persistedRun + } + for _, execution := range sourceRun.NodeExecutions { + artifact := checkpointArtifact(execution) + if artifact == nil { + continue + } + state := sourceStateFromCheckpoint(workflow, sourceRun, execution, artifact) + if state == nil { + continue + } + if err := runtime.store.UpsertSourceState(ctx, *state); err != nil { + return err + } + } + return nil +} + +func scanExecutionNode(workflow *domain.WorkflowRecord, node domain.WorkflowNode) domain.WorkflowNode { + if workflow == nil || workflow.ScanConfig == nil { + return node + } + if !scanWorkflowSourceNode(node) { + return node + } + dataSource := chooseValue(strings.ToUpper(strings.TrimSpace(workflow.ScanConfig.DataSource)), "BOS") + sourcePath := strings.TrimSpace(workflow.ScanConfig.SourcePath) + if sourcePath == "" && dataSource == "BOS" { + sourcePath = bosSourcePath(workflow.ScanConfig.Bucket, workflow.ScanConfig.Prefix) + } + copy := node + copy.Data.Bucket = workflow.ScanConfig.Bucket + copy.Data.Prefix = workflow.ScanConfig.Prefix + copy.Data.Pattern = workflow.ScanConfig.Pattern + copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", dataSource) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", workflow.ScanConfig.Endpoint) + copy.Data.Hyperparams = upsertNodeHyperparam( + copy.Data.Hyperparams, + "SCAN_MODE", + chooseValue(strings.ToUpper(strings.TrimSpace(workflow.ScanConfig.ScanMode)), "INCREMENTAL"), + ) + bosAccessKeyID := "" + bosSecretAccessKey := "" + if dataSource == "BOS" { + bosAccessKeyID = workflow.ScanConfig.AccessKeyID + bosSecretAccessKey = workflow.ScanConfig.SecretAccessKey + } + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", bosAccessKeyID) + copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", bosSecretAccessKey) + return copy +} + func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration { if workflow == nil || workflow.RunTimeoutSeconds <= 0 { return 0 @@ -375,6 +1012,69 @@ func syncCurrentNodeID(run *domain.RunRecord) { } } +func isTerminalRunStatus(status domain.RunStatus) bool { + return status == domain.RunStatusCompleted || + status == domain.RunStatusFailed || + status == domain.RunStatusCancelled +} + +func (runtime *Runtime) loadRunControlState(ctx context.Context, runID string) (*domain.RunRecord, error) { + if strings.TrimSpace(runID) == "" { + return nil, nil + } + return runtime.store.GetRun(ctx, runID) +} + +func syncStopMetadata(target *domain.RunRecord, source *domain.RunRecord) { + if target == nil || source == nil { + return + } + target.StopMode = source.StopMode + target.StopRequestedAt = source.StopRequestedAt +} + +func finalizeCancelledExecution(execution domain.NodeExecutionRecord, message string) (domain.NodeExecutionRecord, bool) { + if execution.Status == domain.NodeExecutionStatusCompleted || + execution.Status == domain.NodeExecutionStatusFailed || + execution.Status == domain.NodeExecutionStatusCancelled { + return execution, false + } + execution.Status = domain.NodeExecutionStatusCancelled + execution.EndedAt = nowISO() + execution.Error = message + execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel)) + return execution, true +} + +func (runtime *Runtime) finalizeCancelledRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, message string) (*domain.RunRecord, error) { + for nodeID, execution := range executionMap { + updated, changed := finalizeCancelledExecution(execution, message) + if changed { + executionMap[nodeID] = updated + applyExecution(currentRun, updated) + } + } + currentRun.Status = domain.RunStatusCancelled + currentRun.EndedAt = nowISO() + if strings.TrimSpace(currentRun.Error) == "" { + currentRun.Error = message + } + syncCurrentNodeID(currentRun) + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + if workflow != nil { + workflow.Status = domain.WorkflowStatusFailed + workflow.LastRunID = currentRun.ID + workflow.UpdatedAt = nowISO() + if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil { + return nil, err + } + } + return currentRun, nil +} + func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool { for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) { dependency, ok := executionMap[candidate.Source] @@ -385,7 +1085,56 @@ func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[strin return true } -func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, nodeID string) { +func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord) map[string]struct{} { + if run == nil || run.Trigger != domain.RunTriggerRerunSubtree || strings.TrimSpace(run.RestartFromNodeID) == "" { + return nil + } + visited := map[string]struct{}{} + queue := []string{strings.TrimSpace(run.RestartFromNodeID)} + for len(queue) > 0 { + nodeID := queue[0] + queue = queue[1:] + if _, ok := visited[nodeID]; ok { + continue + } + visited[nodeID] = struct{}{} + for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) { + target := strings.TrimSpace(edge.Target) + if target != "" { + queue = append(queue, target) + } + } + } + return visited +} + +func nodeAllowedForExecution(allowed map[string]struct{}, nodeID string) bool { + if allowed == nil { + return true + } + _, ok := allowed[nodeID] + return ok +} + +func firstActiveExecution(executionMap map[string]domain.NodeExecutionRecord) (string, domain.NodeExecutionStatus) { + for _, status := range []domain.NodeExecutionStatus{ + domain.NodeExecutionStatusWaitingForAudit, + domain.NodeExecutionStatusRunning, + domain.NodeExecutionStatusQueued, + } { + for nodeID, execution := range executionMap { + if execution.Status == status { + return nodeID, status + } + } + } + return "", "" +} + +func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) { + if !nodeAllowedForExecution(allowed, nodeID) { + return + } if _, queued := readySet[nodeID]; queued { return } @@ -403,9 +1152,9 @@ func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]do readySet[nodeID] = struct{}{} } -func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, nodeID string) { +func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) { for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) { - queueNodeIfReady(workflow, executionMap, readyQueue, readySet, edge.Target) + queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, edge.Target) } } @@ -421,6 +1170,16 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor claimed.UpdatedAt = nowISO() return claimed, runtime.store.UpsertRun(ctx, *claimed) } + if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 { + currentRun := claimed + message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。" + currentRun.Status = domain.RunStatusFailed + currentRun.EndedAt = nowISO() + currentRun.Error = message + currentRun.UpdatedAt = nowISO() + appendLog(currentRun, logEntry("ERROR", message, "system")) + return currentRun, runtime.store.UpsertRun(ctx, *currentRun) + } currentRun, err := runtime.store.GetRun(ctx, claimed.ID) if err != nil { @@ -429,6 +1188,9 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor if currentRun == nil { currentRun = claimed } + if isTerminalRunStatus(currentRun.Status) { + return currentRun, nil + } settings, err := runtime.store.GetSettings(ctx) if err != nil { return nil, err @@ -442,6 +1204,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor for _, execution := range currentRun.NodeExecutions { executionMap[execution.NodeID] = execution } + allowedNodes := rerunAllowedNodeSet(*workflow, currentRun) readyQueue := make([]string, 0) readySet := make(map[string]struct{}) auditQueue := make([]string, 0) @@ -464,7 +1227,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor executionMap[node.ID] = execution applyExecution(currentRun, execution) for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) { - queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, target) + queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, target) } currentRun.Status = domain.RunStatusRunning syncCurrentNodeID(currentRun) @@ -478,9 +1241,45 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor } } + if activeNodeID, activeStatus := firstActiveExecution(executionMap); activeNodeID != "" { + if strings.TrimSpace(currentRun.CurrentNodeID) == "" { + currentRun.CurrentNodeID = activeNodeID + } + switch activeStatus { + case domain.NodeExecutionStatusWaitingForAudit: + currentRun.Status = domain.RunStatusWaitingForAudit + default: + currentRun.Status = domain.RunStatusRunning + } + currentRun.UpdatedAt = nowISO() + syncCurrentNodeID(currentRun) + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + workflow.Status = domain.WorkflowStatusRunning + workflow.LastRunID = currentRun.ID + workflow.UpdatedAt = nowISO() + if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil { + return nil, err + } + return currentRun, nil + } + if len(readyQueue) == 0 && len(executionMap) == 0 { - for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) { - queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, nodeID) + if allowedNodes != nil && strings.TrimSpace(currentRun.RestartFromNodeID) != "" { + queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, currentRun.RestartFromNodeID) + } else { + for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) { + queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, nodeID) + } + } + } + if len(readyQueue) == 0 && len(executionMap) > 0 && currentRun.CurrentNodeID == "" { + for _, execution := range currentRun.NodeExecutions { + if execution.Status != domain.NodeExecutionStatusCompleted { + continue + } + enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, execution.NodeID) } } @@ -494,9 +1293,39 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor defer cancelExecution() results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit) runningNodes := make(map[string]struct{}) + var persistedControl *domain.RunRecord + gracefulStopRequested := false + forceStopRequested := false for { + persistedControl, err = runtime.loadRunControlState(ctx, currentRun.ID) + if err != nil { + return nil, err + } + gracefulStopRequested = false + forceStopRequested = false + if persistedControl != nil { + syncStopMetadata(currentRun, persistedControl) + if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful { + gracefulStopRequested = true + currentRun.Status = domain.RunStatusStopping + } + if persistedControl.Status == domain.RunStatusCancelled { + forceStopRequested = persistedControl.StopMode == domain.RunStopModeForce + currentRun.Status = domain.RunStatusCancelled + if currentRun.EndedAt == "" { + currentRun.EndedAt = persistedControl.EndedAt + } + if strings.TrimSpace(currentRun.Error) == "" { + currentRun.Error = persistedControl.Error + } + } + } + if forceStopRequested { + cancelExecution() + } + launched := false - for len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 { + for !gracefulStopRequested && !forceStopRequested && len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 { nodeID := readyQueue[0] readyQueue = readyQueue[1:] delete(readySet, nodeID) @@ -505,6 +1334,9 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor if node == nil { continue } + if !nodeAllowedForExecution(allowedNodes, node.ID) { + continue + } if existing, ok := executionMap[node.ID]; ok { if existing.Status == domain.NodeExecutionStatusCompleted || existing.Status == domain.NodeExecutionStatusRunning || @@ -522,13 +1354,15 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor inputArtifacts := gatherInputArtifacts(*workflow, executionMap, node.ID) execution := domain.NodeExecutionRecord{ - NodeID: node.ID, - NodeLabel: node.Data.Label, - Status: domain.NodeExecutionStatusRunning, - StartedAt: nowISO(), - Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)}, - Artifacts: []domain.ExecutionArtifact{}, - ResourceSeries: []domain.ResourceSample{}, + NodeID: node.ID, + NodeLabel: node.Data.Label, + Status: domain.NodeExecutionStatusRunning, + StartedAt: nowISO(), + Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)}, + Artifacts: []domain.ExecutionArtifact{}, + DataRefs: []domain.DataRefRecord{}, + ResourceSeries: []domain.ResourceSample{}, + StageManifestPath: "", } executionMap[node.ID] = execution applyExecution(currentRun, execution) @@ -536,7 +1370,10 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor launched = true runSnapshot := *currentRun - nodeSnapshot := *node + nodeSnapshot, err := runtime.prepareExecutionNode(ctx, workflow, *node) + if err != nil { + return nil, err + } inputSnapshot := cloneArtifacts(inputArtifacts) nodeExecutionCtx := executionCtx cancelNodeExecution := func() {} @@ -566,6 +1403,12 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor } if len(runningNodes) == 0 { + if forceStopRequested { + return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流") + } + if gracefulStopRequested || currentRun.Status == domain.RunStatusStopping { + return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已停止工作流") + } if len(readyQueue) > 0 { continue } @@ -630,8 +1473,37 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor outcome := <-results delete(runningNodes, outcome.nodeID) + persistedControl, err := runtime.loadRunControlState(ctx, currentRun.ID) + if err != nil { + return nil, err + } + gracefulStopRequested := false + forceStopRequested := false + if persistedControl != nil { + syncStopMetadata(currentRun, persistedControl) + if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful { + gracefulStopRequested = true + } + if persistedControl.Status == domain.RunStatusCancelled && persistedControl.StopMode == domain.RunStopModeForce { + forceStopRequested = true + } + } execution := executionMap[outcome.nodeID] if outcome.err != nil { + if forceStopRequested || (currentRun.StopMode == domain.RunStopModeForce && errors.Is(executionCtx.Err(), context.Canceled)) { + cancelledExecution, _ := finalizeCancelledExecution(execution, "用户已强制停止工作流") + executionMap[outcome.nodeID] = cancelledExecution + applyExecution(currentRun, cancelledExecution) + currentRun.Status = domain.RunStatusCancelled + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + if len(runningNodes) == 0 { + return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流") + } + continue + } cancelExecution() execution.Status = domain.NodeExecutionStatusFailed execution.EndedAt = nowISO() @@ -672,21 +1544,44 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor execution.EndedAt = nowISO() execution.Logs = append(execution.Logs, outcome.result.Logs...) execution.Artifacts = outcome.result.Artifacts + execution.DataRefs = outcome.result.DataRefs + execution.StageManifestPath = outcome.result.StageManifestPath execution.ResourceSeries = outcome.result.ResourceSeries executionMap[outcome.nodeID] = execution applyExecution(currentRun, execution) if outcome.reference != nil { currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace}) } - enqueueDependents(*workflow, executionMap, &readyQueue, readySet, outcome.nodeID) - currentRun.Status = domain.RunStatusRunning + if !gracefulStopRequested && !forceStopRequested { + enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, outcome.nodeID) + } + if gracefulStopRequested { + currentRun.Status = domain.RunStatusStopping + } else if forceStopRequested { + currentRun.Status = domain.RunStatusCancelled + } else { + currentRun.Status = domain.RunStatusRunning + } syncCurrentNodeID(currentRun) currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } + if forceStopRequested && len(runningNodes) == 0 { + return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流") + } } + if currentRun.Status == domain.RunStatusStopping || currentRun.Status == domain.RunStatusCancelled { + message := "用户已停止工作流" + if currentRun.StopMode == domain.RunStopModeForce { + message = "用户已强制停止工作流" + } + return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, message) + } + if err := runtime.commitSourceStates(ctx, workflow, currentRun); err != nil { + return nil, err + } currentRun.Status = domain.RunStatusCompleted currentRun.EndedAt = nowISO() syncCurrentNodeID(currentRun) @@ -696,7 +1591,6 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor } workflow.Status = domain.WorkflowStatusCompleted workflow.LastRunID = currentRun.ID - workflow.LastRunAt = nowISO() workflow.UpdatedAt = nowISO() if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil { return nil, err @@ -713,6 +1607,9 @@ func edgeTargets(edges []domain.WorkflowEdge) []string { } func (runtime *Runtime) ProcessOnce(ctx context.Context) (*domain.RunRecord, error) { + if err := runtime.reconcileActiveRuns(ctx); err != nil { + return nil, err + } if err := runtime.enqueueStreamingWorkflows(ctx); err != nil { return nil, err } diff --git a/internal/worker/runtime_test.go b/internal/worker/runtime_test.go index ed0e8b77..07eebb31 100644 --- a/internal/worker/runtime_test.go +++ b/internal/worker/runtime_test.go @@ -2,24 +2,99 @@ package worker import ( "context" + "encoding/json" + "fmt" "os" "os/exec" "path/filepath" + "sort" "strings" "testing" "time" "droboticflow/internal/domain" + "droboticflow/internal/k8s" "droboticflow/internal/store" ) -func localRunnerCommand(t *testing.T) string { +type testLauncher struct { + launch func(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error) + inspect func(context.Context, string, string) (*k8s.JobInspection, error) +} + +func (launcher testLauncher) Launch(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { + if launcher.launch == nil { + return nil, fmt.Errorf("test launcher launch not configured") + } + return launcher.launch(ctx, input) +} + +func (launcher testLauncher) Inspect(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) { + if launcher.inspect == nil { + return &k8s.JobInspection{State: k8s.JobStateMissing}, nil + } + return launcher.inspect(ctx, runID, nodeID) +} + +func operatorRunnerPath(t *testing.T) string { t.Helper() path, err := filepath.Abs("../../backend/operator-runner/main.py") if err != nil { t.Fatalf("resolve operator runner: %v", err) } - return "python3 " + path + return path +} + +func newTestRuntime(t *testing.T, dataStore store.Store) (*Runtime, error) { + t.Helper() + runnerPath := operatorRunnerPath(t) + return &Runtime{ + store: dataStore, + operatorRunnerImage: "operator-runner:latest", + workspaceRoot: t.TempDir(), + launcher: testLauncher{ + launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { + env := append([]string{}, os.Environ()...) + keys := make([]string, 0, len(input.Env)) + for key := range input.Env { + keys = append(keys, key) + } + sort.Strings(keys) + env = append(env, + "RUN_ID="+input.RunID, + "WORKFLOW_ID="+input.WorkflowID, + "NODE_ID="+input.NodeID, + "NODE_LABEL="+input.NodeLabel, + ) + for _, key := range keys { + env = append(env, key+"="+input.Env[key]) + } + cmd := exec.CommandContext(ctx, "python3", runnerPath) + cmd.Env = env + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("%w: %s", err, string(output)) + } + return &k8s.LaunchResult{ + Reference: &k8s.JobReference{ + JobName: fmt.Sprintf("%s-%s", input.RunID, input.NodeID), + Namespace: "droboticflow", + }, + Output: string(output), + }, nil + }, + }, + }, nil +} + +func newTestRuntimeWithLauncher(t *testing.T, dataStore store.Store, launcher jobLauncher) *Runtime { + t.Helper() + return &Runtime{ + store: dataStore, + operatorRunnerImage: "operator-runner:latest", + workspaceRoot: t.TempDir(), + launcher: launcher, + } } func mustNodeExecution(t *testing.T, run *domain.RunRecord, nodeID string) domain.NodeExecutionRecord { @@ -44,6 +119,280 @@ func mustArtifact(t *testing.T, execution domain.NodeExecutionRecord, name strin return domain.ExecutionArtifact{} } +func mustArtifactPayload(t *testing.T, artifact domain.ExecutionArtifact) map[string]any { + t.Helper() + if artifact.Data != nil { + if dataRef, ok := artifact.Data["dataRef"].(map[string]any); ok { + if payloadPath, ok := dataRef["payloadPath"].(string); ok && strings.TrimSpace(payloadPath) != "" { + payload, err := os.ReadFile(payloadPath) + if err != nil { + t.Fatalf("read artifact payload %s: %v", payloadPath, err) + } + var decoded map[string]any + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("decode artifact payload %s: %v", payloadPath, err) + } + return decoded + } + } + return artifact.Data + } + if strings.TrimSpace(artifact.Path) != "" { + payload, err := os.ReadFile(artifact.Path) + if err == nil { + var decoded map[string]any + if err := json.Unmarshal(payload, &decoded); err == nil { + return decoded + } + } + } + t.Fatalf("artifact %s has no structured payload", artifact.Name) + return nil +} + +func TestProcessRunFailsInvalidWorkflowWithDuplicateInputEdges(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "非法工作流项目", + Description: "duplicate input edges", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "非法工作流", + Description: "duplicate input edges", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source-a", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "上游 A", + Category: "IO", + Description: "source a", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, + }, + }, + { + ID: "source-b", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "上游 B", + Category: "IO", + Description: "source b", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, + }, + }, + { + ID: "upload", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "结果上传", + Category: "IO", + Description: "upload", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "result", Type: domain.PortTypeJSON}}, + }, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "edge-a", Source: "source-a", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, + {ID: "edge-b", Source: "source-b", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + + processed, err := runtime.processRun(ctx, run) + if err != nil { + t.Fatalf("process run: %v", err) + } + if processed.Status != domain.RunStatusFailed { + t.Fatalf("expected failed run, got %s", processed.Status) + } + if !strings.Contains(processed.Error, "同一输入端口存在多条上游连线") { + t.Fatalf("expected duplicate input edge error, got %q", processed.Error) + } + if len(processed.NodeExecutions) != 0 { + t.Fatalf("expected no node executions for invalid workflow, got %#v", processed.NodeExecutions) + } +} + +func TestSourceStateFromCheckpointPrefersCheckpointFilePayload(t *testing.T) { + workflow := &domain.WorkflowRecord{ID: "workflow-1"} + run := &domain.RunRecord{ID: "run-1"} + checkpointPath := filepath.Join(t.TempDir(), "source-checkpoint.json") + writeFile(t, checkpointPath, `{ + "sourceFingerprint": "fp-1", + "dataSource": "BOS", + "sourcePath": "bos://datastorage/algo/X-platform-test", + "bucket": "datastorage", + "prefix": "algo/X-platform-test", + "pattern": "", + "scanMode": "INCREMENTAL", + "seenFingerprints": ["a", "b"], + "listedCount": 28, + "newCount": 0, + "scanStartedAt": "2026-04-12T15:45:42Z", + "scanCompletedAt": "2026-04-12T15:45:55Z" +}`) + execution := domain.NodeExecutionRecord{ + NodeID: "scan-source", + Artifacts: []domain.ExecutionArtifact{ + { + Name: "source-checkpoint.json", + Path: checkpointPath, + Data: map[string]any{ + "sourceFingerprint": "fp-1", + "listedCount": 28.0, + "newCount": 28.0, + }, + }, + }, + } + + state := sourceStateFromCheckpoint(workflow, run, execution, &execution.Artifacts[0]) + if state == nil { + t.Fatalf("expected source state") + } + if state.LastNewCount != 0 { + t.Fatalf("expected checkpoint file payload to win, got lastNewCount=%d", state.LastNewCount) + } + if state.LastListedCount != 28 { + t.Fatalf("expected lastListedCount=28, got %d", state.LastListedCount) + } + if state.LastRunID != run.ID { + t.Fatalf("expected lastRunID=%s, got %s", run.ID, state.LastRunID) + } +} + +func TestCommitSourceStatesPrefersPersistedRunSnapshot(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + workflow := &domain.WorkflowRecord{ID: "workflow-1"} + persistedRun := domain.RunRecord{ + ID: "run-1", + WorkflowID: workflow.ID, + NodeExecutions: []domain.NodeExecutionRecord{{ + NodeID: "scan-source", + Artifacts: []domain.ExecutionArtifact{{ + Name: "source-checkpoint.json", + Data: map[string]any{ + "sourceFingerprint": "fp-1", + "dataSource": "BOS", + "sourcePath": "bos://datastorage/algo/X-platform-test", + "bucket": "datastorage", + "prefix": "algo/X-platform-test", + "scanMode": "INCREMENTAL", + "seenFingerprints": []any{"a", "b"}, + "listedCount": 28.0, + "newCount": 0.0, + "scanStartedAt": "2026-04-12T15:45:42Z", + "scanCompletedAt": "2026-04-12T15:45:55Z", + }, + }}, + }}, + } + if err := dataStore.UpsertRun(ctx, persistedRun); err != nil { + t.Fatalf("upsert persisted run: %v", err) + } + inMemoryRun := &domain.RunRecord{ + ID: "run-1", + WorkflowID: workflow.ID, + NodeExecutions: []domain.NodeExecutionRecord{{ + NodeID: "scan-source", + Artifacts: []domain.ExecutionArtifact{{ + Name: "source-checkpoint.json", + Data: map[string]any{ + "sourceFingerprint": "fp-1", + "listedCount": 28.0, + "newCount": 28.0, + }, + }}, + }}, + } + + if err := runtime.commitSourceStates(ctx, workflow, inMemoryRun); err != nil { + t.Fatalf("commit source states: %v", err) + } + state, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", "fp-1") + if err != nil { + t.Fatalf("get source state: %v", err) + } + if state == nil { + t.Fatalf("expected committed source state") + } + if state.LastNewCount != 0 { + t.Fatalf("expected committed source state to use persisted run snapshot, got %d", state.LastNewCount) + } +} + +func TestCommitSourceStatesPreservesSeenFingerprintsFromTypedSlice(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + workflow := &domain.WorkflowRecord{ID: "workflow-typed-slice"} + persistedRun := domain.RunRecord{ + ID: "run-typed-slice", + WorkflowID: workflow.ID, + NodeExecutions: []domain.NodeExecutionRecord{{ + NodeID: "scan-source", + Artifacts: []domain.ExecutionArtifact{{ + Name: "source-checkpoint.json", + Data: map[string]any{ + "sourceFingerprint": "fp-typed-slice", + "dataSource": "BOS", + "sourcePath": "bos://datastorage/algo/X-platform-test/raw", + "bucket": "datastorage", + "prefix": "algo/X-platform-test/raw", + "scanMode": "INCREMENTAL", + "seenFingerprints": []string{"a", "b", "c"}, + "listedCount": 4.0, + "newCount": 0.0, + }, + }}, + }}, + } + if err := dataStore.UpsertRun(ctx, persistedRun); err != nil { + t.Fatalf("upsert persisted run: %v", err) + } + + if err := runtime.commitSourceStates(ctx, workflow, &persistedRun); err != nil { + t.Fatalf("commit source states: %v", err) + } + state, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", "fp-typed-slice") + if err != nil { + t.Fatalf("get source state: %v", err) + } + if state == nil { + t.Fatalf("expected committed source state") + } + if len(state.SeenFingerprints) != 3 { + t.Fatalf("expected 3 seen fingerprints, got %#v", state.SeenFingerprints) + } + if strings.Join(state.SeenFingerprints, ",") != "a,b,c" { + t.Fatalf("unexpected seen fingerprints %#v", state.SeenFingerprints) + } +} + func writeFile(t *testing.T, path string, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { @@ -78,6 +427,159 @@ func createRosbagDB3(t *testing.T, path string) { } } +func TestChooseImagePrefersRuntimeOperatorRunnerForBuiltins(t *testing.T) { + node := domain.WorkflowNode{ + Data: domain.WorkflowNodeData{ + Image: "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:old-tag", + }, + } + image := chooseImage(node, "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:new-tag") + if image != "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:new-tag" { + t.Fatalf("expected runtime image override for built-in operator-runner, got %q", image) + } +} + +func createTestVideoMP4(t *testing.T, path string, pattern string, fps int, frames int, width int, height int) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + command := exec.Command( + "python3", + "-c", + strings.Join([]string{ + "import cv2, numpy as np, sys", + "path, pattern, fps, frames, width, height = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), int(sys.argv[5]), int(sys.argv[6])", + "fourcc = cv2.VideoWriter_fourcc(*'mp4v')", + "writer = cv2.VideoWriter(path, fourcc, fps, (width, height))", + "assert writer.isOpened(), 'video writer did not open'", + "for index in range(frames):", + " frame = np.zeros((height, width, 3), dtype=np.uint8)", + " if pattern == 'sharp':", + " x = int((index * 3) % max(1, width - 18))", + " cv2.rectangle(frame, (x, 8), (x + 16, 24), (255, 255, 255), -1)", + " cv2.putText(frame, str(index % 10), (6, height - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1, cv2.LINE_AA)", + " elif pattern == 'blur':", + " cv2.rectangle(frame, (16, 16), (width - 16, height - 16), (220, 220, 220), -1)", + " frame = cv2.GaussianBlur(frame, (23, 23), 0)", + " else:", + " x = int((index * 5) % max(1, width - 20))", + " frame[:] = (40, 60, 90)", + " cv2.circle(frame, (x + 10, height // 2), 10, (80, 140, 190), -1)", + " writer.write(frame)", + "writer.release()", + "capture = cv2.VideoCapture(path)", + "assert capture.isOpened(), 'video capture did not open'", + "capture.release()", + "print(path)", + }, "\n"), + path, + pattern, + fmt.Sprintf("%d", fps), + fmt.Sprintf("%d", frames), + fmt.Sprintf("%d", width), + fmt.Sprintf("%d", height), + ) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("create test mp4 %s: %v: %s", path, err, string(output)) + } +} + +func createXPlatformDataset(t *testing.T) string { + t.Helper() + root := filepath.Join(t.TempDir(), "algo", "X-platform-test") + fps := 10 + width := 96 + height := 64 + + type videoSpec struct { + Name string + Pattern string + Frames int + Duration float64 + } + type collectionSpec struct { + Name string + Type string + Place string + Date string + DeviceID string + HasIntrinsics bool + VideoDirectory string + Videos []videoSpec + } + + collections := []collectionSpec{ + { + Name: "BJ_001_0001_OsmoNano_2026-03-19_14-51-43", + Type: "家庭", + Place: "蓟门里小区", + Date: "20260319", + DeviceID: "C00000", + HasIntrinsics: true, + VideoDirectory: "DJI_001", + Videos: []videoSpec{ + {Name: "DJI_0001_D.mp4", Pattern: "sharp", Frames: 40, Duration: 4.0}, + {Name: "DJI_0002_D.mp4", Pattern: "blur", Frames: 20, Duration: 2.0}, + }, + }, + { + Name: "BJ_002_0002_OsmoNano_2026-03-20_10-30-00", + Type: "办公", + Place: "中关村", + Date: "20260320", + DeviceID: "C00001", + HasIntrinsics: false, + VideoDirectory: "DJI_001", + Videos: []videoSpec{ + {Name: "DJI_0001_D.mp4", Pattern: "sharp", Frames: 20, Duration: 2.0}, + }, + }, + } + + for _, collection := range collections { + collectionRoot := filepath.Join(root, collection.Name) + videoRoot := filepath.Join(collectionRoot, collection.VideoDirectory) + videoMetaRows := make([]map[string]any, 0, len(collection.Videos)) + for index, video := range collection.Videos { + videoPath := filepath.Join(videoRoot, video.Name) + createTestVideoMP4(t, videoPath, video.Pattern, fps, video.Frames, width, height) + videoMetaRows = append(videoMetaRows, map[string]any{ + "filename": video.Name, + "codec": "mp4v", + "resolution": fmt.Sprintf("%dx%d", width, height), + "frameRate": float64(fps), + "bitrate": 2000000 + index*100000, + "duration": video.Duration, + "startTime": 1773904218 + index*60, + }) + } + metaJSON, err := json.Marshal(map[string]any{ + "type": collection.Type, + "place": collection.Place, + "mission": "无", + "project_id": collection.Name, + "date": collection.Date, + "device_id": collection.DeviceID, + }) + if err != nil { + t.Fatalf("marshal meta json: %v", err) + } + writeFile(t, filepath.Join(collectionRoot, "meta.json"), string(metaJSON)) + videoMetaJSON, err := json.Marshal(videoMetaRows) + if err != nil { + t.Fatalf("marshal video_meta json: %v", err) + } + writeFile(t, filepath.Join(collectionRoot, "video_meta.json"), string(videoMetaJSON)) + writeFile(t, filepath.Join(videoRoot, "DJI_meta.json"), `{"ok":true}`) + if collection.HasIntrinsics { + writeFile(t, filepath.Join(collectionRoot, "intrinsics.json"), `{"fx": 1000, "fy": 1000}`) + } + } + + return root +} + func waitForRunStatus(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected domain.RunStatus, timeout time.Duration) *domain.RunRecord { t.Helper() deadline := time.Now().Add(timeout) @@ -156,6 +658,7 @@ func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) { {ID: "out-1", Name: "file", Type: domain.PortTypeFile}, }, Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "worker-runtime-test"}, }, }, @@ -188,7 +691,7 @@ func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) { if err != nil { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -220,6 +723,116 @@ func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) { } } +func TestRuntimeStopsGracefullyWithoutDispatchingNewNodes(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "停止项目", Description: "graceful"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "停止工作流", + Description: "graceful stop path", + Type: domain.WorkflowTypeBatch, + MaxConcurrentRuns: 1, + NodeTimeoutSeconds: 30, + RunTimeoutSeconds: 60, + MaxRetryAttempts: 0, + RetryBackoffSeconds: 0, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "source", + Outputs: []domain.PortDefinition{ + {ID: "payload", Name: "payload", Type: domain.PortTypeJSON}, + }, + Hyperparams: []domain.HyperParam{ + {Key: "COMMAND", Value: `sleep 1; printf '{"value":1}' > payload.json`}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "sink", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "结果上传", + Category: "IO", + Description: "sink", + Inputs: []domain.PortDefinition{ + {ID: "input", Name: "input", Type: domain.PortTypeJSON}, + }, + }, + Position: domain.Position{X: 120, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "e1", Source: "source", SourceHandle: "payload", Target: "sink", TargetHandle: "input"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + + done := make(chan error, 1) + go func() { + _, processErr := runtime.ProcessOnce(ctx) + done <- processErr + }() + + waitForRunningNodeCount(t, ctx, dataStore, run.ID, 1, 5*time.Second) + stored, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get running run: %v", err) + } + stored.Status = domain.RunStatus("stopping") + stored.StopMode = domain.RunStopModeGraceful + stored.StopRequestedAt = nowISO() + stored.UpdatedAt = nowISO() + if err := dataStore.UpsertRun(ctx, *stored); err != nil { + t.Fatalf("request graceful stop: %v", err) + } + + select { + case err := <-done: + if err != nil { + t.Fatalf("process run: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for graceful stop") + } + + finalRun, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get final run: %v", err) + } + if finalRun == nil || finalRun.Status != domain.RunStatusCancelled { + t.Fatalf("expected cancelled run after graceful stop, got %#v", finalRun) + } + if sink := mustNodeExecution(t, finalRun, "source"); sink.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected source node to complete before stop, got %s", sink.Status) + } + for _, item := range finalRun.NodeExecutions { + if item.NodeID == "sink" { + t.Fatalf("expected sink node not to start after graceful stop, got %#v", item) + } + } +} + func TestRuntimePausesForAuditAndResumes(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -277,7 +890,7 @@ func TestRuntimePausesForAuditAndResumes(t *testing.T) { if err != nil { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -423,7 +1036,7 @@ func TestRuntimeRoutesArtifactsByOutputHandle(t *testing.T) { if err != nil { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -451,8 +1064,10 @@ func TestRuntimeRoutesArtifactsByOutputHandle(t *testing.T) { dropUpload := mustNodeExecution(t, completed, "drop-upload") keepSummary := mustArtifact(t, keepUpload, "uploaded-results.json") dropSummary := mustArtifact(t, dropUpload, "uploaded-results.json") - keepItems, _ := keepSummary.Data["items"].([]any) - dropItems, _ := dropSummary.Data["items"].([]any) + keepSummaryPayload := mustArtifactPayload(t, keepSummary) + dropSummaryPayload := mustArtifactPayload(t, dropSummary) + keepItems, _ := keepSummaryPayload["items"].([]any) + dropItems, _ := dropSummaryPayload["items"].([]any) if len(keepItems) != 1 || len(dropItems) != 1 { t.Fatalf("expected one routed payload per sink, got keep=%d drop=%d", len(keepItems), len(dropItems)) } @@ -515,7 +1130,7 @@ func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { if err != nil { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -563,11 +1178,14 @@ func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ - Label: "真实数据源", + Label: "BOS 增量扫描", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, - Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: datasetRoot}}, + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "LOCAL"}, + {Key: "SOURCE_PATH", Value: datasetRoot}, + }, }, Position: domain.Position{X: 0, Y: 0}, }, @@ -595,7 +1213,7 @@ func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { if err != nil { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -607,12 +1225,13 @@ func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { t.Fatalf("get run: %v", err) } summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "lerobot-summary.json") - preview, _ := summary.Data["preview"].(map[string]any) + summaryPayload := mustArtifactPayload(t, summary) + preview, _ := summaryPayload["preview"].(map[string]any) if preview["format"] != "lerobot" { t.Fatalf("expected lerobot preview, got %#v", preview) } - if summary.Data["datasetType"] != "lerobot" { - t.Fatalf("expected datasetType lerobot, got %#v", summary.Data) + if summaryPayload["datasetType"] != "lerobot" { + t.Fatalf("expected datasetType lerobot, got %#v", summaryPayload) } }) @@ -637,11 +1256,14 @@ func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ - Label: "真实数据源", + Label: "BOS 增量扫描", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, - Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}}, + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "LOCAL"}, + {Key: "SOURCE_PATH", Value: dbPath}, + }, }, Position: domain.Position{X: 0, Y: 0}, }, @@ -670,7 +1292,7 @@ func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { if err != nil { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -682,7 +1304,8 @@ func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { t.Fatalf("get run: %v", err) } summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "rosbag-summary.json") - preview, _ := summary.Data["preview"].(map[string]any) + summaryPayload := mustArtifactPayload(t, summary) + preview, _ := summaryPayload["preview"].(map[string]any) if preview["format"] != "rosbag2" { t.Fatalf("expected rosbag2 preview, got %#v", preview) } @@ -693,6 +1316,723 @@ func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { }) } +func TestRuntimeProcessesXPlatformPipeline(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + datasetRoot := createXPlatformDataset(t) + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "XPlatform 项目", Description: "x-platform"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "XPlatform 处理流水线", + Description: "scan -> validate -> quality -> filter -> slice -> annotate -> package -> sample", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "LOCAL"}, + {Key: "SOURCE_PATH", Value: datasetRoot}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "index", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "X 平台数据集索引", + Category: "DATASET", + Description: "index dataset", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "dataset", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 220, Y: 0}, + }, + { + ID: "validate", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "采集目录校验", + Category: "DATASET", + Description: "validate dataset", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "validated", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 440, Y: 0}, + }, + { + ID: "quality", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "视频质检", + Category: "VIDEO", + Description: "quality", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "quality", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{ + {Key: "MIN_CLARITY", Value: "0.8"}, + {Key: "MAX_SHAKE", Value: "0.1"}, + {Key: "MAX_HAND_OCCLUSION", Value: "0.1"}, + }, + }, + Position: domain.Position{X: 660, Y: 0}, + }, + { + ID: "filter", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "数据过滤", + Category: "VIDEO", + Description: "filter dataset", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "filtered", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{ + {Key: "DATE_START", Value: "2026-02-01"}, + {Key: "DATE_END", Value: "2026-04-05"}, + {Key: "PLACE", Value: "蓟门里小区"}, + {Key: "TYPE", Value: "家庭"}, + {Key: "MIN_CLARITY", Value: "0.8"}, + {Key: "MAX_SHAKE", Value: "0.1"}, + {Key: "MAX_HAND_OCCLUSION", Value: "0.1"}, + }, + }, + Position: domain.Position{X: 880, Y: 0}, + }, + { + ID: "slice", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "视频切片", + Category: "VIDEO", + Description: "slice video", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "clips", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "SEGMENT_DURATION_SECONDS", Value: "2"}}, + }, + Position: domain.Position{X: 1100, Y: 0}, + }, + { + ID: "annotate", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "视频标注", + Category: "VIDEO", + Description: "annotate clips", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "annotations", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "FRAME_INTERVAL_SECONDS", Value: "0.5"}}, + }, + Position: domain.Position{X: 1320, Y: 0}, + }, + { + ID: "package", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "成品打包", + Category: "VIDEO", + Description: "package results", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "packages", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 1540, Y: 0}, + }, + { + ID: "sample", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "抽检采样", + Category: "QA", + Description: "sample packages", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "samples", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{ + {Key: "RULES_JSON", Value: `[{"id":"clarity-check","field":"quality.clarity","operator":"lt","value":1,"samplePercent":100},{"id":"confidence-check","field":"annotation.minConfidence","operator":"lt","value":1,"samplePercent":100}]`}, + }, + }, + Position: domain.Position{X: 1760, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "source-index", Source: "source", SourceHandle: "out-1", Target: "index", TargetHandle: "in-1"}, + {ID: "index-validate", Source: "index", SourceHandle: "out-1", Target: "validate", TargetHandle: "in-1"}, + {ID: "validate-quality", Source: "validate", SourceHandle: "out-1", Target: "quality", TargetHandle: "in-1"}, + {ID: "quality-filter", Source: "quality", SourceHandle: "out-1", Target: "filter", TargetHandle: "in-1"}, + {ID: "filter-slice", Source: "filter", SourceHandle: "out-1", Target: "slice", TargetHandle: "in-1"}, + {ID: "slice-annotate", Source: "slice", SourceHandle: "out-1", Target: "annotate", TargetHandle: "in-1"}, + {ID: "annotate-package", Source: "annotate", SourceHandle: "out-1", Target: "package", TargetHandle: "in-1"}, + {ID: "package-sample", Source: "package", SourceHandle: "out-1", Target: "sample", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + + completed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if completed == nil || completed.Status != domain.RunStatusCompleted { + t.Fatalf("expected completed run, got %#v", completed) + } + + indexArtifact := mustArtifact(t, mustNodeExecution(t, completed, "index"), "dataset-index.json") + indexPayload := mustArtifactPayload(t, indexArtifact) + if indexPayload["collectionCount"] != float64(2) { + t.Fatalf("expected collectionCount=2, got %#v", indexPayload) + } + if indexPayload["videoCount"] != float64(3) { + t.Fatalf("expected videoCount=3, got %#v", indexPayload) + } + + validateArtifact := mustArtifact(t, mustNodeExecution(t, completed, "validate"), "dataset-validation.json") + validatePayload := mustArtifactPayload(t, validateArtifact) + if validatePayload["validCollectionCount"] != float64(2) { + t.Fatalf("expected validCollectionCount=2, got %#v", validatePayload) + } + + qualityArtifact := mustArtifact(t, mustNodeExecution(t, completed, "quality"), "quality-report.json") + qualityPayload := mustArtifactPayload(t, qualityArtifact) + if qualityPayload["videoCount"] != float64(3) { + t.Fatalf("expected quality videoCount=3, got %#v", qualityPayload) + } + if qualityPayload["passedCount"] != float64(2) { + t.Fatalf("expected passedCount=2, got %#v", qualityPayload) + } + + filterArtifact := mustArtifact(t, mustNodeExecution(t, completed, "filter"), "filtered-videos.json") + filterPayload := mustArtifactPayload(t, filterArtifact) + if filterPayload["count"] != float64(1) { + t.Fatalf("expected filter count=1, got %#v", filterPayload) + } + + sliceExecution := mustNodeExecution(t, completed, "slice") + clipsArtifact := mustArtifact(t, sliceExecution, "clips-manifest.json") + clipsPayload := mustArtifactPayload(t, clipsArtifact) + if clipsPayload["clipCount"] != float64(2) { + t.Fatalf("expected clipCount=2, got %#v", clipsPayload) + } + clipVideos := 0 + for _, artifact := range sliceExecution.Artifacts { + if strings.HasSuffix(artifact.Name, ".mp4") { + clipVideos++ + } + } + if clipVideos != 2 { + t.Fatalf("expected 2 clip videos, got %d", clipVideos) + } + + annotateExecution := mustNodeExecution(t, completed, "annotate") + annotationsArtifact := mustArtifact(t, annotateExecution, "annotations-manifest.json") + annotationsPayload := mustArtifactPayload(t, annotationsArtifact) + if annotationsPayload["annotationCount"] != float64(2) { + t.Fatalf("expected annotationCount=2, got %#v", annotationsPayload) + } + + packageExecution := mustNodeExecution(t, completed, "package") + packagesArtifact := mustArtifact(t, packageExecution, "packages-manifest.json") + packagesPayload := mustArtifactPayload(t, packagesArtifact) + if packagesPayload["packageCount"] != float64(2) { + t.Fatalf("expected packageCount=2, got %#v", packagesPayload) + } + packageFiles := 0 + for _, artifact := range packageExecution.Artifacts { + if strings.Contains(artifact.Path, "/packages/") && (strings.HasSuffix(artifact.Name, ".json") || strings.HasSuffix(artifact.Name, ".mp4")) { + packageFiles++ + } + } + if packageFiles < 8 { + t.Fatalf("expected packaged files to be emitted, got %d artifacts", packageFiles) + } + + sampleArtifact := mustArtifact(t, mustNodeExecution(t, completed, "sample"), "audit-samples.json") + samplePayload := mustArtifactPayload(t, sampleArtifact) + if samplePayload["sampledCount"] != float64(2) { + t.Fatalf("expected sampledCount=2, got %#v", samplePayload) + } +} + +func TestRuntimeCommitsIncrementalSourceStateOnlyOnSuccessfulRuns(t *testing.T) { + ctx := context.Background() + + t.Run("commits state after successful run and next run only emits new files", func(t *testing.T) { + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "增量项目", Description: "incremental"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + sourceRoot := t.TempDir() + writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "增量源工作流", + Description: "incremental source", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "LOCAL"}, + {Key: "SOURCE_PATH", Value: sourceRoot}, + {Key: "SCAN_MODE", Value: "INCREMENTAL"}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + runOne, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create first run: %v", err) + } + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process first run: %v", err) + } + + completedOne, err := dataStore.GetRun(ctx, runOne.ID) + if err != nil { + t.Fatalf("get first run: %v", err) + } + manifestOne := mustArtifact(t, mustNodeExecution(t, completedOne, "source"), "source-manifest.json") + manifestOnePayload := mustArtifactPayload(t, manifestOne) + if manifestOnePayload["newCount"] != float64(1) { + t.Fatalf("expected newCount=1 on first run, got %#v", manifestOnePayload) + } + + state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestOnePayload["sourceFingerprint"].(string))) + if err != nil { + t.Fatalf("get source state: %v", err) + } + if state == nil || len(state.SeenFingerprints) != 1 { + t.Fatalf("expected committed source state after first run, got %#v", state) + } + + writeFile(t, filepath.Join(sourceRoot, "batch-b", "meta.json"), `{"project":"B"}`) + runTwo, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled) + if err != nil { + t.Fatalf("create second run: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process second run: %v", err) + } + + completedTwo, err := dataStore.GetRun(ctx, runTwo.ID) + if err != nil { + t.Fatalf("get second run: %v", err) + } + manifestTwo := mustArtifact(t, mustNodeExecution(t, completedTwo, "source"), "source-manifest.json") + manifestTwoPayload := mustArtifactPayload(t, manifestTwo) + if manifestTwoPayload["listedCount"] != float64(2) { + t.Fatalf("expected listedCount=2 on second run, got %#v", manifestTwoPayload) + } + if manifestTwoPayload["newCount"] != float64(1) { + t.Fatalf("expected newCount=1 on second run, got %#v", manifestTwoPayload) + } + entries, ok := manifestTwoPayload["entries"].([]any) + if !ok || len(entries) != 1 { + t.Fatalf("expected one emitted entry on second run, got %#v", manifestTwoPayload["entries"]) + } + entry, ok := entries[0].(map[string]any) + if !ok || !strings.Contains(fmt.Sprint(entry["path"]), "batch-b/meta.json") { + t.Fatalf("expected second run to emit only newly added file, got %#v", entries[0]) + } + + state, err = dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestTwoPayload["sourceFingerprint"].(string))) + if err != nil { + t.Fatalf("get source state after second run: %v", err) + } + if state == nil || len(state.SeenFingerprints) != 2 { + t.Fatalf("expected 2 seen fingerprints after second run, got %#v", state) + } + if state.LastNewCount != 1 || state.LastListedCount != 2 { + t.Fatalf("unexpected source state stats after second run: %#v", state) + } + }) + + t.Run("directory incremental mode emits entire newly discovered directories", func(t *testing.T) { + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "目录增量项目", Description: "incremental-directory"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + sourceRoot := t.TempDir() + writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) + writeFile(t, filepath.Join(sourceRoot, "batch-a", "DJI_001", "clip-a.mp4"), "a") + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "目录增量源工作流", + Description: "incremental directory source", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "LOCAL"}, + {Key: "SOURCE_PATH", Value: sourceRoot}, + {Key: "SCAN_MODE", Value: "INCREMENTAL"}, + {Key: "INCREMENTAL_UNIT", Value: "DIRECTORY"}, + {Key: "INCREMENTAL_DIRECTORY_DEPTH", Value: "1"}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + runOne, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create first run: %v", err) + } + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process first run: %v", err) + } + + completedOne, err := dataStore.GetRun(ctx, runOne.ID) + if err != nil { + t.Fatalf("get first run: %v", err) + } + manifestOne := mustArtifact(t, mustNodeExecution(t, completedOne, "source"), "source-manifest.json") + manifestOnePayload := mustArtifactPayload(t, manifestOne) + if manifestOnePayload["newCount"] != float64(2) { + t.Fatalf("expected first run to emit full batch-a directory, got %#v", manifestOnePayload["newCount"]) + } + if manifestOnePayload["incrementalUnit"] != "DIRECTORY" { + t.Fatalf("expected directory incremental unit, got %#v", manifestOnePayload["incrementalUnit"]) + } + + writeFile(t, filepath.Join(sourceRoot, "batch-a", "DJI_001", "clip-b.mp4"), "b") + writeFile(t, filepath.Join(sourceRoot, "batch-b", "meta.json"), `{"project":"B"}`) + writeFile(t, filepath.Join(sourceRoot, "batch-b", "DJI_001", "clip-c.mp4"), "c") + runTwo, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled) + if err != nil { + t.Fatalf("create second run: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process second run: %v", err) + } + + completedTwo, err := dataStore.GetRun(ctx, runTwo.ID) + if err != nil { + t.Fatalf("get second run: %v", err) + } + manifestTwo := mustArtifact(t, mustNodeExecution(t, completedTwo, "source"), "source-manifest.json") + manifestTwoPayload := mustArtifactPayload(t, manifestTwo) + if manifestTwoPayload["listedCount"] != float64(5) { + t.Fatalf("expected listedCount=5 on second run, got %#v", manifestTwoPayload["listedCount"]) + } + if manifestTwoPayload["newCount"] != float64(2) { + t.Fatalf("expected newCount=2 on second run, got %#v", manifestTwoPayload["newCount"]) + } + entries, ok := manifestTwoPayload["entries"].([]any) + if !ok || len(entries) != 2 { + t.Fatalf("expected two emitted entries for new directory, got %#v", manifestTwoPayload["entries"]) + } + for _, rawEntry := range entries { + entry, ok := rawEntry.(map[string]any) + if !ok { + t.Fatalf("unexpected entry payload: %#v", rawEntry) + } + if !strings.Contains(fmt.Sprint(entry["path"]), "batch-b/") { + t.Fatalf("expected second run to emit only newly discovered directory contents, got %#v", entry["path"]) + } + } + + state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestTwoPayload["sourceFingerprint"].(string))) + if err != nil { + t.Fatalf("get source state after second run: %v", err) + } + if state == nil || len(state.SeenFingerprints) != 2 { + t.Fatalf("expected two seen directory keys after second run, got %#v", state) + } + if state.IncrementalUnit != "DIRECTORY" || state.DirectoryDepth != 1 { + t.Fatalf("expected committed directory incremental state, got %#v", state) + } + }) + + t.Run("does not commit state when downstream run fails", func(t *testing.T) { + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "失败增量项目", Description: "incremental-fail"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + sourceRoot := t.TempDir() + writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "增量失败工作流", + Description: "incremental fail", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "LOCAL"}, + {Key: "SOURCE_PATH", Value: sourceRoot}, + {Key: "SCAN_MODE", Value: "INCREMENTAL"}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "fail", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "fail", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "echo boom >&2; exit 1"}}, + }, + Position: domain.Position{X: 240, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "source-fail", Source: "source", SourceHandle: "out-1", Target: "fail", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process failed run: %v", err) + } + + failedRun, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get failed run: %v", err) + } + if failedRun == nil || failedRun.Status != domain.RunStatusFailed { + t.Fatalf("expected failed run, got %#v", failedRun) + } + manifest := mustArtifact(t, mustNodeExecution(t, failedRun, "source"), "source-manifest.json") + manifestPayload := mustArtifactPayload(t, manifest) + state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestPayload["sourceFingerprint"].(string))) + if err != nil { + t.Fatalf("get source state after failed run: %v", err) + } + if state != nil { + t.Fatalf("expected no committed source state after failed run, got %#v", state) + } + }) +} + +func TestPrepareExecutionNodeResolvesStorageConnectionSecrets(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 主连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://gz.bcebos.com", + AccessKeyID: "ak-runtime-1234", + SecretAccessKey: "sk-runtime-5678", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + + workflow := &domain.WorkflowRecord{ + ID: "workflow-scan", + Kind: domain.WorkflowKindScan, + ScanConfig: &domain.ScanConfig{ + DataSource: "BOS", + ConnectionID: connection.ID, + Bucket: "datastorage", + Prefix: "algo/X-platform-test", + ScanMode: "INCREMENTAL", + }, + } + node := domain.WorkflowNode{ + ID: "node-bos-scan-runtime", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "BOS 增量扫描", + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "BOS"}, + }, + }, + } + + prepared, err := runtime.prepareExecutionNode(ctx, workflow, node) + if err != nil { + t.Fatalf("prepare execution node: %v", err) + } + params := nodeHyperparamsMap(prepared) + if params["BOS_ENDPOINT"] != "https://gz.bcebos.com" { + t.Fatalf("expected resolved endpoint, got %q", params["BOS_ENDPOINT"]) + } + if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-1234" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-5678" { + t.Fatalf("expected resolved bos secrets, got %#v", params) + } + if params["SOURCE_PATH"] != "bos://datastorage/algo/X-platform-test" { + t.Fatalf("expected source path from bucket/prefix, got %q", params["SOURCE_PATH"]) + } +} + +func TestPrepareExecutionNodeResolvesNodeLevelStorageConnectionSecrets(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "BOS 节点连接", + Provider: domain.StorageProviderBOS, + Endpoint: "https://su.bcebos.com", + AccessKeyID: "ak-node-1234", + SecretAccessKey: "sk-node-5678", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + + node := domain.WorkflowNode{ + ID: "node-oss-source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "OSS 数据源", + ConnectionID: connection.ID, + Bucket: "datastorage", + Prefix: "factory/output", + Hyperparams: []domain.HyperParam{ + {Key: "SOURCE_PATH", Value: ""}, + {Key: "DATA_SOURCE", Value: "BOS"}, + }, + }, + } + + prepared, err := runtime.prepareExecutionNode(ctx, nil, node) + if err != nil { + t.Fatalf("prepare execution node: %v", err) + } + params := nodeHyperparamsMap(prepared) + if params["DATA_SOURCE"] != "BOS" { + t.Fatalf("expected BOS data source, got %q", params["DATA_SOURCE"]) + } + if params["BOS_ENDPOINT"] != "https://su.bcebos.com" { + t.Fatalf("expected resolved endpoint, got %q", params["BOS_ENDPOINT"]) + } + if params["BOS_ACCESS_KEY_ID"] != "ak-node-1234" || params["BOS_SECRET_ACCESS_KEY"] != "sk-node-5678" { + t.Fatalf("expected resolved node bos secrets, got %#v", params) + } + if params["SOURCE_PATH"] != "bos://datastorage/factory/output" { + t.Fatalf("expected source path from node bucket/prefix, got %q", params["SOURCE_PATH"]) + } +} + +func TestPrepareExecutionNodeResolvesMongoConnectionSecrets(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + connection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{ + Name: "业务 MongoDB", + URI: "mongodb://business-mongo:27017", + Database: "business_data", + }) + if err != nil { + t.Fatalf("create mongo connection: %v", err) + } + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + + node := domain.WorkflowNode{ + ID: "node-mongo-write", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "MongoDB 写入", + Hyperparams: []domain.HyperParam{ + {Key: "MONGO_CONNECTION_ID", Value: connection.ID}, + {Key: "MONGO_COLLECTION", Value: "qualityReports"}, + }, + }, + } + + prepared, err := runtime.prepareExecutionNode(ctx, nil, node) + if err != nil { + t.Fatalf("prepare execution node: %v", err) + } + params := nodeHyperparamsMap(prepared) + if params["MONGO_URI"] != "mongodb://business-mongo:27017" { + t.Fatalf("expected resolved mongo uri, got %q", params["MONGO_URI"]) + } + if params["MONGO_DB"] != "business_data" { + t.Fatalf("expected resolved mongo db, got %q", params["MONGO_DB"]) + } + if params["MONGO_TARGET"] != "business" { + t.Fatalf("expected mongo target business, got %q", params["MONGO_TARGET"]) + } +} + func TestRuntimeProcessAvailableDispatchesUpToGlobalLimit(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -749,7 +2089,7 @@ func TestRuntimeProcessAvailableDispatchesUpToGlobalLimit(t *testing.T) { t.Fatalf("create run B: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -849,7 +2189,7 @@ func TestRuntimeProcessAvailableRespectsWorkflowConcurrencyOverride(t *testing.T t.Fatalf("create free run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -887,6 +2227,427 @@ func TestRuntimeProcessAvailableRespectsWorkflowConcurrencyOverride(t *testing.T waitForRunStatus(t, ctx, dataStore, runLimitedB.ID, domain.RunStatusCompleted, 8*time.Second) } +func TestRuntimeProcessAvailableSkipsStreamingWorkflowWithMissingProject(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "有效项目", Description: "valid"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + validWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "正常工作流", + Description: "valid workflow", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "command", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "command", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > result.json"}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + }) + if err != nil { + t.Fatalf("create valid workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, validWorkflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create queued run: %v", err) + } + + orphanWorkflow := domain.WorkflowRecord{ + ID: "workflow-orphan-streaming", + ProjectID: "project-missing", + Name: "孤儿增量工作流", + Description: "orphan streaming workflow", + Type: domain.WorkflowTypeStreaming, + Schedule: &domain.WorkflowSchedule{Interval: 30}, + Nodes: []domain.WorkflowNode{}, + Edges: []domain.WorkflowEdge{}, + Status: domain.WorkflowStatusDraft, + CreatedAt: nowISO(), + UpdatedAt: nowISO(), + } + if err := dataStore.UpsertWorkflow(ctx, orphanWorkflow); err != nil { + t.Fatalf("upsert orphan workflow: %v", err) + } + + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available should skip orphan workflow instead of failing: %v", err) + } + + waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) +} + +func TestShouldEnqueueScheduledRunEveryHoursUsesAnchorTime(t *testing.T) { + workflow := domain.WorkflowRecord{ + ID: "workflow-hours", + Type: domain.WorkflowTypeStreaming, + Schedule: &domain.WorkflowSchedule{ + Mode: domain.WorkflowScheduleModeEveryHours, + IntervalHours: 4, + AnchorAt: "2026-04-21T10:15:00Z", + }, + } + + if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 15, 0, 0, time.UTC)) { + t.Fatalf("expected first hourly window to enqueue at anchor time") + } + + workflow.LastRunAt = "2026-04-21T10:15:00Z" + if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 14, 14, 59, 0, time.UTC)) { + t.Fatalf("expected hourly schedule to wait until full interval elapsed") + } + if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 14, 15, 0, 0, time.UTC)) { + t.Fatalf("expected hourly schedule to enqueue on the next anchored slot") + } +} + +func TestShouldEnqueueScheduledRunEveryDaysAtFixedTimeUsesAnchorDate(t *testing.T) { + workflow := domain.WorkflowRecord{ + ID: "workflow-days", + Type: domain.WorkflowTypeStreaming, + Schedule: &domain.WorkflowSchedule{ + Mode: domain.WorkflowScheduleModeEveryDaysAtTime, + DayInterval: 2, + TimeOfDay: "18:30", + Timezone: "Asia/Shanghai", + AnchorAt: "2026-04-21T09:00:00Z", + }, + } + + if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 29, 59, 0, time.UTC)) { + t.Fatalf("expected daily schedule to wait until configured local time") + } + if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 30, 0, 0, time.UTC)) { + t.Fatalf("expected daily schedule to enqueue at configured local time") + } + + workflow.LastRunAt = "2026-04-21T10:30:00Z" + if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 23, 10, 29, 59, 0, time.UTC)) { + t.Fatalf("expected next two-day slot to remain pending before local HH:mm") + } + if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 23, 10, 30, 0, 0, time.UTC)) { + t.Fatalf("expected next two-day slot to enqueue at configured local HH:mm") + } +} + +func TestRuntimeProcessRunResumesQueuedPartialRunFromCompletedDependencies(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "resume"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "恢复工作流", + Description: "resume", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "真实数据源", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"items\":[{\"name\":\"sample\"}],\"count\":1}"}}, + }, + }, + { + ID: "transform", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "JSON 映射重命名", + Category: "DATA", + Description: "transform", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "FROM_KEY", Value: "items"}, {Key: "TO_KEY", Value: "entries"}}, + }, + }, + { + ID: "upload", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "sink", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > done.json"}}, + }, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "source-transform", Source: "source", SourceHandle: "out-1", Target: "transform", TargetHandle: "in-1"}, + {ID: "transform-upload", Source: "transform", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + run.Status = domain.RunStatusQueued + run.CurrentNodeID = "" + run.NodeExecutions = []domain.NodeExecutionRecord{ + { + NodeID: "source", + NodeLabel: "真实数据源", + Status: domain.NodeExecutionStatusCompleted, + StartedAt: nowISO(), + EndedAt: nowISO(), + Logs: []domain.LogEntryRecord{}, + Artifacts: []domain.ExecutionArtifact{{ID: "artifact-source", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "payload.json", Type: domain.PortTypeJSON, Path: filepath.Join(t.TempDir(), "payload.json"), Data: map[string]any{"items": []any{map[string]any{"name": "sample"}}, "count": 1}}}, + DataRefs: []domain.DataRefRecord{}, + ResourceSeries: []domain.ResourceSample{}, + }, + } + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + + completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) + if execution := mustNodeExecution(t, completed, "transform"); execution.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected transform to complete after resume, got %s", execution.Status) + } + if execution := mustNodeExecution(t, completed, "upload"); execution.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected upload to complete after resume, got %s", execution.Status) + } +} + +func TestRuntimeReconcileActiveRunsRequeuesClaimedRunWithoutJobs(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "僵尸运行恢复", + Description: "recover", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"ok\":true}"}}}}}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + run.Status = domain.RunStatusRunning + run.StartedAt = time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano) + run.UpdatedAt = run.StartedAt + run.NodeExecutions = []domain.NodeExecutionRecord{} + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) + if err := runtime.reconcileActiveRuns(ctx); err != nil { + t.Fatalf("reconcile active runs: %v", err) + } + + recovered, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get recovered run: %v", err) + } + if recovered == nil || recovered.Status != domain.RunStatusQueued { + t.Fatalf("expected run to be re-queued, got %#v", recovered) + } + if recovered.StartedAt != "" { + t.Fatalf("expected startedAt cleared after requeue, got %q", recovered.StartedAt) + } +} + +func TestRuntimeReconcileActiveRunsRecoversSucceededNodeAndResumesDownstream(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover success"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "成功节点恢复", + Description: "recover success", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "真实数据源", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"items\":[{\"name\":\"sample\"}],\"count\":1}"}}, + }, + }, + { + ID: "annotate", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "视频标注", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "annotations", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "MODEL", Value: "stub"}}, + }, + }, + { + ID: "upload", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > done.json"}}, + }, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "source-annotate", Source: "source", SourceHandle: "out-1", Target: "annotate", TargetHandle: "in-1"}, + {ID: "annotate-upload", Source: "annotate", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + sourceArtifact := domain.ExecutionArtifact{ + ID: "artifact-source", + RunID: run.ID, + NodeID: "source", + PortID: "out-1", + Name: "payload.json", + Type: domain.PortTypeJSON, + Path: filepath.Join(t.TempDir(), "payload.json"), + Data: map[string]any{"items": []any{map[string]any{"name": "sample"}}, "count": 1}, + } + run.Status = domain.RunStatusRunning + run.CurrentNodeID = "annotate" + run.StartedAt = time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano) + run.UpdatedAt = run.StartedAt + run.NodeExecutions = []domain.NodeExecutionRecord{ + { + NodeID: "source", + NodeLabel: "真实数据源", + Status: domain.NodeExecutionStatusCompleted, + StartedAt: run.StartedAt, + EndedAt: run.StartedAt, + Logs: []domain.LogEntryRecord{}, + Artifacts: []domain.ExecutionArtifact{sourceArtifact}, + DataRefs: []domain.DataRefRecord{}, + ResourceSeries: []domain.ResourceSample{}, + }, + { + NodeID: "annotate", + NodeLabel: "视频标注", + Status: domain.NodeExecutionStatusRunning, + StartedAt: run.StartedAt, + Logs: []domain.LogEntryRecord{{Timestamp: run.StartedAt, Level: "INFO", Message: "Starting node execution", Operator: "视频标注"}}, + Artifacts: []domain.ExecutionArtifact{}, + DataRefs: []domain.DataRefRecord{}, + ResourceSeries: []domain.ResourceSample{}, + }, + } + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + launcher := testLauncher{ + launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { + env := append([]string{}, os.Environ()...) + keys := make([]string, 0, len(input.Env)) + for key := range input.Env { + keys = append(keys, key) + } + sort.Strings(keys) + env = append(env, "RUN_ID="+input.RunID, "WORKFLOW_ID="+input.WorkflowID, "NODE_ID="+input.NodeID, "NODE_LABEL="+input.NodeLabel) + for _, key := range keys { + env = append(env, key+"="+input.Env[key]) + } + cmd := exec.CommandContext(ctx, "python3", operatorRunnerPath(t)) + cmd.Env = env + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("%w: %s", err, string(output)) + } + return &k8s.LaunchResult{ + Reference: &k8s.JobReference{JobName: fmt.Sprintf("%s-%s", input.RunID, input.NodeID), Namespace: "droboticflow"}, + Output: string(output), + }, nil + }, + inspect: func(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) { + if runID == run.ID && nodeID == "annotate" { + return &k8s.JobInspection{ + Reference: &k8s.JobReference{JobName: "job-annotate", Namespace: "droboticflow"}, + State: k8s.JobStateSucceeded, + Output: "DROBOTICFLOW_RESULT={\"artifacts\":[{\"id\":\"annotation-1\",\"runId\":\"" + run.ID + "\",\"nodeId\":\"annotate\",\"portId\":\"out-1\",\"name\":\"annotation.json\",\"type\":\"json\",\"size\":\"1 KB\",\"path\":\"/tmp/annotation.json\",\"data\":{\"items\":[{\"label\":\"walking\"}],\"count\":1}}],\"logs\":[],\"resourceSeries\":[]}", + }, nil + } + return &k8s.JobInspection{State: k8s.JobStateMissing}, nil + }, + } + + runtime := newTestRuntimeWithLauncher(t, dataStore, launcher) + if err := runtime.reconcileActiveRuns(ctx); err != nil { + t.Fatalf("reconcile active runs: %v", err) + } + recovered, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get recovered run: %v", err) + } + if recovered == nil || recovered.Status != domain.RunStatusQueued { + t.Fatalf("expected recovered run queued, got %#v", recovered) + } + if execution := mustNodeExecution(t, recovered, "annotate"); execution.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected annotate completed after recovery, got %s", execution.Status) + } + + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once after recovery: %v", err) + } + completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) + if execution := mustNodeExecution(t, completed, "upload"); execution.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected downstream upload completed, got %s", execution.Status) + } +} + func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -919,7 +2680,10 @@ func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) { Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, - Hyperparams: []domain.HyperParam{{Key: "SOURCE_TEXT", Value: "parallel"}}, + Hyperparams: []domain.HyperParam{ + {Key: "DATA_SOURCE", Value: "INLINE"}, + {Key: "SOURCE_TEXT", Value: "parallel"}, + }, }, Position: domain.Position{X: 0, Y: 0}, }, @@ -978,7 +2742,7 @@ func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -999,6 +2763,156 @@ func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) { } } +func TestRuntimeRerunSubtreeOnlyExecutesSelectedSubgraph(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "子链路重跑项目", Description: "rerun subtree"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "子链路重跑工作流", + Description: "rerun subtree", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "start-a", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "FLOW", + Description: "branch a start", + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"a"}' > branch-a.json`}}, + }, + }, + { + ID: "downstream-a", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "FLOW", + Description: "branch a end", + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"a-downstream"}' > branch-a-downstream.json`}}, + }, + }, + { + ID: "start-b", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "FLOW", + Description: "branch b start", + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"b"}' > branch-b.json`}}, + }, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "a-to-downstream", Source: "start-a", Target: "downstream-a"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + run.Trigger = domain.RunTriggerRerunSubtree + run.RestartFromNodeID = "start-a" + run.NodeExecutions = []domain.NodeExecutionRecord{} + run.Status = domain.RunStatusQueued + run.UpdatedAt = nowISO() + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert rerun subtree run: %v", err) + } + + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available: %v", err) + } + + completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) + if execution := mustNodeExecution(t, completed, "start-a"); execution.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected start-a to complete, got %s", execution.Status) + } + if execution := mustNodeExecution(t, completed, "downstream-a"); execution.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected downstream-a to complete, got %s", execution.Status) + } + for _, execution := range completed.NodeExecutions { + if execution.NodeID == "start-b" { + t.Fatalf("expected unrelated branch start-b not to execute, got %#v", execution) + } + } +} + +func TestProcessRunKeepsClaimedRunRunningWhenExistingNodeExecutionIsActive(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + runtime := &Runtime{ + store: dataStore, + operatorRunnerImage: "runner:latest", + workspaceRoot: "/workspace", + launcher: testLauncher{}, + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: "", + Name: "resume-active-node", + Description: "resume active execution", + Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source"}}}, + Edges: []domain.WorkflowEdge{}, + Type: domain.WorkflowTypeBatch, + Kind: domain.WorkflowKindStandard, + Schedule: nil, + ScanConfig: nil, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + run.Status = domain.RunStatusQueued + run.CurrentNodeID = "source" + run.NodeExecutions = []domain.NodeExecutionRecord{ + { + NodeID: "source", + NodeLabel: "Source", + Status: domain.NodeExecutionStatusRunning, + StartedAt: nowISO(), + }, + } + run.UpdatedAt = nowISO() + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert run: %v", err) + } + + processed, err := runtime.ProcessOnce(ctx) + if err != nil { + t.Fatalf("process once: %v", err) + } + if processed == nil { + t.Fatalf("expected processed run") + } + if processed.Status != domain.RunStatusRunning { + t.Fatalf("expected running status, got %s", processed.Status) + } + if processed.CurrentNodeID != "source" { + t.Fatalf("expected currentNodeID=source, got %s", processed.CurrentNodeID) + } + if execution := mustNodeExecution(t, processed, "source"); execution.Status != domain.NodeExecutionStatusRunning { + t.Fatalf("expected source execution running, got %s", execution.Status) + } +} + func TestRuntimeProcessRunFailsWhenWorkflowTimeoutReached(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -1038,7 +2952,7 @@ func TestRuntimeProcessRunFailsWhenWorkflowTimeoutReached(t *testing.T) { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -1113,7 +3027,7 @@ func TestRuntimeProcessRunAutoRetriesFailedWorkflows(t *testing.T) { t.Fatalf("create initial run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -1226,7 +3140,7 @@ func TestRuntimeProcessRunFailsWhenNodeTimeoutReached(t *testing.T) { t.Fatalf("create run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } @@ -1290,7 +3204,7 @@ func TestRuntimeProcessAvailableHonorsRetryBackoff(t *testing.T) { t.Fatalf("create initial run: %v", err) } - runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } diff --git a/k8s/business-mongodb.test.yaml b/k8s/business-mongodb.test.yaml new file mode 100644 index 00000000..356a9032 --- /dev/null +++ b/k8s/business-mongodb.test.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Service +metadata: + name: business-mongodb + namespace: droboticflow + labels: + app.kubernetes.io/name: business-mongodb + app.kubernetes.io/component: business-data +spec: + selector: + app.kubernetes.io/name: business-mongodb + ports: + - name: mongo + port: 27017 + targetPort: mongo +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: business-mongodb + namespace: droboticflow + labels: + app.kubernetes.io/name: business-mongodb + app.kubernetes.io/component: business-data +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: business-mongodb + template: + metadata: + labels: + app.kubernetes.io/name: business-mongodb + app.kubernetes.io/component: business-data + spec: + imagePullSecrets: + - name: droboticflow-registry + containers: + - name: mongo + image: ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-mongo:7-amd64 + imagePullPolicy: IfNotPresent + ports: + - name: mongo + containerPort: 27017 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: data + mountPath: /data/db + volumes: + - name: data + emptyDir: {} diff --git a/k8s/droboticflow.template.yaml b/k8s/droboticflow.template.yaml index 4c7af64f..ec49e0d8 100644 --- a/k8s/droboticflow.template.yaml +++ b/k8s/droboticflow.template.yaml @@ -53,122 +53,13 @@ data: WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__" K8S_NAMESPACE: "__NAMESPACE__" K8S_IMAGE_PULL_SECRET_NAME: "__K8S_IMAGE_PULL_SECRET_NAME__" + WORKSPACE_PVC_NAME: "__WORKSPACE_PVC_NAME__" + WORKSPACE_MOUNT_PATH: "__WORKSPACE_MOUNT_PATH__" OPERATOR_RUNNER_IMAGE: "__OPERATOR_RUNNER_IMAGE__" AI_API_BASE_URL: "__AI_API_BASE_URL__" AI_MODEL: "__AI_MODEL__" --- apiVersion: v1 -kind: ConfigMap -metadata: - name: mongodb-init - namespace: __NAMESPACE__ -data: - init-app-user.sh: | - #!/bin/sh - set -eu - mongosh --quiet </dev/null | base64 --decode 2>/dev/null || true } -kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$K8S_NAMESPACE" +validate_external_mongo_uri() { + local uri="$1" -if kubectl -n "$K8S_NAMESPACE" get statefulset mongodb >/dev/null 2>&1; then - kubectl -n "$K8S_NAMESPACE" delete statefulset mongodb --ignore-not-found --wait=true - kubectl -n "$K8S_NAMESPACE" delete pvc mongodb-data-mongodb-0 --ignore-not-found + if [[ -z "$uri" ]]; then + echo "ERROR: MONGO_URI is required and must point to an external MongoDB instance. Export MONGO_URI before running scripts/deploy-k8s.sh." >&2 + exit 1 + fi + + shopt -s nocasematch + if [[ "$uri" =~ (^|[@/])mongodb:27017([/?]|$) ]] || \ + [[ "$uri" =~ localhost ]] || \ + [[ "$uri" =~ 127\.0\.0\.1 ]] || \ + [[ "$uri" =~ \.svc([:/?]|$) ]]; then + echo "ERROR: MONGO_URI must reference an external MongoDB instance. Cluster-local or localhost targets are not allowed in the main Kubernetes deployment path." >&2 + exit 1 + fi + shopt -u nocasematch +} + +if [[ -z "$MONGO_URI" ]] && kubectl --request-timeout=5s -n "$K8S_NAMESPACE" get secret droboticflow-mongo-secret >/dev/null 2>&1; then + MONGO_URI="$(read_secret_value droboticflow-mongo-secret MONGO_URI)" fi +validate_external_mongo_uri "$MONGO_URI" + +kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$K8S_NAMESPACE" + if [[ "$CREATE_REGISTRY_SECRET" == "true" ]]; then : "${REGISTRY_USERNAME:?REGISTRY_USERNAME is required when CREATE_REGISTRY_SECRET=true}" : "${REGISTRY_PASSWORD:?REGISTRY_PASSWORD is required when CREATE_REGISTRY_SECRET=true}" - kubectl -n "$K8S_NAMESPACE" create secret docker-registry droboticflow-registry \ + kubectl -n "$K8S_NAMESPACE" create secret docker-registry "$K8S_IMAGE_PULL_SECRET_NAME" \ --docker-server="$REGISTRY_SERVER" \ --docker-username="$REGISTRY_USERNAME" \ --docker-password="$REGISTRY_PASSWORD" \ @@ -55,27 +86,12 @@ if [[ "$CREATE_REGISTRY_SECRET" == "true" ]]; then -o yaml | kubectl apply -f - fi -if kubectl -n "$K8S_NAMESPACE" get secret droboticflow-mongo-secret >/dev/null 2>&1; then - MONGO_ROOT_USERNAME="${MONGO_ROOT_USERNAME:-$(read_secret_value droboticflow-mongo-secret MONGO_ROOT_USERNAME)}" - MONGO_ROOT_PASSWORD="${MONGO_ROOT_PASSWORD:-$(read_secret_value droboticflow-mongo-secret MONGO_ROOT_PASSWORD)}" - MONGO_APP_USERNAME="${MONGO_APP_USERNAME:-$(read_secret_value droboticflow-mongo-secret MONGO_APP_USERNAME)}" - MONGO_APP_PASSWORD="${MONGO_APP_PASSWORD:-$(read_secret_value droboticflow-mongo-secret MONGO_APP_PASSWORD)}" -fi - if kubectl -n "$K8S_NAMESPACE" get secret droboticflow-ai-secret >/dev/null 2>&1; then AI_API_KEY="${AI_API_KEY:-$(read_secret_value droboticflow-ai-secret AI_API_KEY)}" fi -MONGO_ROOT_USERNAME="${MONGO_ROOT_USERNAME:-root}" -MONGO_ROOT_PASSWORD="${MONGO_ROOT_PASSWORD:-$(openssl rand -hex 16)}" -MONGO_APP_USERNAME="${MONGO_APP_USERNAME:-droboticflow}" -MONGO_APP_PASSWORD="${MONGO_APP_PASSWORD:-$(openssl rand -hex 16)}" - kubectl -n "$K8S_NAMESPACE" create secret generic droboticflow-mongo-secret \ - --from-literal=MONGO_ROOT_USERNAME="$MONGO_ROOT_USERNAME" \ - --from-literal=MONGO_ROOT_PASSWORD="$MONGO_ROOT_PASSWORD" \ - --from-literal=MONGO_APP_USERNAME="$MONGO_APP_USERNAME" \ - --from-literal=MONGO_APP_PASSWORD="$MONGO_APP_PASSWORD" \ + --from-literal=MONGO_URI="$MONGO_URI" \ --dry-run=client \ -o yaml | kubectl apply -f - @@ -91,6 +107,8 @@ env \ MONGO_CONNECT_RETRY_DELAY_MS="$MONGO_CONNECT_RETRY_DELAY_MS" \ WORKER_POLL_INTERVAL_MS="$WORKER_POLL_INTERVAL_MS" \ K8S_IMAGE_PULL_SECRET_NAME="$K8S_IMAGE_PULL_SECRET_NAME" \ + WORKSPACE_PVC_NAME="$WORKSPACE_PVC_NAME" \ + WORKSPACE_MOUNT_PATH="$WORKSPACE_MOUNT_PATH" \ API_NODE_PORT="$API_NODE_PORT" \ API_REPLICAS="$API_REPLICAS" \ WORKER_REPLICAS="$WORKER_REPLICAS" \ @@ -100,10 +118,8 @@ env \ API_IMAGE="$API_IMAGE" \ WORKER_IMAGE="$WORKER_IMAGE" \ OPERATOR_RUNNER_IMAGE="$OPERATOR_RUNNER_IMAGE" \ - MONGO_IMAGE="$MONGO_IMAGE" \ "$ROOT_DIR/scripts/render-manifests.sh" | kubectl apply -f - -kubectl -n "$K8S_NAMESPACE" rollout status deployment/mongodb --timeout=300s kubectl -n "$K8S_NAMESPACE" rollout status deployment/droboticflow-api --timeout=300s kubectl -n "$K8S_NAMESPACE" rollout status deployment/droboticflow-worker --timeout=300s diff --git a/scripts/generate_operator_docs.py b/scripts/generate_operator_docs.py new file mode 100644 index 00000000..16cc1919 --- /dev/null +++ b/scripts/generate_operator_docs.py @@ -0,0 +1,614 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import urllib.request +from collections import defaultdict +from pathlib import Path +from typing import Any + + +DEFAULT_API_BASE_URL = "http://127.0.0.1:3001" +DEFAULT_BUILTIN_OUTPUT_DIR = "docs/operators" +DEFAULT_CUSTOM_OUTPUT_DIR = "docs/custom-operators/instances" + +BUILTIN_HANDLER_HINTS: dict[str, tuple[str, str]] = { + "operator-builtin-real-source": ("backend/operator-runner/main.py", "builtin_source"), + "operator-builtin-bos-incremental-scan": ("backend/operator-runner/main.py", "builtin_source"), + "operator-builtin-model-inference": ("backend/operator-runner/main.py", "builtin_generic"), + "operator-builtin-merge": ("backend/operator-runner/main.py", "builtin_merge"), + "operator-builtin-upload": ("backend/operator-runner/main.py", "builtin_upload"), + "operator-builtin-human-review": ("backend/operator-runner/main.py", "builtin_condition_router"), + "operator-builtin-directory-scan": ("backend/operator-runner/main.py", "builtin_directory_scan"), + "operator-builtin-csv-to-json": ("backend/operator-runner/main.py", "builtin_csv_to_json"), + "operator-builtin-json-to-csv": ("backend/operator-runner/main.py", "builtin_json_to_csv"), + "operator-builtin-json-field-extract": ("backend/operator-runner/main.py", "builtin_json_extract"), + "operator-builtin-json-filter": ("backend/operator-runner/main.py", "builtin_json_filter"), + "operator-builtin-json-map-rename": ("backend/operator-runner/main.py", "builtin_json_map"), + "operator-builtin-template-render": ("backend/operator-runner/main.py", "builtin_text_template"), + "operator-builtin-conditional-route": ("backend/operator-runner/main.py", "builtin_condition_router"), + "operator-builtin-xplatform-dataset-index": ("backend/operator-runner/main.py", "builtin_xplatform_index"), + "operator-builtin-xplatform-dataset-validate": ("backend/operator-runner/main.py", "builtin_collection_validate"), + "operator-builtin-video-quality": ("backend/operator-runner/main.py", "builtin_video_quality"), + "operator-builtin-video-filter": ("backend/operator-runner/main.py", "builtin_dataset_filter"), + "operator-builtin-video-slice": ("backend/operator-runner/main.py", "builtin_video_slicing"), + "operator-builtin-video-annotate": ("backend/operator-runner/main.py", "builtin_video_annotation"), + "operator-builtin-package": ("backend/operator-runner/main.py", "builtin_result_package"), + "operator-builtin-audit-sample": ("backend/operator-runner/main.py", "builtin_audit_sampling"), + "operator-builtin-mongodb-write": ("backend/operator-runner/main.py", "builtin_mongodb_write"), + "operator-builtin-lerobot-summary": ("backend/operator-runner/main.py", "builtin_lerobot_summary"), + "operator-builtin-rosbag-summary": ("backend/operator-runner/main.py", "builtin_rosbag_summary"), + "operator-builtin-shell-command": ("backend/operator-runner/main.py", "builtin_generic"), +} + +PORT_TYPE_HINTS = { + "json": "适合结构化清单、规则结果、摘要对象等可直接序列化的数据。", + "image": "适合单帧图片、截图、标注图或图片引用。", + "video": "适合视频文件或 clip 引用,通常不会以内联字节返回。", + "tensor": "适合模型中间结果或高维数组,通常需要外置存储。", + "file": "适合 CSV、压缩包、文档、配置文件等任意文件产物。", +} + +TRANSMISSION_HINTS = { + "bus": "通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。", + "oss": "更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。", + "shm": "偏向节点内共享内存/本地工作目录语义,要求上下游执行环境紧耦合。", + "stream": "偏向流式消费场景,需要上下游在时序和消费方式上兼容。", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate detailed markdown manuals for builtin and custom operators") + parser.add_argument("--api-base-url", default=DEFAULT_API_BASE_URL) + parser.add_argument("--input-json", default="", help="Read operator payload from a local JSON file instead of calling /api/operators") + parser.add_argument("--builtin-output-dir", default=DEFAULT_BUILTIN_OUTPUT_DIR) + parser.add_argument("--custom-output-dir", default=DEFAULT_CUSTOM_OUTPUT_DIR) + return parser.parse_args() + + +def request_json(url: str) -> Any: + with urllib.request.urlopen(url, timeout=60) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def load_operator_items(args: argparse.Namespace) -> list[dict[str, Any]]: + if args.input_json: + payload = json.loads(Path(args.input_json).read_text(encoding="utf-8")) + else: + api_base_url = args.api_base_url.rstrip("/") + payload = request_json(f"{api_base_url}/api/operators") + if isinstance(payload, dict) and isinstance(payload.get("items"), list): + items = payload.get("items", []) + elif isinstance(payload, list): + items = payload + elif isinstance(payload, dict) and str(payload.get("name", "")).strip(): + items = [payload] + else: + raise SystemExit("operator payload must be one of: {items:[...]}, [...], or a single operator object") + return [item for item in items if isinstance(item, dict)] + + +def slugify(value: str) -> str: + cleaned = re.sub(r"[\s/]+", "-", value.strip()) + cleaned = re.sub(r"[^0-9A-Za-z\-_.\u4e00-\u9fff]+", "-", cleaned) + cleaned = re.sub(r"-{2,}", "-", cleaned).strip("-") + return cleaned or "operator" + + +def choose_builtin_records(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for item in items: + if str(item.get("type", "")).strip() != "builtin": + continue + name = str(item.get("name", "")).strip() + if name: + grouped[name].append(item) + chosen: list[dict[str, Any]] = [] + for name in sorted(grouped): + records = grouped[name] + records.sort(key=lambda item: (0 if str(item.get("type")) == "builtin" else 1, str(item.get("id", "")))) + chosen.append(records[0]) + return chosen + + +def choose_custom_records(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + chosen = [item for item in items if str(item.get("type", "")).strip() == "custom" and str(item.get("name", "")).strip()] + chosen.sort(key=lambda item: (str(item.get("category", "")).strip(), str(item.get("name", "")).strip(), str(item.get("id", "")).strip())) + return chosen + + +def clean_markdown_dir(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + for item in path.glob("*.md"): + item.unlink() + + +def stringify(value: Any) -> str: + if value is None: + return "" + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + +def render_port_detail(port: dict[str, Any], direction: str) -> str: + port_id = str(port.get("id", "")).strip() or "-" + name = str(port.get("name", "")).strip() or "-" + port_type = str(port.get("type", "")).strip() or "-" + transmission = str(port.get("transmission", "")).strip() or "-" + description = str(port.get("description", "")).strip() or "未额外说明。" + type_hint = PORT_TYPE_HINTS.get(port_type, "表示该端口承载的是平台允许的对应数据形态。") + transmission_hint = TRANSMISSION_HINTS.get(transmission, "表示该端口采用平台约定的传输语义。") + return ( + f"- `{port_id}` / **{name}** / `{port_type}` / `{transmission}`:{description} " + f"作为{direction}端口时,{type_hint} {transmission_hint}" + ) + + +def render_hyperparam_detail(param: dict[str, Any]) -> str: + key = str(param.get("key", "")).strip() or "-" + default_value = stringify(param.get("value", "")).strip() + default_text = f"`{default_value}`" if default_value else "空字符串或由工作流显式填写" + description = str(param.get("description", "")).strip() or "未提供描述。" + ptype = str(param.get("type", "")).strip() or "string" + required_hint = "通常建议在工作流中显式配置。" if not default_value else "可直接使用默认值,也可在节点级覆盖。" + return f"- `{key}`:类型 `{ptype}`,默认值 {default_text}。{description} {required_hint}" + + +def render_skill_definition_block(operator: dict[str, Any]) -> list[str]: + skill_definition = str(operator.get("skillDefinition", "")).strip() + if not skill_definition: + return [] + return ["## 平台语义", "", skill_definition, ""] + + +def operator_positioning(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + category = str(operator.get("category", "")).strip() or "OTHER" + operator_type = str(operator.get("type", "")).strip() or "unknown" + lines = [f"- 该算子属于 `{category}` 类,当前登记类型是 `{operator_type}`。"] + if operator_type == "builtin": + lines.append("- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。") + else: + lines.append("- 该算子属于自定义算子,平台只负责登记元数据与调度,实际业务逻辑由镜像、脚本或命令实现。") + if name in {"真实数据源", "BOS 增量扫描", "目录扫描"}: + lines.append("- 它更偏向工作流入口,职责是把外部世界的数据引用整理成平台统一 artifact。") + if name in {"结果上传", "MongoDB 写入"}: + lines.append("- 它更偏向工作流出口,职责是把平台内部产物同步到外部对象存储或业务数据库。") + if name in {"视频质检", "视频切片", "视频标注", "成品打包"}: + lines.append("- 它位于视频主处理链路中,对共享 PVC、本地文件路径和 manifest 契约依赖较强。") + if name in {"JSON 字段提取", "JSON 过滤", "JSON 映射重命名", "JSON 转 CSV", "CSV 转 JSON", "条件分流", "数据合并", "文本模板渲染"}: + lines.append("- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。") + return lines + + +def operator_runtime_contract(operator: dict[str, Any]) -> list[str]: + operator_id = str(operator.get("id", "")).strip() + operator_type = str(operator.get("type", "")).strip() + lines = [ + "- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。", + "- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。", + "- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。", + ] + if operator_type == "builtin": + source_file, handler_name = BUILTIN_HANDLER_HINTS.get(operator_id, ("backend/operator-runner/main.py", "builtin_generic")) + lines.append(f"- 当前实现入口通常可追到 `{source_file}` 中的 `{handler_name}(...)`。") + else: + lines.append("- 当前类型是自定义算子,建议在元数据中通过 `skillDefinition` 明确输入要求、超参数含义、产物结构和失败语义。") + if operator_id in {"operator-builtin-real-source", "operator-builtin-bos-incremental-scan"}: + lines.append("- 该类源算子没有显式上游输入,主要依赖节点超参数和运行时注入的存储连接信息。") + if operator_id == "operator-builtin-upload": + lines.append("- 该类末端算子通常没有输出端口,验收重点是外部对象存储中的真实写入结果。") + return lines + + +def operator_boundaries(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + lines = [ + "- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。", + "- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。", + ] + if name == "结果上传": + lines.append("- 上传节点只应负责收集并落地结果,不应顺便做复杂转换;转换应在上传前完成。") + if name == "人工审核": + lines.append("- 审核节点的职责是等待外部决策并路由分支,不应继续承担后置业务处理。") + if name in {"Shell 命令执行", "模型推理"}: + lines.append("- 这类通用执行算子适合快速验证,但当逻辑稳定后应考虑沉淀为专用内建算子或受控自定义算子。") + return lines + + +def operator_preparation(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + lines = [ + "- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。", + "- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。", + "- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。", + ] + if name in {"BOS 增量扫描", "结果上传", "MongoDB 写入"}: + lines.append("- 需要预先准备连接配置、凭据和目标路径,优先通过平台连接配置而不是把密钥散落到节点超参数里。") + if name in {"视频质检", "视频切片", "视频标注", "成品打包", "Shell 命令执行"}: + lines.append("- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。") + if name == "人工审核": + lines.append("- 需要提前确认审核 API、审核页面或人工流程是否可用,否则工作流会停在等待状态。") + return lines + + +def operator_data_path(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + lines: list[str] = [] + if name in {"BOS 增量扫描", "真实数据源"}: + lines.extend( + [ + "- 源算子通常先输出轻量级 manifest,记录 `entries`、`catalogEntries`、`bos://` 路径或摘要信息,而不是直接下载整个数据集。", + "- 下游第一次真正读取文件内容时,才会通过 runner 的物化逻辑把远程对象缓存到共享 PVC 或本地工作目录。", + ] + ) + elif name == "结果上传": + lines.extend( + [ + "- 该节点读取上游的 `artifact.path`、`dataRef.payloadPath` 或本地文件目录,然后把真实文件上传到对象存储。", + "- 平台运行记录里通常只保留上传摘要、目标路径和状态,不重复存放完整文件内容。", + ] + ) + elif name in {"视频质检", "视频切片", "视频标注", "成品打包"}: + lines.extend( + [ + "- 视频类处理主要依赖共享 PVC 的本地路径执行,必要时会把 `bos://...` 输入缓存到 `_bos_cache` 或当前节点工作目录。", + "- 上下游之间传递的重点是 clip 路径、manifest、annotation JSON、package 目录等引用,而不是媒体字节本身。", + ] + ) + elif name == "MongoDB 写入": + lines.extend( + [ + "- 该节点主要消费结构化 JSON payload,并把整理后的文档写入目标 Mongo 集合。", + "- 平台主库只保留写入摘要、错误信息和执行状态,避免把已写入业务库的整批文档再次冗余保存。", + ] + ) + else: + lines.extend( + [ + "- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。", + "- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。", + ] + ) + return lines + + +def operator_combinations(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + lines: list[str] = [] + if name in {"BOS 增量扫描", "真实数据源"}: + lines.extend( + [ + "- 常接 `X 平台数据集索引`、`目录扫描`、`模型推理` 或任意自定义 JSON 处理节点,作为工作流入口。", + "- 当需要持续监听新增目录或新增对象时,应把它放在整条 DAG 的最上游,并让后续节点只消费本轮增量。 ", + ] + ) + elif name == "X 平台数据集索引": + lines.extend( + [ + "- 上游通常是 `BOS 增量扫描` 或 `真实数据源`,下游再接 `采集目录校验`、`数据过滤`、`视频质检` 等节点。", + "- 如果后续要做抽检或上传,建议把索引结果先标准化为结构化 manifest,再进入视频主链路。", + ] + ) + elif name == "采集目录校验": + lines.append("- 适合放在索引之后,用来把不完整目录尽早筛掉,减少后续视频处理成本。 ") + elif name == "视频质检": + lines.append("- 常位于 `视频过滤` 或 `数据过滤` 之前,为后续筛选提供清晰度、抖动度、遮挡率等量化依据。") + elif name == "视频切片": + lines.append("- 常接在视频筛选之后,并把 clip 结果交给 `视频标注`、`成品打包` 或单独的 `结果上传`。") + elif name == "视频标注": + lines.append("- 上游一般是 `视频切片`,下游则是 `成品打包`、`结果上传` 或人工质检支路。") + elif name == "成品打包": + lines.append("- 建议在打包后单独接一个上传节点;如果同时上传 clip、标注和 package,应该拆成多条上传支路。") + elif name == "MongoDB 写入": + lines.append("- 通常接在 JSON 提取、映射、模板渲染或清洗节点之后,确保入库文档已经满足目标库 schema。") + elif name == "人工审核": + lines.append("- 审核节点之后应明确分成 `approved` 和 `rejected` 两条支路,避免后续节点混收不同语义的数据。") + else: + lines.append("- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。") + return lines + + +def operator_examples(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + if name == "BOS 增量扫描": + return [ + "- 示例配置:`SOURCE_PATH=bos://wjzh-data-migration/wjzh-0330-Data-migration`,`SCAN_MODE=INCREMENTAL`,`INCREMENTAL_UNIT=DIRECTORY`。", + "- 典型输出:`source-manifest.json`,其中 `entries` 表示本轮增量对象,`catalogEntries` 表示本轮完整列举结果。", + ] + if name == "结果上传": + return [ + "- 典型输入:来自上游的 `packages-manifest.json`、`annotation.json`、`clip.mp4` 或任意 `dataRef.payloadPath`。", + "- 典型输出:`uploaded-results.json`,记录每个对象的目标路径、对象大小、状态和错误信息。", + ] + if name == "视频切片": + return [ + "- 典型输入:带有视频路径的 manifest 或上游过滤结果。", + "- 典型输出:`clips-manifest.json` 与多个 `*_clip001.mp4` 文件,clip 实体位于当前节点 `OUTPUT_DIR/clips/` 下。", + ] + if name == "视频标注": + return [ + "- 典型输入:来自切片节点的 clip 视频与 `clips-manifest.json`。", + "- 典型输出:`annotations-manifest.json` 与逐 clip 的标注 JSON 文件。", + ] + if name == "成品打包": + return [ + "- 典型输入:clip、annotation、quality、配置文件或传感器补充数据。", + "- 典型输出:`packages-manifest.json` 和每个成品包目录。 ", + ] + if name == "MongoDB 写入": + return [ + "- 典型输入:结构化 JSON 行列表、单文档对象或经过模板渲染后的入库 payload。", + "- 典型输出:写入摘要、命中集合名、成功数量、失败数量和错误原因。", + ] + return [ + "- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。", + "- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。", + ] + + +def operator_success_signals(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + lines = [ + "- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。", + "- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。", + ] + if name in {"结果上传", "MongoDB 写入"}: + lines.append("- 外部系统中存在真实副作用,例如对象已上传、文档已写入,而不只是平台内显示成功。") + if name == "人工审核": + lines.append("- 审核节点收到明确的 approve/reject 决策,并将正确的分支结果送达下游。") + if name in {"BOS 增量扫描", "真实数据源"}: + lines.append("- 输出 manifest 与真实源路径内容一致,没有漏扫、重扫或错误分类。") + return lines + + +def operator_failures(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + lines = [ + "- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。", + "- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。", + "- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。", + ] + if name in {"BOS 增量扫描", "真实数据源", "结果上传"}: + lines.append("- 对象存储异常时,先核对 `SOURCE_PATH`、bucket/prefix、endpoint、AK/SK 和目标对象是否真实存在。") + if name in {"视频质检", "视频切片", "视频标注", "成品打包"}: + lines.append("- 视频链路异常时,先在 runner 容器内用 `ffprobe`/`ffmpeg` 验证输入文件是否可读,再检查缓存路径和磁盘空间。") + if name == "MongoDB 写入": + lines.append("- 若写库失败,先确认目标集合不在保留名单中,再核对连接串、认证信息和文档结构是否可序列化。") + if name == "人工审核": + lines.append("- 节点停在等待状态通常不是故障,而是还没有收到人工决策。") + return lines + + +def operator_acceptance(operator: dict[str, Any]) -> list[str]: + name = str(operator.get("name", "")).strip() + lines = [ + "- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。", + "- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。", + "- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。", + ] + if name in {"BOS 增量扫描", "真实数据源"}: + lines.append("- 对照真实源目录核验 `entries`、目录数、对象数和增量语义是否正确。") + if name == "结果上传": + lines.append("- 到目标对象存储前缀核对真实对象、对象大小和命名规则,确认并非只在平台侧生成了摘要。") + if name == "MongoDB 写入": + lines.append("- 到目标 Mongo 集合查询最近写入数据,确认记录数量、关键字段和值类型都符合预期。") + return lines + + +def render_config_example(operator: dict[str, Any]) -> str: + params = {} + for item in operator.get("hyperparams") or []: + key = str(item.get("key", "")).strip() + if not key: + continue + raw_value = item.get("value", "") + params[key] = raw_value if raw_value != "" else "<按节点填写>" + example = { + "label": str(operator.get("name", "")).strip(), + "operatorId": str(operator.get("id", "")).strip(), + "type": str(operator.get("type", "")).strip(), + "category": str(operator.get("category", "")).strip(), + "hyperparams": params, + } + return json.dumps(example, ensure_ascii=False, indent=2) + + +def render_operator_doc(operator: dict[str, Any], related_index: str, related_docs: list[tuple[str, str]]) -> str: + name = str(operator.get("name", "")).strip() + description = str(operator.get("description", "")).strip() or "无描述" + operator_id = str(operator.get("id", "")).strip() + operator_type = str(operator.get("type", "")).strip() + category = str(operator.get("category", "")).strip() or "OTHER" + inputs = operator.get("inputs") or [] + outputs = operator.get("outputs") or [] + hyperparams = operator.get("hyperparams") or [] + + lines = [ + f"# {name}", + "", + f"- 算子 ID:`{operator_id}`", + f"- 类别:`{category}`", + f"- 类型:`{operator_type}`", + "", + "## 作用", + "", + description, + "", + "## 定位与职责边界", + "", + ] + lines.extend(operator_positioning(operator)) + lines.extend([""]) + lines.extend(render_skill_definition_block(operator)) + lines.extend(["## 运行时契约", ""]) + lines.extend(operator_runtime_contract(operator)) + lines.extend(["", "## 设计边界", ""]) + lines.extend(operator_boundaries(operator)) + lines.extend(["", "## 数据流示意", "", "```mermaid", "flowchart LR", f" A[上游输入或运行时注入] --> B[{name}]", " B --> C[artifact / stage manifest / 外部副作用]", "```", ""]) + lines.extend(["## 输入端口详解", ""]) + if inputs: + lines.extend([render_port_detail(port, "输入") for port in inputs]) + else: + lines.append("- 无显式输入端口。通常作为源算子使用,主要依赖节点超参数和运行时环境。") + lines.extend(["", "## 输出端口详解", ""]) + if outputs: + lines.extend([render_port_detail(port, "输出") for port in outputs]) + else: + lines.append("- 无显式输出端口。通常意味着该节点以落库、上传或触发外部副作用为主。") + lines.extend(["", "## 关键超参数", ""]) + if hyperparams: + lines.extend([render_hyperparam_detail(param) for param in hyperparams]) + else: + lines.append("- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。") + lines.extend(["", "## 执行前准备", ""]) + lines.extend(operator_preparation(operator)) + lines.extend(["", "## 数据通信与存储路径", ""]) + lines.extend(operator_data_path(operator)) + lines.extend(["", "## 典型上下游编排", ""]) + lines.extend(operator_combinations(operator)) + lines.extend(["", "## 节点配置示例", "", "```json", render_config_example(operator), "```", ""]) + lines.extend(["## 输入输出示例", ""]) + lines.extend(operator_examples(operator)) + lines.extend(["", "## 成功判定", ""]) + lines.extend(operator_success_signals(operator)) + lines.extend(["", "## 常见失败与排查", ""]) + lines.extend(operator_failures(operator)) + lines.extend(["", "## 验收检查清单", ""]) + lines.extend(operator_acceptance(operator)) + lines.extend(["", "## 相关文档", "", f"- [{category} 类索引]({related_index})"]) + lines.extend([f"- [{label}]({path})" for label, path in related_docs]) + lines.append("") + return "\n".join(lines) + + +def write_builtin_docs(items: list[dict[str, Any]], output_dir: Path) -> int: + clean_markdown_dir(output_dir) + grouped: dict[str, list[tuple[str, str]]] = defaultdict(list) + for operator in items: + name = str(operator.get("name", "")).strip() + file_name = f"{slugify(name)}.md" + (output_dir / file_name).write_text( + render_operator_doc( + operator, + "README.md", + [ + ("算子开发文档", "../operator-authoring.md"), + ("内建算子总手册", "../builtin-operators-manual.zh-CN.md"), + ], + ), + encoding="utf-8", + ) + grouped[str(operator.get("category", "")).strip() or "OTHER"].append((name, file_name)) + + readme_lines = [ + "# 内建算子分册", + "", + "当前目录按“每个内建算子一份 Markdown”组织,内容基于当前平台真实算子定义按算子名称去重生成。", + "", + f"- 当前收录数量:`{len(items)}`", + "- 文档定位:单算子详细手册,重点覆盖输入输出契约、运行时路径、配置示例、排障与验收。", + "- 配套总览:[`../builtin-operators-manual.zh-CN.md`](../builtin-operators-manual.zh-CN.md)", + "", + "## 使用建议", + "", + "- 先阅读总手册理解平台级规则,再进入本目录阅读单算子分册。", + "- 设计 DAG 时优先确认端口类型、artifact 语义、外部副作用和验收口径,而不是只看算子名称。", + "", + "## 分类索引", + "", + ] + for category in sorted(grouped): + readme_lines.append(f"### {category}") + readme_lines.append("") + for name, file_name in sorted(grouped[category], key=lambda item: item[0]): + readme_lines.append(f"- [{name}]({file_name})") + readme_lines.append("") + (output_dir / "README.md").write_text("\n".join(readme_lines), encoding="utf-8") + return len(items) + + +def write_custom_docs(items: list[dict[str, Any]], output_dir: Path) -> int: + clean_markdown_dir(output_dir) + grouped: dict[str, list[tuple[str, str, str]]] = defaultdict(list) + for operator in items: + name = str(operator.get("name", "")).strip() + operator_id = str(operator.get("id", "")).strip() + suffix = slugify(operator_id[-8:] if len(operator_id) >= 8 else operator_id or name) + file_name = f"{slugify(name)}-{suffix}.md" + (output_dir / file_name).write_text( + render_operator_doc( + operator, + "README.md", + [ + ("自定义算子总手册", "../../custom-operator-manual.zh-CN.md"), + ("自定义算子模板与示例", "../README.md"), + ], + ), + encoding="utf-8", + ) + grouped[str(operator.get("category", "")).strip() or "OTHER"].append((name, operator_id, file_name)) + + readme_lines = [ + "# 自定义算子实例分册", + "", + "当前目录专门存放从算子注册表导出的“实例级自定义算子 Markdown”。", + "", + f"- 当前实例数量:`{len(items)}`", + "- 配套总览:[`../README.md`](../README.md)", + "- 配套手册:[`../../custom-operator-manual.zh-CN.md`](../../custom-operator-manual.zh-CN.md)", + "", + ] + if not items: + readme_lines.extend( + [ + "## 当前状态", + "", + "当前导出的算子目录中没有 `custom` 类型实例,因此本目录暂时只有索引页。", + "", + "建议做法:", + "", + "- 先参考上一级目录中的模板与示例撰写自定义算子说明。", + "- 当环境中出现真实自定义算子后,再重新运行生成脚本为每个实例导出单独 Markdown。", + "", + ] + ) + else: + readme_lines.extend(["## 分类索引", ""]) + for category in sorted(grouped): + readme_lines.append(f"### {category}") + readme_lines.append("") + for name, operator_id, file_name in sorted(grouped[category], key=lambda item: item[0]): + readme_lines.append(f"- [{name}]({file_name}),实例 ID:`{operator_id}`") + readme_lines.append("") + (output_dir / "README.md").write_text("\n".join(readme_lines), encoding="utf-8") + return len(items) + + +def main() -> int: + args = parse_args() + items = load_operator_items(args) + builtin_items = choose_builtin_records(items) + custom_items = choose_custom_records(items) + builtin_count = write_builtin_docs(builtin_items, Path(args.builtin_output_dir)) + custom_count = write_custom_docs(custom_items, Path(args.custom_output_dir)) + print( + json.dumps( + { + "status": "ok", + "builtinCount": builtin_count, + "customCount": custom_count, + "builtinOutputDir": args.builtin_output_dir, + "customOutputDir": args.custom_output_dir, + }, + ensure_ascii=False, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/load-deploy-env.sh b/scripts/load-deploy-env.sh new file mode 100644 index 00000000..ef990819 --- /dev/null +++ b/scripts/load-deploy-env.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +if [[ -n "${DROBOTICFLOW_DEPLOY_ENV_LOADED:-}" ]]; then + return 0 +fi + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DEPLOY_ENV="${DEPLOY_ENV:-${1:-}}" + +if [[ -z "$DEPLOY_ENV" ]]; then + return 0 +fi + +PRESET_PATH="$ROOT_DIR/scripts/deploy-envs/$DEPLOY_ENV.env" +if [[ ! -f "$PRESET_PATH" ]]; then + echo "ERROR: unknown deployment environment '$DEPLOY_ENV' (expected $PRESET_PATH)" >&2 + return 1 +fi + +LOCAL_CONFIG_ROOT="${XDG_CONFIG_HOME:-$HOME/.config}/droboticflow" + +set -a +source "$PRESET_PATH" + +for candidate in \ + "$ROOT_DIR/.env.$DEPLOY_ENV.local" \ + "$ROOT_DIR/.env.$DEPLOY_ENV.secrets" \ + "$LOCAL_CONFIG_ROOT/deploy/$DEPLOY_ENV.env" \ + "$LOCAL_CONFIG_ROOT/deploy/$DEPLOY_ENV.secrets.env" +do + if [[ -f "$candidate" ]]; then + source "$candidate" + fi +done +set +a + +if [[ -z "${KUBECONFIG:-}" ]]; then + default_kubeconfig="$LOCAL_CONFIG_ROOT/kubeconfigs/$DEPLOY_ENV.config" + if [[ -f "$default_kubeconfig" ]]; then + export KUBECONFIG="$default_kubeconfig" + fi +fi + +export DEPLOY_ENV +export DROBOTICFLOW_DEPLOY_ENV_LOADED="$DEPLOY_ENV" diff --git a/scripts/object-storage-acceptance.mjs b/scripts/object-storage-acceptance.mjs new file mode 100644 index 00000000..b4043a0b --- /dev/null +++ b/scripts/object-storage-acceptance.mjs @@ -0,0 +1,190 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { chromium } from "playwright"; + +const BASE_URL = (process.env.BASE_URL || "http://114.111.24.223:30080").replace(/\/+$/, ""); + +const bucketName = "datastorage"; +const testPrefix = `object-storage-acceptance-${Date.now()}`; +const createdKey = `${testPrefix}/object-storage-test.json`; +const renamedKey = `${testPrefix}/object-storage-renamed.json`; +const uploadKey = `${testPrefix}/nested/object-upload.txt`; +const uploadPayload = "upload payload from object storage acceptance"; +const uploadFilePath = path.join(os.tmpdir(), "object-storage-upload.txt"); + +fs.writeFileSync(uploadFilePath, uploadPayload, "utf8"); + +async function requestJSON(input, init) { + const response = await fetch(input, init); + const text = await response.text(); + if (!response.ok) { + throw new Error(text || `Request failed: ${response.status}`); + } + return text ? JSON.parse(text) : null; +} + +async function ensureConnection() { + const connectionsPayload = await requestJSON(`${BASE_URL}/api/storage/connections`); + const items = connectionsPayload?.items || []; + const matched = items.find((item) => item.bucket === bucketName); + if (matched?.id) { + return matched; + } + const fallback = items[0]; + assert.ok(fallback?.id, "expected at least one storage connection"); + return requestJSON(`${BASE_URL}/api/storage/connections/${fallback.id}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: fallback.name, + description: fallback.description || "", + provider: "BOS", + endpoint: fallback.endpoint || "", + bucket: bucketName, + }), + }); +} + +async function cleanupArtifacts(connectionId) { + try { + await fetch( + `${BASE_URL}/api/object-storage/objects?connectionId=${encodeURIComponent(connectionId)}&key=${encodeURIComponent(testPrefix)}&recursive=true`, + { method: "DELETE" }, + ); + } catch {} +} + +async function waitForObjectRow(page, key) { + const locator = page.locator(`[data-object-key="${key}"]`); + await locator.waitFor({ state: "visible" }); + return locator; +} + +async function run() { + const connection = await ensureConnection(); + assert.ok(connection?.id, "expected an object storage connection"); + + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage(); + page.on("dialog", (dialog) => dialog.accept()); + + try { + await page.goto(`${BASE_URL}/object-storage`, { waitUntil: "networkidle" }); + await page.waitForSelector('[data-testid="object-storage-connection-select"]'); + await page.waitForFunction( + (id) => + Array.from( + document.querySelector('[data-testid="object-storage-connection-select"]')?.querySelectorAll("option") || [], + ).some((option) => option.value === id), + connection.id, + ); + await page.selectOption('[data-testid="object-storage-connection-select"]', connection.id); + await page.waitForSelector('[data-testid="bound-bucket-name"]'); + await page.waitForFunction( + (expected) => document.querySelector('[data-testid="bound-bucket-name"]')?.textContent?.includes(expected), + bucketName, + ); + + await page.click('[data-testid="object-create-draft-button"]'); + await page.fill('[data-testid="object-key-input"]', createdKey); + await page.fill('[data-testid="object-content-type-input"]', "application/json"); + await page.fill( + '[data-testid="object-editor"]', + JSON.stringify({ message: "hello object storage", source: "playwright" }, null, 2), + ); + await page.click('[data-testid="object-save-button"]'); + await waitForObjectRow(page, testPrefix); + await page.dblclick(`[data-object-key="${testPrefix}"]`); + await waitForObjectRow(page, createdKey); + await page.click(`[data-object-key="${createdKey}"]`); + await page.waitForFunction(() => { + const preview = document.querySelector("pre"); + return preview && preview.textContent?.includes("hello object storage"); + }); + + await page.fill('[data-testid="rename-key-input"]', renamedKey); + await page.click('[data-testid="object-rename-button"]'); + await waitForObjectRow(page, renamedKey); + + await page.fill('[data-testid="upload-key-input"]', uploadKey); + await page.setInputFiles('[data-testid="upload-file-input"]', uploadFilePath); + await page.click('[data-testid="object-upload-button"]'); + await waitForObjectRow(page, `${testPrefix}/nested`); + await page.dblclick(`[data-object-key="${testPrefix}/nested"]`); + await waitForObjectRow(page, uploadKey); + await page.click(`[data-object-key="${uploadKey}"]`); + await page.waitForFunction( + (text) => { + const preview = document.querySelector("pre"); + return preview && preview.textContent?.includes(text); + }, + uploadPayload, + ); + + await page.click('button:has-text("返回上级")'); + await waitForObjectRow(page, renamedKey); + await page.click(`[data-object-key="${renamedKey}"]`); + await page.click('[data-testid="object-delete-button"]'); + await page.waitForFunction( + (key) => !document.querySelector(`[data-object-key="${key}"]`), + renamedKey, + ); + + await page.click(`[data-object-key="${testPrefix}/nested"]`); + await page.click('[data-testid="object-delete-button"]'); + await page.waitForFunction( + (key) => !document.querySelector(`[data-object-key="${key}"]`), + `${testPrefix}/nested`, + ); + + await page.click('button:has-text("根目录")'); + await waitForObjectRow(page, "algo"); + await page.dblclick('[data-object-key="algo"]'); + await page.dblclick('[data-object-key="algo/X-platform-test"]'); + await page.dblclick('[data-object-key="algo/X-platform-test/raw"]'); + await page.dblclick( + '[data-object-key="algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43"]', + ); + await page.dblclick( + '[data-object-key="algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001"]', + ); + const videoKey = + "algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260318112834_0001_D.MP4"; + await page.click(`[data-object-key="${videoKey}"]`); + await page.waitForSelector("video"); + await page.waitForFunction(() => { + const element = document.querySelector("video"); + return element && element.readyState >= 2; + }); + + const overview = await requestJSON( + `${BASE_URL}/api/object-storage?connectionId=${encodeURIComponent(connection.id)}`, + ); + assert.equal(overview?.bucket, bucketName, "expected bound datastorage bucket in overview"); + + console.log( + JSON.stringify( + { + connectionId: connection.id, + bucket: bucketName, + testPrefix, + verifiedVideoKey: videoKey, + status: "ok", + }, + null, + 2, + ), + ); + } finally { + await cleanupArtifacts(connection.id); + await browser.close(); + } +} + +run().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/playwright-functional-check.mjs b/scripts/playwright-functional-check.mjs index 1251f24d..09812862 100644 --- a/scripts/playwright-functional-check.mjs +++ b/scripts/playwright-functional-check.mjs @@ -1,16 +1,22 @@ import { chromium } from "playwright"; +import fs from "node:fs"; +import path from "node:path"; const baseURL = (process.env.BASE_URL || process.argv[2] || "http://114.111.24.223:30080").replace(/\/+$/, ""); const apiBaseURL = `${baseURL}`; +const artifactDir = path.resolve("output/playwright"); +const videoDir = path.join(artifactDir, "video"); const suffix = Date.now().toString(36); - const names = { admin: `PW Admin ${suffix}`, adminEmail: `pw-admin-${suffix}@playwright.local`, extraUser: `PW Operator ${suffix}`, extraUserEmail: `pw-operator-${suffix}@playwright.local`, project: `PW Project ${suffix}`, + scanWorkflow: `PW Scan Workflow ${suffix}`, workflow: `PW Workflow ${suffix}`, + edgeRunningWorkflow: `PW Edge Running ${suffix}`, + edgeFailedWorkflow: `PW Edge Failed ${suffix}`, template: `PW Template ${suffix}`, projectTemplate: `PW Project Template ${suffix}`, auditProject: `PW Audit ${suffix}`, @@ -86,20 +92,33 @@ async function gotoWithRetry(page, url) { throw lastError; } +async function capture(page, name, options = {}) { + fs.mkdirSync(artifactDir, { recursive: true }); + const filePath = path.join(artifactDir, `${name}.png`); + await page.screenshot({ path: filePath, fullPage: true, ...options }); + return filePath; +} + async function clickWithRetry(page, locator) { let lastError; for (let attempt = 1; attempt <= 6; attempt += 1) { try { await locator.waitFor({ state: "visible" }); await locator.scrollIntoViewIfNeeded(); - const box = await locator.boundingBox(); - if (!box) { - throw new Error("Unable to resolve click target bounding box"); - } - await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); + await locator.click({ timeout: 2000 }); return; } catch (error) { lastError = error; + try { + const box = await locator.boundingBox(); + if (!box) { + throw new Error("Unable to resolve click target bounding box"); + } + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); + return; + } catch (mouseError) { + lastError = mouseError; + } if (attempt === 6) { break; } @@ -109,6 +128,36 @@ async function clickWithRetry(page, locator) { throw lastError; } +async function waitForWorkflowEditor(page) { + await page.waitForURL(/\/workflow\//); + await page.getByRole("button", { name: "工作流配置" }).waitFor(); + await page.locator('[data-testid="operator-library-count"]').waitFor(); +} + +async function ensureButtonState({ + page, + activeLocator, + inactiveLocator, + activate, +}) { + const isVisible = async (locator) => { + try { + return await locator.isVisible(); + } catch { + return false; + } + }; + + if (await isVisible(activeLocator)) { + return; + } + await activate(); + await activeLocator.waitFor(); + if (inactiveLocator && (await isVisible(inactiveLocator))) { + throw new Error("button state did not settle to the expected active label"); + } +} + async function cleanupTestData() { log("cleanup test data"); const users = (await api("/api/users")).items; @@ -119,6 +168,17 @@ async function cleanupTestData() { } const projects = (await api("/api/projects")).items; + for (const project of projects) { + const workflows = (await api(`/api/projects/${encodeURIComponent(project.id)}/workflows`)).items; + for (const workflow of workflows) { + if (String(workflow.name || "").startsWith("PW ")) { + await api(`/api/workflows/${encodeURIComponent(workflow.id)}`, { + method: "DELETE", + }); + } + } + } + for (const project of projects) { if (String(project.name || "").startsWith("PW ")) { await api(`/api/projects/${encodeURIComponent(project.id)}`, { method: "DELETE" }); @@ -138,6 +198,15 @@ async function cleanupTestData() { await api(`/api/operators/${encodeURIComponent(operator.id)}`, { method: "DELETE" }); } } + + const scanWorkflows = (await api("/api/scan-configs")).items; + for (const workflow of scanWorkflows) { + if (String(workflow.name || "").startsWith("PW ")) { + await api(`/api/scan-configs/${encodeURIComponent(workflow.id)}`, { + method: "DELETE", + }); + } + } } async function createAuditWorkflow(projectName, workflowName) { @@ -201,6 +270,36 @@ async function createAuditWorkflow(projectName, workflowName) { return { project, workflow, run }; } +async function createWorkflow(projectId, name, nodes, edges, type = "batch") { + return api(`/api/projects/${projectId}/workflows`, { + method: "POST", + body: JSON.stringify({ + name, + description: `${name} created by Playwright`, + type, + nodes, + edges, + }), + }); +} + +async function clickWorkflowEdge(page, label, fallbackIndex = 0) { + const labelledLocator = page.locator(`[aria-label="${label}"]`).first(); + const labelledCount = await labelledLocator.count().catch(() => 0); + if (labelledCount > 0) { + await clickWithRetry(page, labelledLocator); + return; + } + + const edgeLocator = page.locator(".react-flow__edge").nth(fallbackIndex); + await edgeLocator.waitFor({ state: "visible" }); + const box = await edgeLocator.boundingBox(); + if (!box) { + throw new Error(`Unable to resolve workflow edge ${label}`); + } + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); +} + async function fillLabeledInput(page, labelText, value) { const group = page.locator(`label:has-text("${labelText}")`).locator(".."); const input = group.locator("input, textarea, select").first(); @@ -210,10 +309,18 @@ async function fillLabeledInput(page, labelText, value) { async function main() { await cleanupTestData(); + fs.mkdirSync(videoDir, { recursive: true }); const browser = await chromium.launch({ headless: true }); - const context = await browser.newContext({ viewport: { width: 1440, height: 1100 } }); + const context = await browser.newContext({ + viewport: { width: 1440, height: 1100 }, + recordVideo: { + dir: videoDir, + size: { width: 1440, height: 1100 }, + }, + }); const page = await context.newPage(); + let recordedVideoPath = ""; try { log("open bootstrap"); @@ -228,11 +335,14 @@ async function main() { await page.locator("select").first().selectOption("admin"); await page.getByRole("button", { name: "创建并进入系统" }).click(); } - await page.getByRole("heading", { name: "DroboticFlow 仪表盘" }).waitFor(); + await page.getByRole("heading", { name: "Drobodata 仪表盘" }).waitFor(); + await capture(page, "01-dashboard"); log("settings save round-trip"); await page.getByRole("link", { name: "系统配置" }).click(); await page.getByRole("heading", { name: "系统配置" }).waitFor(); + await page.getByRole("button", { name: /保存配置/ }).waitFor(); + await capture(page, "02-settings"); const pollGroup = page.locator('label:has-text("轮询间隔 (ms)")').locator(".."); const pollInput = pollGroup.locator("input"); const originalPoll = Number(await pollInput.inputValue()); @@ -246,6 +356,7 @@ async function main() { log("operators create edit delete"); await page.getByRole("link", { name: "算子管理" }).click(); await page.getByRole("heading", { name: "算子管理" }).waitFor(); + await capture(page, "03-operators"); await page.getByRole("button", { name: "注册算子" }).click(); await page.getByPlaceholder("例如: 图像锐化").fill(`PW Operator ${suffix}`); await page.getByPlaceholder("描述该算子的功能与用途...").fill("Playwright created operator"); @@ -262,6 +373,53 @@ async function main() { await page.getByText("没有匹配的算子,请调整搜索条件。").waitFor(); await page.getByPlaceholder("搜索算子...").fill(""); + log("database browser overview and query"); + await page.getByRole("link", { name: "业务数据库" }).click(); + await page.getByRole("heading", { name: "业务数据库" }).waitFor(); + await page.getByText("管理用户自定义的业务 MongoDB 连接").waitFor(); + const connectionSelect = page + .locator('label:has-text("当前连接")') + .locator("..") + .locator("select") + .first(); + const optionCount = await connectionSelect.locator("option").count(); + if (optionCount > 1) { + await connectionSelect.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + } + await capture(page, "04-database-browser"); + + log("data assets load"); + await page.getByRole("link", { name: "数据资产" }).click(); + await page.getByRole("heading", { name: "数据资产" }).waitFor(); + await capture(page, "05-data-assets-source"); + const packageTab = page.getByRole("button", { name: /成品数据/ }).first(); + if (await packageTab.isVisible().catch(() => false)) { + await clickWithRetry(page, packageTab); + await page.waitForTimeout(800); + await capture(page, "06-data-assets-packages"); + } + + log("object storage overview"); + await page.getByRole("link", { name: "对象存储" }).click(); + await page.getByRole("heading", { name: "对象存储" }).waitFor(); + await page.waitForTimeout(1000); + await capture(page, "07-object-storage"); + + log("sampling page"); + await page.getByRole("link", { name: "抽检任务" }).click(); + await page.getByRole("heading", { name: "抽检任务" }).waitFor(); + await capture(page, "08-sampling"); + + log("legacy scan workflow route redirects back into project workflows"); + assert( + (await page.getByRole("link", { name: "扫描工作流" }).count()) === 0, + "sidebar should not expose a dedicated scan workflow entry", + ); + await gotoWithRetry(page, `${baseURL}/scan-configs`); + await page.waitForURL(/\/projects$/); + await page.getByRole("heading", { name: "项目管理" }).waitFor(); + log("projects create and open"); await page.getByRole("link", { name: "项目管理" }).click(); await page.getByRole("heading", { name: "项目管理" }).waitFor(); @@ -275,14 +433,232 @@ async function main() { const projectList = (await api("/api/projects")).items; const project = projectList.find((item) => item.name === names.project); assert(project, "created project not found via api"); + await gotoWithRetry(page, `${baseURL}/project/${project.id}`); + await capture(page, "09-project-detail"); + + log("project detail create scan-capable workflow through BOS operator"); + await page.getByRole("heading", { name: names.project }).waitFor(); + await page.getByRole("button", { name: "新建工作流" }).click(); + await page.getByText("默认先创建普通工作流").waitFor(); + await page.getByPlaceholder("例如: 实时障碍物检测").fill(names.scanWorkflow); + await page.getByRole("button", { name: "确认创建" }).click(); + await waitForWorkflowEditor(page); + await page.getByPlaceholder("搜索算子...").fill("BOS 增量扫描"); + const scanOperatorCard = page + .locator("div") + .filter({ hasText: "BOS 增量扫描" }) + .first(); + await scanOperatorCard.locator('button[title="点击添加到画布"]').click(); + const scanCanvasNode = page + .locator(".react-flow__node") + .filter({ hasText: "BOS 增量扫描" }) + .first(); + await scanCanvasNode.waitFor(); + await scanCanvasNode.dblclick(); + await page.getByText("扫描源配置").waitFor(); + await capture(page, "10-workflow-editor-scan"); + await page.locator('label:has-text("Bucket")').waitFor(); + await page.locator('label:has-text("Prefix")').waitFor(); + await page.getByRole("button", { name: "工作流配置" }).click(); + await page.getByText("手动执行").waitFor(); + await page.getByText("周期执行").waitFor(); + await page.getByRole("button", { name: "工作流配置" }).click(); + + const storageConnections = (await api("/api/storage/connections")).items || []; + if (storageConnections.length > 0) { + log("scan source operator favorite storage paths"); + const connectionId = storageConnections[0].id; + await page + .locator('label:has-text("已选连接")') + .locator("..") + .locator("select") + .first() + .selectOption(connectionId); + await page.waitForFunction(() => { + const labels = Array.from(document.querySelectorAll("label")); + const bucketLabel = labels.find((item) => item.textContent?.includes("Bucket")); + const select = bucketLabel?.parentElement?.querySelector("select"); + return ( + !!select && + Array.from(select.options).some((option) => option.value === "datastorage") + ); + }); + const bucketSelect = page + .locator('label:has-text("Bucket")') + .locator("..") + .locator("select") + .first(); + const prefixGroup = page.locator('label:has-text("Prefix")').locator(".."); + const prefixInput = prefixGroup.locator("input").first(); + await bucketSelect.selectOption("datastorage"); + await prefixInput.fill("algo/X-platform-test"); + await prefixInput.blur(); + await ensureButtonState({ + page, + activeLocator: page.getByRole("button", { name: "取消收藏当前路径" }).first(), + inactiveLocator: page.getByRole("button", { name: "收藏当前路径" }).first(), + activate: () => + clickWithRetry( + page, + page.getByRole("button", { name: "收藏当前路径" }).first(), + ), + }); + await page.getByText("常用路径").first().waitFor(); + await bucketSelect.selectOption(""); + await prefixInput.fill(""); + await prefixInput.blur(); + const browseButton = prefixGroup.locator('button[title="浏览前缀"]').first(); + const browseDisabled = await browseButton.evaluate((node) => + node.hasAttribute("disabled"), + ); + assert(!browseDisabled, "favorite storage path should keep OSS browse enabled"); + await browseButton.click(); + await page.getByRole("heading", { name: "OSS 资源浏览器" }).waitFor(); + await page.evaluate(() => { + const candidates = Array.from( + document.querySelectorAll('button[title="datastorage/algo/X-platform-test"]'), + ); + const target = candidates + .map((node) => ({ node, x: node.getBoundingClientRect().x })) + .sort((a, b) => a.x - b.x)[0]?.node; + if (!target) { + throw new Error("favorite storage path chip not found"); + } + target.click(); + }); + await page.waitForFunction(() => { + const labels = Array.from(document.querySelectorAll("label")); + const bucketLabel = labels.find((item) => item.textContent?.includes("Bucket")); + const prefixLabel = labels.find((item) => item.textContent?.includes("Prefix")); + const bucketSelect = bucketLabel?.parentElement?.querySelector("select"); + const prefixInput = prefixLabel?.parentElement?.querySelector("input"); + return ( + bucketSelect?.value === "datastorage" && + prefixInput?.value === "algo/X-platform-test" + ); + }); + await page.evaluate(() => { + const heading = Array.from(document.querySelectorAll("h3")).find((item) => + item.textContent?.includes("OSS 资源浏览器"), + ); + const container = heading?.closest(".relative"); + const closeButton = container?.querySelector("button"); + if (closeButton instanceof HTMLButtonElement) { + closeButton.click(); + } + }); + await page + .getByRole("heading", { name: "OSS 资源浏览器" }) + .waitFor({ state: "hidden" }); + const saveScanWorkflowRequest = page.waitForResponse((response) => + response.request().method() === "PUT" && + /\/api\/workflows\/[^/]+$/.test(response.url()) && + response.ok(), + ); + await page.getByRole("button", { name: "保存" }).click(); + await saveScanWorkflowRequest; + const savedScanWorkflowId = page.url().match(/\/workflow\/([^?]+)/)?.[1]; + assert(savedScanWorkflowId, `unable to resolve saved scan workflow id from ${page.url()}`); + const savedScanWorkflow = await api(`/api/workflows/${encodeURIComponent(savedScanWorkflowId)}`); + const savedScanSourceNode = (savedScanWorkflow.nodes || []).find( + (item) => item?.data?.label === "BOS 增量扫描", + ); + assert(savedScanSourceNode?.id, "saved scan workflow did not persist BOS 增量扫描 node id"); + + log("scan source operator favorite storage paths survive fresh browser context"); + const verifyContext = await browser.newContext({ + viewport: { width: 1440, height: 1100 }, + }); + const verifyPage = await verifyContext.newPage(); + try { + await gotoWithRetry(verifyPage, page.url()); + await verifyPage + .locator(".react-flow__node") + .filter({ hasText: "BOS 增量扫描" }) + .first() + .dblclick(); + await verifyPage.locator('label:has-text("Bucket")').waitFor(); + await verifyPage.getByText("我的常用路径").first().waitFor(); + await verifyPage + .getByRole("button", { name: "取消收藏当前路径" }) + .first() + .waitFor(); + } finally { + await verifyContext.close(); + } + + log("scan source operator state commit and reset"); + const scanWorkflowId = savedScanWorkflowId; + const scanSourceNodeId = savedScanSourceNode.id; + const scanRun = await api(`/api/workflows/${encodeURIComponent(scanWorkflowId)}/run`, { + method: "POST", + body: JSON.stringify({ trigger: "manual" }), + }); + await poll( + () => api(`/api/runs/${encodeURIComponent(scanRun.id)}`), + (run) => run.status === "completed", + "scan source workflow completion", + 180000, + 3000, + ); + await poll( + () => + api( + `/api/workflows/${encodeURIComponent(scanWorkflowId)}/nodes/${encodeURIComponent(scanSourceNodeId)}/source-state`, + ), + (state) => state?.current?.lastRunId === scanRun.id, + "scan source state commit", + 180000, + 3000, + ); + await page.reload({ waitUntil: "networkidle" }); + await page + .locator(".react-flow__node") + .filter({ hasText: "BOS 增量扫描" }) + .first() + .dblclick(); + await page.getByText("增量状态").waitFor(); + await page.getByText("最近写入 Run").waitFor(); + await page + .locator(".node-config-panel") + .getByText(scanRun.id, { exact: true }) + .first() + .waitFor(); + await clickWithRetry( + page, + page.getByRole("button", { name: "重置当前状态" }).first(), + ); + await poll( + () => + api( + `/api/workflows/${encodeURIComponent(scanWorkflowId)}/nodes/${encodeURIComponent(scanSourceNodeId)}/source-state`, + ), + (state) => !state?.current, + "scan source state reset", + 120000, + 2000, + ); + await page.getByText("当前配置下还没有已提交的增量状态").waitFor(); + } + await gotoWithRetry(page, `${baseURL}/project/${project.id}`); log("project detail create workflow"); await page.getByRole("heading", { name: names.project }).waitFor(); await page.getByRole("button", { name: "新建工作流" }).click(); + await page.getByText("默认先创建普通工作流").waitFor(); await page.getByPlaceholder("例如: 实时障碍物检测").fill(names.workflow); await page.getByRole("button", { name: "确认创建" }).click(); + await waitForWorkflowEditor(page); + await capture(page, "11-workflow-editor"); await page.getByRole("button", { name: "运行工作流" }).waitFor(); + await page.getByRole("button", { name: "工作流配置" }).click(); + await page.getByText("手动执行").waitFor(); + await page.getByText("周期执行").waitFor(); + await clickWithRetry(page, page.getByRole("button", { name: "周期执行" }).last()); + await page.getByText("调度间隔 (秒)").waitFor(); + await clickWithRetry(page, page.getByRole("button", { name: "手动执行" }).last()); + await page.getByRole("button", { name: "工作流配置" }).click(); const workflows = (await api(`/api/projects/${project.id}/workflows`)).items; const workflow = workflows.find((item) => item.name === names.workflow); @@ -354,7 +730,11 @@ async function main() { }); await page.reload({ waitUntil: "networkidle" }); - await page.getByText("真实数据源").dblclick(); + await page + .locator(".react-flow__node") + .filter({ hasText: "真实数据源" }) + .first() + .dblclick(); const sourceGroup = page.locator('label:has-text("SOURCE_TEXT")').locator("..").locator(".."); await sourceGroup.locator("input").fill(`playwright-ui-${suffix}`); await page.getByRole("button", { name: "保存" }).click(); @@ -388,6 +768,7 @@ async function main() { await page.getByRole("heading", { name: "执行监控" }).waitFor(); await page.getByPlaceholder("搜索运行 ID、工作流或项目...").fill(names.workflow); await page.getByText(names.workflow).waitFor(); + await capture(page, "12-monitoring"); const runRow = page.locator("tr").filter({ hasText: names.workflow }).first(); await runRow.hover(); await runRow.getByTitle("查看日志").click(); @@ -421,8 +802,221 @@ async function main() { 2000, ); + log("edge output panel for running, completed, and failed workflows"); + const edgeRunningWorkflow = await createWorkflow( + project.id, + names.edgeRunningWorkflow, + [ + { + id: "source-1", + type: "operator", + data: { + label: "真实数据源", + category: "IO", + description: "Edge output running source", + inputs: [], + outputs: [{ id: "out-1", name: "manifest", type: "json" }], + hyperparams: [ + { + key: "SOURCE_TEXT", + value: `edge-running-${suffix}`, + }, + ], + }, + position: { x: 80, y: 80 }, + }, + { + id: "shell-1", + type: "operator", + data: { + label: "Shell 命令执行", + category: "UTIL", + description: "Slow shell", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "result", type: "json" }], + hyperparams: [ + { + key: "COMMAND", + value: "sleep 8; printf '{\"ok\":true}' > result.json", + }, + ], + }, + position: { x: 380, y: 80 }, + }, + ], + [ + { + id: "source-shell", + source: "source-1", + sourceHandle: "out-1", + target: "shell-1", + targetHandle: "in-1", + }, + ], + ); + const edgeRunningRun = await api(`/api/workflows/${edgeRunningWorkflow.id}/run`, { + method: "POST", + body: JSON.stringify({ trigger: "manual" }), + }); + const edgeRunningRunDetail = await poll( + async () => api(`/api/runs/${edgeRunningRun.id}`), + (item) => + item.status === "running" && + item.nodeExecutions?.some( + (execution) => + execution.nodeId === "source-1" && + execution.status === "completed" && + execution.artifacts?.length > 0, + ) && + item.nodeExecutions?.some( + (execution) => + execution.nodeId === "shell-1" && execution.status === "running", + ), + `edge running workflow ${edgeRunningRun.id} to enter running state`, + 120000, + 1500, + ); + const edgeRunningArtifact = + edgeRunningRunDetail.nodeExecutions + ?.find((execution) => execution.nodeId === "source-1") + ?.artifacts?.find((artifact) => artifact.portId === "out-1") || + edgeRunningRunDetail.artifacts?.find( + (artifact) => + artifact.nodeId === "source-1" && artifact.portId === "out-1", + ); + assert(edgeRunningArtifact?.name, "running edge artifact not found"); + + await gotoWithRetry(page, `${baseURL}/workflow/${edgeRunningWorkflow.id}`); + await page.locator(".react-flow").waitFor(); + await page.getByText(`Run ${edgeRunningRun.id}`).waitFor(); + await clickWorkflowEdge(page, "真实数据源 到 Shell 命令执行 连线"); + await page + .locator(".edge-output-panel") + .getByText("连线输出", { exact: true }) + .waitFor(); + await page + .locator(".edge-output-panel") + .getByText("正在执行中", { exact: true }) + .waitFor(); + await page + .locator(".edge-output-panel") + .getByText(edgeRunningArtifact.name, { exact: true }) + .waitFor(); + + await poll( + async () => api(`/api/runs/${edgeRunningRun.id}`), + (item) => item.status === "completed", + `edge running workflow ${edgeRunningRun.id} completion`, + 120000, + 2000, + ); + await gotoWithRetry(page, `${baseURL}/workflow/${edgeRunningWorkflow.id}`); + await page.getByText(`Run ${edgeRunningRun.id}`).waitFor(); + await clickWorkflowEdge(page, "真实数据源 到 Shell 命令执行 连线"); + await page + .locator(".edge-output-panel") + .getByText("执行完成", { exact: true }) + .waitFor(); + await page + .locator(".edge-output-panel") + .getByText(edgeRunningArtifact.name, { exact: true }) + .waitFor(); + + const edgeFailedWorkflow = await createWorkflow( + project.id, + names.edgeFailedWorkflow, + [ + { + id: "source-1", + type: "operator", + data: { + label: "真实数据源", + category: "IO", + description: "Edge output failed source", + inputs: [], + outputs: [{ id: "out-1", name: "manifest", type: "json" }], + hyperparams: [ + { + key: "SOURCE_TEXT", + value: `edge-failed-${suffix}`, + }, + ], + }, + position: { x: 80, y: 80 }, + }, + { + id: "shell-1", + type: "operator", + data: { + label: "Shell 命令执行", + category: "UTIL", + description: "Failing shell", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "result", type: "json" }], + hyperparams: [ + { + key: "COMMAND", + value: "echo edge-preview-failed >&2; exit 1", + }, + ], + }, + position: { x: 380, y: 80 }, + }, + ], + [ + { + id: "source-shell", + source: "source-1", + sourceHandle: "out-1", + target: "shell-1", + targetHandle: "in-1", + }, + ], + ); + const edgeFailedRun = await api(`/api/workflows/${edgeFailedWorkflow.id}/run`, { + method: "POST", + body: JSON.stringify({ trigger: "manual" }), + }); + const edgeFailedRunDetail = await poll( + async () => api(`/api/runs/${edgeFailedRun.id}`), + (item) => + item.status === "failed" && + item.nodeExecutions?.some( + (execution) => + execution.nodeId === "source-1" && + execution.status === "completed" && + execution.artifacts?.length > 0, + ), + `edge failed workflow ${edgeFailedRun.id} failure`, + 120000, + 1500, + ); + const edgeFailedArtifact = + edgeFailedRunDetail.nodeExecutions + ?.find((execution) => execution.nodeId === "source-1") + ?.artifacts?.find((artifact) => artifact.portId === "out-1") || + edgeFailedRunDetail.artifacts?.find( + (artifact) => + artifact.nodeId === "source-1" && artifact.portId === "out-1", + ); + assert(edgeFailedArtifact?.name, "failed edge artifact not found"); + + await gotoWithRetry(page, `${baseURL}/workflow/${edgeFailedWorkflow.id}`); + await page.locator(".react-flow").waitFor(); + await page.getByText(`Run ${edgeFailedRun.id}`).waitFor(); + await clickWorkflowEdge(page, "真实数据源 到 Shell 命令执行 连线"); + await page + .locator(".edge-output-panel") + .getByText("执行失败", { exact: true }) + .waitFor(); + await page + .locator(".edge-output-panel") + .getByText(edgeFailedArtifact.name, { exact: true }) + .waitFor(); + log("export template from workflow editor"); - await runRow.dblclick(); + await gotoWithRetry(page, `${baseURL}/workflow/${workflow.id}?runId=${editorRun.id}`); + await page.locator(".react-flow").waitFor(); await page.getByRole("button", { name: "导出模板" }).click(); await page.locator('input').filter({ has: page.locator('.. >> text=模板名称') }).first().fill(names.template); await page.getByRole("button", { name: "创建模板" }).click(); @@ -433,6 +1027,7 @@ async function main() { await page.getByRole("heading", { name: "工作流模板" }).waitFor(); await page.getByPlaceholder("搜索模板...").fill(names.template); await page.getByText(names.template).waitFor(); + await capture(page, "13-templates"); const templateCard = page.locator("div").filter({ hasText: names.template }).first(); await templateCard .locator("button") @@ -455,6 +1050,7 @@ async function main() { log("users create edit delete"); await page.getByRole("link", { name: "用户管理" }).click(); await page.getByRole("heading", { name: "用户管理" }).waitFor(); + await capture(page, "14-users"); await clickWithRetry(page, page.locator("button").filter({ hasText: /^创建用户$/ }).first()); await page.getByRole("heading", { name: "创建用户" }).waitFor(); await fillLabeledInput(page, "用户名", names.extraUser); @@ -474,11 +1070,12 @@ async function main() { const auditTwo = await createAuditWorkflow(`${names.auditProject} B`, `PW Audit Workflow B ${suffix}`); await gotoWithRetry(page, `${baseURL}/audit`); await page.getByRole("heading", { name: /任务审核/ }).waitFor(); - await page.getByText(`${names.auditProject} A`).click(); - await page.getByRole("button", { name: /审核通过/ }).click(); - await page.getByText(`${names.auditProject} B`).click(); - await page.getByRole("button", { name: /审核拒绝/ }).click(); - await page.getByRole("button", { name: "已处理" }).click(); + await capture(page, "15-audit"); + await clickWithRetry(page, page.getByText(`${names.auditProject} A`).first()); + await clickWithRetry(page, page.getByRole("button", { name: /审核通过/ })); + await clickWithRetry(page, page.getByText(`${names.auditProject} B`).first()); + await clickWithRetry(page, page.getByRole("button", { name: /审核拒绝/ })); + await clickWithRetry(page, page.getByRole("button", { name: "已处理" })); await poll( async () => api(`/api/runs/${auditOne.run.id}`), @@ -493,11 +1090,19 @@ async function main() { log("functional check completed"); } catch (error) { - await page.screenshot({ path: `/tmp/droboticflow-playwright-failure-${suffix}.png`, fullPage: true }).catch(() => {}); + await page.screenshot({ path: path.join(artifactDir, `failure-${suffix}.png`), fullPage: true }).catch(() => {}); throw error; } finally { + await page.close().catch(() => {}); + if (page.video()) { + recordedVideoPath = await page.video().path().catch(() => ""); + } await context.close(); await browser.close(); + if (recordedVideoPath && fs.existsSync(recordedVideoPath)) { + const targetPath = path.join(artifactDir, "playwright-functional-check.webm"); + fs.copyFileSync(recordedVideoPath, targetPath); + } } } diff --git a/scripts/playwright-preview-operators-check.mjs b/scripts/playwright-preview-operators-check.mjs index ff9e4aba..4b7a6991 100644 --- a/scripts/playwright-preview-operators-check.mjs +++ b/scripts/playwright-preview-operators-check.mjs @@ -119,7 +119,7 @@ async function ensureUIReady(page) { await page.locator("select").first().selectOption("admin"); await page.getByRole("button", { name: "创建并进入系统" }).click(); } - await page.getByRole("heading", { name: "DroboticFlow 仪表盘" }).waitFor(); + await page.getByRole("heading", { name: "Drobodata 仪表盘" }).waitFor(); } async function createProject() { diff --git a/scripts/reconcile-system-data.sh b/scripts/reconcile-system-data.sh index f167ae9f..b3c861bd 100755 --- a/scripts/reconcile-system-data.sh +++ b/scripts/reconcile-system-data.sh @@ -129,6 +129,24 @@ builtin_operators = [ {"key": "SOURCE_PATH", "value": "", "description": "从本地路径读取真实数据"}, ], }, + { + "name": "BOS 增量扫描", + "type": "builtin", + "category": "IO", + "description": "按 bucket/prefix 增量扫描 BOS,仅输出本次新增对象并提交 checkpoint 给 runtime。", + "inputs": [], + "outputs": [{"id": "out-1", "name": "manifest", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "统一工作流里的有状态源算子。每次 run 只产出 BOS 中新增的对象,状态由 runtime 在 run 成功后提交。", + "hyperparams": [ + {"key": "DATA_SOURCE", "value": "BOS", "description": "固定为 BOS"}, + {"key": "SOURCE_PATH", "value": "", "description": "完整 bos://bucket/prefix 路径"}, + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com", "description": "BOS Endpoint"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "", "description": "BOS Access Key ID"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "", "description": "BOS Secret Access Key"}, + {"key": "SCAN_MODE", "value": "INCREMENTAL", "description": "固定为 INCREMENTAL"}, + ], + }, { "name": "模型推理", "type": "builtin", @@ -290,6 +308,116 @@ builtin_operators = [ {"key": "EXPECTED_JSON", "value": "", "description": "JSON 格式比较值,优先于 EXPECTED"}, ], }, + { + "name": "X 平台数据集索引", + "type": "builtin", + "category": "DATASET", + "description": "扫描 X-platform 采集目录,解析 meta.json、video_meta.json 和视频文件索引。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "dataset", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "面向 BJ_***/DJI_***/meta.json/video_meta.json 结构的真实数据集索引算子,输出采集目录和视频清单。", + "hyperparams": [ + {"key": "SOURCE_PATH", "value": "", "description": "X-platform 根目录或单个采集目录"}, + ], + }, + { + "name": "采集目录校验", + "type": "builtin", + "category": "DATASET", + "description": "校验采集目录是否包含 meta.json、video_meta.json,以及可选 intrinsics.json。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "validated", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "对 X-platform 数据集索引结果执行真实结构校验,输出每个采集目录的有效性报告。", + "hyperparams": [ + {"key": "REQUIRE_INTRINSICS", "value": "false", "description": "是否要求 intrinsics.json 必须存在"}, + ], + }, + { + "name": "视频质检", + "type": "builtin", + "category": "VIDEO", + "description": "基于真实视频帧计算清晰度、抖动度和无手率估计。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "quality", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "使用 OpenCV 对真实 mp4 做质检分析,输出 clarity / shake / handOcclusion / passed。", + "hyperparams": [ + {"key": "MIN_CLARITY", "value": "0.8", "description": "清晰度最低阈值"}, + {"key": "MAX_SHAKE", "value": "0.1", "description": "抖动度最高阈值"}, + {"key": "MAX_HAND_OCCLUSION", "value": "0.1", "description": "无手率最高阈值"}, + {"key": "CLARITY_SCALE", "value": "35", "description": "清晰度归一化尺度"}, + ], + }, + { + "name": "数据过滤", + "type": "builtin", + "category": "VIDEO", + "description": "按日期、地点、场景和质检阈值过滤有效视频。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "filtered", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "针对 X-platform 质检结果执行真实筛选,输出符合条件的视频列表。", + "hyperparams": [ + {"key": "DATE_START", "value": "", "description": "开始日期,支持 YYYY-MM-DD / YYYYMMDD"}, + {"key": "DATE_END", "value": "", "description": "结束日期,支持 YYYY-MM-DD / YYYYMMDD"}, + {"key": "PLACE", "value": "", "description": "采集地点"}, + {"key": "TYPE", "value": "", "description": "采集场景类型"}, + {"key": "MIN_CLARITY", "value": "0.8", "description": "清晰度最低阈值"}, + {"key": "MAX_SHAKE", "value": "0.1", "description": "抖动度最高阈值"}, + {"key": "MAX_HAND_OCCLUSION", "value": "0.1", "description": "无手率最高阈值"}, + ], + }, + { + "name": "视频切片", + "type": "builtin", + "category": "VIDEO", + "description": "将真实视频按固定时长切分为多个 mp4 片段。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "clips", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "使用 OpenCV 将真实视频切成固定长度片段,并输出 clip manifest。", + "hyperparams": [ + {"key": "SEGMENT_DURATION_SECONDS", "value": "300", "description": "每个切片的秒数"}, + ], + }, + { + "name": "视频标注", + "type": "builtin", + "category": "VIDEO", + "description": "基于真实视频片段生成阶段划分、动作标签和对象提示。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "annotations", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "读取切片视频并做真实帧间运动分析,输出 annotation JSON 和动作分段。", + "hyperparams": [ + {"key": "FRAME_INTERVAL_SECONDS", "value": "1.0", "description": "采样帧间隔秒数"}, + ], + }, + { + "name": "成品打包", + "type": "builtin", + "category": "VIDEO", + "description": "将切片视频、质检结果、标注结果和配置文件组装成成品数据包。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "packages", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "按 clip 组织真实 package 目录,输出 packages manifest 和所有成品文件。", + }, + { + "name": "抽检采样", + "type": "builtin", + "category": "QA", + "description": "按质检和标注规则对成品数据执行真实抽检采样。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "samples", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "根据 RULES_JSON 对成品数据包做规则筛选和稳定抽样,输出抽检任务清单。", + "hyperparams": [ + {"key": "RULES_JSON", "value": "[]", "description": "抽检规则数组,包含 field/operator/value/samplePercent"}, + ], + }, { "name": "LeRobot 数据集概览", "type": "builtin", diff --git a/scripts/render-manifests.sh b/scripts/render-manifests.sh index 539f81c9..594ef4f3 100755 --- a/scripts/render-manifests.sh +++ b/scripts/render-manifests.sh @@ -3,6 +3,18 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" TEMPLATE="$ROOT_DIR/k8s/droboticflow.template.yaml" +DEPLOY_ENV="${DEPLOY_ENV:-}" + +if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then + DEPLOY_ENV="$1" + shift +fi + +if [[ -n "$DEPLOY_ENV" ]]; then + export DEPLOY_ENV + # shellcheck source=scripts/load-deploy-env.sh + source "$ROOT_DIR/scripts/load-deploy-env.sh" +fi K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" MONGO_DB="${MONGO_DB:-droboticflow}" @@ -10,6 +22,8 @@ MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}" MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}" WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}" K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}" +WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}" +WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}" API_NODE_PORT="${API_NODE_PORT:-30080}" API_REPLICAS="${API_REPLICAS:-1}" WORKER_REPLICAS="${WORKER_REPLICAS:-1}" @@ -19,7 +33,6 @@ AI_MODEL="${AI_MODEL:-glm-5}" API_IMAGE="${API_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-api:latest}" WORKER_IMAGE="${WORKER_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-worker:latest}" OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:latest}" -MONGO_IMAGE="${MONGO_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-mongo:7}" rendered="$(cat "$TEMPLATE")" rendered="${rendered//__NAMESPACE__/$K8S_NAMESPACE}" @@ -28,6 +41,8 @@ rendered="${rendered//__MONGO_CONNECT_RETRIES__/$MONGO_CONNECT_RETRIES}" rendered="${rendered//__MONGO_CONNECT_RETRY_DELAY_MS__/$MONGO_CONNECT_RETRY_DELAY_MS}" rendered="${rendered//__WORKER_POLL_INTERVAL_MS__/$WORKER_POLL_INTERVAL_MS}" rendered="${rendered//__K8S_IMAGE_PULL_SECRET_NAME__/$K8S_IMAGE_PULL_SECRET_NAME}" +rendered="${rendered//__WORKSPACE_PVC_NAME__/$WORKSPACE_PVC_NAME}" +rendered="${rendered//__WORKSPACE_MOUNT_PATH__/$WORKSPACE_MOUNT_PATH}" rendered="${rendered//__API_NODE_PORT__/$API_NODE_PORT}" rendered="${rendered//__API_REPLICAS__/$API_REPLICAS}" rendered="${rendered//__WORKER_REPLICAS__/$WORKER_REPLICAS}" @@ -36,6 +51,5 @@ rendered="${rendered//__AI_MODEL__/$AI_MODEL}" rendered="${rendered//__API_IMAGE__/$API_IMAGE}" rendered="${rendered//__WORKER_IMAGE__/$WORKER_IMAGE}" rendered="${rendered//__OPERATOR_RUNNER_IMAGE__/$OPERATOR_RUNNER_IMAGE}" -rendered="${rendered//__MONGO_IMAGE__/$MONGO_IMAGE}" printf '%s\n' "$rendered" diff --git a/scripts/wjzh_operator_multi_project_regression.py b/scripts/wjzh_operator_multi_project_regression.py new file mode 100644 index 00000000..e8a2f6ec --- /dev/null +++ b/scripts/wjzh_operator_multi_project_regression.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import sys +import time +from collections import OrderedDict +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent +BASE_PATH = ROOT / "wjzh_operator_regression.py" + +spec = importlib.util.spec_from_file_location("wjzh_regression_base", BASE_PATH) +if spec is None or spec.loader is None: + raise SystemExit(f"failed to load regression base from {BASE_PATH}") +base = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = base +spec.loader.exec_module(base) + + +RUNNER_IMAGE_OVERRIDE = os.getenv("OPERATOR_RUNNER_IMAGE", "").strip() + + +def refresh_workflow_runner_image(config: Any, workflow_id: str, runner_image: str) -> dict[str, Any]: + if not runner_image: + return base.get_workflow(config, workflow_id) + workflow = base.get_workflow(config, workflow_id) + nodes = workflow.get("nodes", []) + changed = False + for node in nodes: + data = node.get("data") or {} + image = str(data.get("image", "")).strip() + if "droboticflow-operator-runner:" not in image: + continue + if image == runner_image: + continue + data["image"] = runner_image + changed = True + if changed: + workflow = base.update_workflow(config, workflow_id, {"nodes": nodes}) + return workflow + + +def build_group_projects(config: Any, timestamp: str) -> OrderedDict[str, dict[str, Any]]: + groups: OrderedDict[str, dict[str, Any]] = OrderedDict( + [ + ( + "基础数据与索引", + { + "description": "真实数据源、目录扫描、模型推理、BOS 扫描、索引、校验与上传回归。", + }, + ), + ( + "JSON与逻辑", + { + "description": "JSON/CSV/分流/合并等逻辑链路回归。", + }, + ), + ( + "审核与业务写入", + { + "description": "人工审核 approve/reject 分支及 Mongo 写入回归。", + }, + ), + ( + "视频处理与打包", + { + "description": "视频质检、过滤、切片、标注、打包、抽检回归。", + }, + ), + ( + "Robotics与增量", + { + "description": "LeRobot / ROSBag 概览和目录级增量循环处理回归。", + }, + ), + ] + ) + for group_name, payload in groups.items(): + payload["project"] = base.create_project( + config, + f"{config.project_prefix} {timestamp} {group_name}", + payload["description"], + ) + return groups + + +def group_name_for_workflow(workflow_name: str) -> str: + if workflow_name.startswith(("00-", "01-")): + return "基础数据与索引" + if workflow_name.startswith("02-"): + return "JSON与逻辑" + if workflow_name.startswith(("03-", "04-", "05-")): + return "审核与业务写入" + if workflow_name.startswith(("06-", "07-", "08-", "10-", "11-")): + return "视频处理与打包" + if workflow_name.startswith(("09-", "12-")): + return "Robotics与增量" + raise base.RegressionError(f"unmapped workflow group for {workflow_name}") + + +def clone_temp_workflows( + config: Any, + temp_project_id: str, + temp_workflows: list[dict[str, Any]], + grouped_projects: OrderedDict[str, dict[str, Any]], +) -> list[dict[str, Any]]: + cloned: list[dict[str, Any]] = [] + for item in temp_workflows: + original = item["workflow"] + target_group = group_name_for_workflow(str(original.get("name", ""))) + target_project_id = grouped_projects[target_group]["project"]["id"] + copy = base.clone_workflow(config, target_project_id, original["id"], original["name"]) + copy = refresh_workflow_runner_image(config, copy["id"], RUNNER_IMAGE_OVERRIDE) + cloned_item = dict(item) + cloned_item["workflow"] = copy + cloned_item["targetGroup"] = target_group + cloned.append(cloned_item) + base.request(config, "DELETE", f"/api/projects/{temp_project_id}", expected=(204,)) + return cloned + + +def create_source_variants_workflow( + config: Any, + project_id: str, + operators_by_name: dict[str, dict[str, Any]], + connection_id: str, + output_root: str, +) -> dict[str, Any]: + nodes = [ + base.build_node( + operators_by_name["真实数据源"], + "source_text", + 0, + -90, + {"SOURCE_TEXT": "alpha\\nbeta\\ngamma"}, + connection_id=connection_id, + label="真实数据源-文本", + ), + base.build_node( + operators_by_name["模型推理"], + "infer_text", + 240, + -90, + {"COMMAND": base.model_inference_command()}, + ), + base.upload_node( + operators_by_name["结果上传"], + connection_id, + "upload_text", + 480, + -90, + f"{output_root}/00-source-variants/text", + ), + base.build_node( + operators_by_name["真实数据源"], + "source_json", + 0, + 80, + { + "SOURCE_JSON": json.dumps( + { + "datasetType": "synthetic-json-source", + "items": [ + {"kind": "synthetic", "value": 1}, + {"kind": "synthetic", "value": 2}, + ], + }, + ensure_ascii=False, + ) + }, + connection_id=connection_id, + label="真实数据源-JSON", + ), + base.upload_node( + operators_by_name["结果上传"], + connection_id, + "upload_json", + 240, + 80, + f"{output_root}/00-source-variants/json", + ), + ] + edges = [ + base.upload_edge("source_text", "infer_text", "source-text-infer"), + base.upload_edge("infer_text", "upload_text", "infer-text-upload"), + base.upload_edge("source_json", "upload_json", "source-json-upload"), + ] + workflow = base.create_workflow( + config, + project_id, + "00-真实数据源多入口", + "覆盖真实数据源的 SOURCE_TEXT / SOURCE_JSON 子功能。", + nodes, + edges, + ) + return refresh_workflow_runner_image(config, workflow["id"], RUNNER_IMAGE_OVERRIDE) + + +def create_manual_sampling_workflow( + config: Any, + project_id: str, + operators_by_name: dict[str, dict[str, Any]], + connection_id: str, + output_root: str, + source_path: str, +) -> dict[str, Any]: + nodes = [ + base.build_node( + operators_by_name["BOS 增量扫描"], + "source", + 0, + 0, + {"SOURCE_PATH": source_path, "SCAN_MODE": "FULL"}, + connection_id=connection_id, + ), + base.build_node(operators_by_name["X 平台数据集索引"], "index", 220, 0), + base.build_node( + operators_by_name["采集目录校验"], + "validate", + 440, + 0, + {"REQUIRE_INTRINSICS": "false"}, + ), + base.build_node( + operators_by_name["Shell 命令执行"], + "batch_pick", + 660, + 0, + {"COMMAND": base.cap_duration_command(45, max_items=4, max_duration=500)}, + label="人工抽检前抽样", + ), + base.build_node( + operators_by_name["视频质检"], + "quality", + 880, + -60, + {"MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}, + ), + base.build_node( + operators_by_name["视频切片"], + "slice", + 1100, + -60, + {"SEGMENT_DURATION_SECONDS": "30"}, + ), + base.build_node( + operators_by_name["视频标注"], + "annotate", + 1320, + -60, + {"FRAME_INTERVAL_SECONDS": "1.0"}, + ), + base.build_node(operators_by_name["成品打包"], "package", 1540, -60), + base.build_node( + operators_by_name["人工抽检采样"], + "manual_sample", + 1760, + -60, + { + "RULES_JSON": json.dumps( + [ + {"id": "low-clarity", "field": "quality.clarity", "operator": "lt", "value": 0.5, "samplePercent": 10}, + {"id": "high-shake", "field": "quality.shake", "operator": "gt", "value": 0.15, "samplePercent": 10}, + ], + ensure_ascii=False, + ) + }, + ), + base.upload_node( + operators_by_name["结果上传"], + connection_id, + "manual_sample_upload", + 1980, + -60, + f"{output_root}/11b-manual-sampling/sample", + ), + ] + edges = [ + base.upload_edge("source", "index", "source-index"), + base.upload_edge("index", "validate", "index-validate"), + base.upload_edge("validate", "batch_pick", "validate-batch-pick"), + base.upload_edge("batch_pick", "quality", "batch-pick-quality"), + base.upload_edge("quality", "slice", "quality-slice"), + base.upload_edge("slice", "annotate", "slice-annotate"), + base.upload_edge("annotate", "package", "annotate-package"), + base.upload_edge("package", "manual_sample", "package-manual-sample"), + base.upload_edge("manual_sample", "manual_sample_upload", "manual-sample-upload"), + ] + workflow = base.create_workflow( + config, + project_id, + "11B-人工抽检采样", + "使用人工抽检采样算子对成品包执行规则采样回归。", + nodes, + edges, + ) + return refresh_workflow_runner_image(config, workflow["id"], RUNNER_IMAGE_OVERRIDE) + + +def create_multi_project_regression(config: Any) -> tuple[OrderedDict[str, dict[str, Any]], list[dict[str, Any]], str, str, str]: + settings = base.request(config, "GET", "/api/settings") + operators = base.request(config, "GET", "/api/operators")["items"] + connection = base.ensure_storage_connection(config) + mongo_connection = base.find_mongo_connection(config) + base.validate_preflight(config, connection, operators, settings, mongo_connection) + operators_by_name = {item["name"]: item for item in operators} + + timestamp = time.strftime("%Y%m%d%H%M%S") + temp_project_id, temp_workflows, mongo_collection, output_root, incremental_source_path = base.create_regression_workflows( + config, + connection["id"], + mongo_connection["id"] if mongo_connection else "", + operators_by_name, + ) + grouped_projects = build_group_projects(config, timestamp) + cloned = clone_temp_workflows(config, temp_project_id, temp_workflows, grouped_projects) + + extras: list[dict[str, Any]] = [] + extras.append( + { + "workflow": create_source_variants_workflow( + config, + grouped_projects["基础数据与索引"]["project"]["id"], + operators_by_name, + connection["id"], + output_root, + ), + "targetGroup": "基础数据与索引", + } + ) + extras.append( + { + "workflow": create_manual_sampling_workflow( + config, + grouped_projects["视频处理与打包"]["project"]["id"], + operators_by_name, + connection["id"], + output_root, + config.source_path, + ), + "targetGroup": "视频处理与打包", + } + ) + + return grouped_projects, cloned + extras, mongo_collection, output_root, incremental_source_path + + +def configure_directory_incremental_workflow( + config: Any, + workflow_id: str, + incremental_source_path: str, + connection_id: str, + interval_seconds: int, +) -> None: + scan_bucket, scan_prefix = base.parse_bos_path(incremental_source_path) + payload: dict[str, Any] = { + "kind": "scan", + "maxConcurrentRuns": 1, + "incrementalUnit": "DIRECTORY", + "directoryDepth": 1, + "scanConfig": { + "enabled": True, + "connectionId": connection_id, + "dataSource": "BOS", + "sourcePath": incremental_source_path, + "bucket": scan_bucket, + "prefix": scan_prefix, + "scanMode": "INCREMENTAL", + }, + "type": "streaming", + "schedule": { + "interval": interval_seconds, + }, + } + base.update_workflow(config, workflow_id, payload) + + +def find_seed_clip_keys(config: Any, connection_id: str, output_root: str) -> list[str]: + seed_clip_keys: list[str] = [] + for suffix in ["06-medium-video/slice", "07-full-chain/slice", "11-batch-fullchain/slice"]: + seed_clip_keys.extend(base.list_uploaded_mp4_keys(config, connection_id, output_root, suffix)) + return list(dict.fromkeys([key for key in seed_clip_keys if key])) + + +def run_grouped_regression(config: Any) -> dict[str, Any]: + grouped_projects, workflows, mongo_collection, output_root, incremental_source_path = create_multi_project_regression(config) + connection = base.ensure_storage_connection(config) + mongo_connection = base.find_mongo_connection(config) + + workflow_items_by_name = {item["workflow"]["name"]: item for item in workflows} + execution_order = [ + "00-源数据目录推理", + "00-真实数据源多入口", + "01-索引校验双上传", + "02-JSON工具链多分支", + "03-人工审核通过", + "04-人工审核拒绝", + "05-Mongo写入", + "06-中大型视频切片", + "07-超大视频全链路", + "08-多目录批量窗口化", + "09-Robotics派生概览", + "10-多目录多视频批量质检", + "11-多目录多视频批量全链路", + "11B-人工抽检采样", + ] + + results: list[dict[str, Any]] = [] + mongo_verification: dict[str, Any] | None = None + for name in execution_order: + item = workflow_items_by_name[name] + workflow = item["workflow"] + rounds = max(int(item.get("rounds", 1) or 1), 1) + for round_index in range(rounds): + current_workflow = workflow + if round_index > 0: + group_name = item["targetGroup"] + current_workflow = base.clone_workflow( + config, + grouped_projects[group_name]["project"]["id"], + workflow["id"], + f"{workflow['name']} 第{round_index + 1}轮", + ) + current_workflow = refresh_workflow_runner_image(config, current_workflow["id"], RUNNER_IMAGE_OVERRIDE) + finished = base.run_workflow_until_complete( + config, + current_workflow["id"], + config.max_wait_seconds, + audit_decision=str(item.get("auditDecision", "")).strip(), + ) + summary = base.summarize_run(finished) + summary["projectId"] = grouped_projects[item["targetGroup"]]["project"]["id"] + summary["projectName"] = grouped_projects[item["targetGroup"]]["project"]["name"] + if rounds > 1: + summary["round"] = round_index + 1 + if item.get("auditDecision"): + summary["auditDecision"] = item["auditDecision"] + results.append(summary) + if finished["status"] != "completed": + raise base.RegressionError( + json.dumps( + { + "status": "workflow-failed", + "workflowName": current_workflow["name"], + "run": summary, + "runLogsTail": base.get_run_logs(config, finished["id"])[-30:], + }, + ensure_ascii=False, + indent=2, + ) + ) + if item.get("mongoCollection"): + documents = base.query_business_collection(config, mongo_connection["id"], str(item["mongoCollection"]), limit=50) + mongo_verification = { + "collection": item["mongoCollection"], + "documentCount": documents.get("total", 0), + "sampleDocumentIds": [doc.get("_id") for doc in documents.get("documents", [])[:10]], + } + if int(mongo_verification["documentCount"] or 0) <= 0: + raise base.RegressionError(f"mongo verification failed for collection {item['mongoCollection']}") + + incremental_item = workflow_items_by_name["12-增量嗅探循环处理"] + incremental_workflow = incremental_item["workflow"] + seed_clip_keys = find_seed_clip_keys(config, connection["id"], output_root) + if not seed_clip_keys: + raise base.RegressionError(f"failed to find seed clips under {output_root}") + + incremental_bucket, incremental_prefix = base.parse_bos_path(incremental_source_path) + configure_directory_incremental_workflow( + config, + incremental_workflow["id"], + incremental_source_path, + connection["id"], + int(incremental_item.get("streamingIntervalSeconds", 20) or 20), + ) + round_summaries: list[dict[str, Any]] = [] + previous_run_id = "" + for round_item in incremental_item.get("incrementalRounds", []): + created_directories = base.ensure_incremental_seed_directories( + config, + connection["id"], + incremental_prefix, + seed_clip_keys, + bucket=incremental_bucket, + round_name=str(round_item.get("name", "round")).strip() or "round", + directory_count=int(round_item.get("directoryCount", 0) or 0), + ) + if round_summaries: + time.sleep(2) + finished = base.wait_for_next_workflow_run( + config, + incremental_workflow["id"], + previous_run_id, + config.max_wait_seconds, + ) + summary = base.summarize_run(finished) + summary["incrementalRound"] = round_item.get("name") + summary["createdDirectories"] = created_directories + summary["projectId"] = grouped_projects[incremental_item["targetGroup"]]["project"]["id"] + summary["projectName"] = grouped_projects[incremental_item["targetGroup"]]["project"]["name"] + results.append(summary) + round_summaries.append(summary) + previous_run_id = str(finished.get("id", "")).strip() + if finished["status"] != "completed": + raise base.RegressionError( + json.dumps( + { + "status": "incremental-workflow-failed", + "workflowName": incremental_workflow["name"], + "run": summary, + "runLogsTail": base.get_run_logs(config, finished["id"])[-30:], + }, + ensure_ascii=False, + indent=2, + ) + ) + + return { + "status": "completed", + "projects": [ + { + "name": payload["project"]["name"], + "id": payload["project"]["id"], + "description": payload["description"], + } + for payload in grouped_projects.values() + ], + "mongoCollection": mongo_collection, + "mongoVerification": mongo_verification, + "incrementalSourcePath": incremental_source_path, + "incrementalVerification": { + "seedClipKeys": seed_clip_keys, + "rounds": round_summaries, + }, + "sourcePath": config.source_path, + "uploadRootPrefix": output_root, + "workflows": results, + } + + +def main() -> int: + config = base.parse_args() + base.delete_projects(config) + payload = run_grouped_regression(config) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except base.RegressionError as err: + print(str(err), file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/wjzh_operator_regression.py b/scripts/wjzh_operator_regression.py new file mode 100755 index 00000000..5ea39b5b --- /dev/null +++ b/scripts/wjzh_operator_regression.py @@ -0,0 +1,1657 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import mimetypes +import os +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from dataclasses import dataclass +from typing import Any + + +DEFAULT_API_BASE_URL = "http://127.0.0.1:3001" +DEFAULT_SOURCE_PATH = "bos://wjzh-data-migration/wjzh-0330-Data-migration" +DEFAULT_UPLOAD_ROOT_PREFIX = "bos://wjzh-data-migration/wjzh-0330-operator-regression" +DEFAULT_PROJECT_PREFIX = "wjzh-0330 大文件算子回归" +DEFAULT_CONNECTION_NAME = "wjzh-data-migration" +MEDIUM_VIDEO = "DJI_20260327140929_0003_D.MP4" +HUGE_VIDEO = "DJI_20260327133133_0002_D.MP4" +HUGE_WINDOW_SECONDS = 120 + + +class RegressionError(RuntimeError): + pass + + +@dataclass +class Config: + api_base_url: str + source_path: str + upload_root_prefix: str + project_prefix: str + storage_connection_id: str + storage_connection_name: str + create_connection_if_missing: bool + mongo_connection_id: str + mongo_collection_prefix: str + bos_endpoint: str + bos_bucket: str + bos_access_key_id: str + bos_secret_access_key: str + delete_all_projects: bool + delete_regression_projects: bool + preflight_only: bool + skip_runtime_smoke: bool + max_wait_seconds: int + + +def env_default(name: str, fallback: str = "") -> str: + return os.getenv(name, fallback).strip() + + +def parse_args() -> Config: + parser = argparse.ArgumentParser(description="Run production-style WJZH operator regression workflows") + parser.add_argument("--api-base-url", default=env_default("API_BASE_URL", DEFAULT_API_BASE_URL)) + parser.add_argument("--source-path", default=env_default("SOURCE_PATH", DEFAULT_SOURCE_PATH)) + parser.add_argument("--upload-root-prefix", default=env_default("UPLOAD_ROOT_PREFIX", DEFAULT_UPLOAD_ROOT_PREFIX)) + parser.add_argument("--project-prefix", default=env_default("PROJECT_PREFIX", DEFAULT_PROJECT_PREFIX)) + parser.add_argument("--storage-connection-id", default=env_default("STORAGE_CONNECTION_ID")) + parser.add_argument("--storage-connection-name", default=env_default("STORAGE_CONNECTION_NAME", DEFAULT_CONNECTION_NAME)) + parser.add_argument( + "--create-connection-if-missing", + action="store_true", + default=env_default("CREATE_CONNECTION_IF_MISSING").lower() in {"1", "true", "yes"}, + ) + parser.add_argument("--mongo-connection-id", default=env_default("MONGO_CONNECTION_ID")) + parser.add_argument("--mongo-collection-prefix", default=env_default("MONGO_COLLECTION_PREFIX", "wjzh_operator_regression")) + parser.add_argument("--bos-endpoint", default=env_default("BOS_ENDPOINT", "https://bj.bcebos.com")) + parser.add_argument("--bos-bucket", default=env_default("BOS_BUCKET", "wjzh-data-migration")) + parser.add_argument("--bos-access-key-id", default=env_default("BOS_ACCESS_KEY_ID")) + parser.add_argument("--bos-secret-access-key", default=env_default("BOS_SECRET_ACCESS_KEY")) + parser.add_argument("--delete-all-projects", action="store_true") + parser.add_argument("--delete-regression-projects", action="store_true") + parser.add_argument("--preflight-only", action="store_true") + parser.add_argument("--skip-runtime-smoke", action="store_true") + parser.add_argument("--max-wait-seconds", type=int, default=int(env_default("MAX_WAIT_SECONDS", "5400") or "5400")) + args = parser.parse_args() + return Config( + api_base_url=args.api_base_url.rstrip("/"), + source_path=args.source_path, + upload_root_prefix=args.upload_root_prefix.rstrip("/"), + project_prefix=args.project_prefix, + storage_connection_id=args.storage_connection_id, + storage_connection_name=args.storage_connection_name, + create_connection_if_missing=bool(args.create_connection_if_missing), + mongo_connection_id=args.mongo_connection_id, + mongo_collection_prefix=args.mongo_collection_prefix.strip() or "wjzh_operator_regression", + bos_endpoint=args.bos_endpoint, + bos_bucket=args.bos_bucket, + bos_access_key_id=args.bos_access_key_id, + bos_secret_access_key=args.bos_secret_access_key, + delete_all_projects=bool(args.delete_all_projects), + delete_regression_projects=bool(args.delete_regression_projects), + preflight_only=bool(args.preflight_only), + skip_runtime_smoke=bool(args.skip_runtime_smoke), + max_wait_seconds=max(args.max_wait_seconds, 60), + ) + + +def request(config: Config, method: str, path: str, payload: Any | None = None, expected: tuple[int, ...] = (200, 201, 202, 204)) -> Any: + body = None + headers = {"Content-Type": "application/json"} + if payload is not None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + req = urllib.request.Request(config.api_base_url + path, data=body, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + raw = resp.read().decode("utf-8", errors="replace") + if resp.status not in expected: + raise RegressionError(f"{method} {path} unexpected {resp.status}: {raw}") + return json.loads(raw) if raw else None + except urllib.error.HTTPError as err: + raw = err.read().decode("utf-8", errors="replace") + raise RegressionError(f"{method} {path} failed {err.code}: {raw}") from err + except urllib.error.URLError as err: + raise RegressionError(f"{method} {path} failed: {err}") from err + + +def request_bytes(config: Config, method: str, path: str, body: bytes | None = None, headers: dict[str, str] | None = None, expected: tuple[int, ...] = (200,)) -> tuple[int, bytes, dict[str, str]]: + req = urllib.request.Request(config.api_base_url + path, data=body, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=300) as resp: + payload = resp.read() + if resp.status not in expected: + raise RegressionError(f"{method} {path} unexpected {resp.status}: {payload.decode('utf-8', errors='replace')}") + return resp.status, payload, dict(resp.headers.items()) + except urllib.error.HTTPError as err: + raw = err.read().decode("utf-8", errors="replace") + raise RegressionError(f"{method} {path} failed {err.code}: {raw}") from err + except urllib.error.URLError as err: + raise RegressionError(f"{method} {path} failed: {err}") from err + + +def object_storage_list(config: Config, connection_id: str, prefix: str, *, bucket: str = "") -> list[dict[str, Any]]: + query = urllib.parse.urlencode( + { + "connectionId": connection_id, + "bucket": bucket, + "prefix": prefix, + } + ) + payload = request(config, "GET", f"/api/object-storage/objects?{query}") + return payload.get("items", []) + + +def object_storage_read(config: Config, connection_id: str, key: str, *, bucket: str = "") -> bytes: + query = urllib.parse.urlencode( + { + "connectionId": connection_id, + "bucket": bucket, + "key": key, + } + ) + _, payload, _ = request_bytes(config, "GET", f"/api/object-storage/content?{query}") + return payload + + +def object_storage_upload(config: Config, connection_id: str, key: str, payload: bytes, *, bucket: str = "", filename: str = "", content_type: str = "") -> Any: + boundary = f"----CodexBoundary{uuid.uuid4().hex}" + filename = filename or os.path.basename(key) or "upload.bin" + content_type = content_type or mimetypes.guess_type(filename)[0] or "application/octet-stream" + body = ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n' + f"Content-Type: {content_type}\r\n\r\n" + ).encode("utf-8") + payload + f"\r\n--{boundary}--\r\n".encode("utf-8") + query = urllib.parse.urlencode( + { + "connectionId": connection_id, + "bucket": bucket, + "key": key, + } + ) + _, raw, _ = request_bytes( + config, + "POST", + f"/api/object-storage/objects/upload?{query}", + body=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + ) + return json.loads(raw.decode("utf-8")) + + +def object_storage_delete_prefix(config: Config, connection_id: str, key: str, *, bucket: str = "") -> None: + query = urllib.parse.urlencode( + { + "connectionId": connection_id, + "bucket": bucket, + "key": key, + "recursive": "true", + } + ) + request_bytes(config, "DELETE", f"/api/object-storage/objects?{query}", expected=(200,)) + + +def parse_bos_path(path: str) -> tuple[str, str]: + raw = str(path or "").strip() + if not raw.startswith("bos://"): + raise RegressionError(f"expected bos:// path, got: {raw}") + bucket_and_key = raw[len("bos://") :] + bucket, _, key = bucket_and_key.partition("/") + return bucket.strip(), key.strip().strip("/") + + +def required_operators() -> list[str]: + return [ + "真实数据源", + "BOS 增量扫描", + "目录扫描", + "模型推理", + "X 平台数据集索引", + "采集目录校验", + "JSON 字段提取", + "文本模板渲染", + "JSON 过滤", + "JSON 映射重命名", + "JSON 转 CSV", + "CSV 转 JSON", + "条件分流", + "数据合并", + "视频质检", + "数据过滤", + "视频切片", + "视频标注", + "成品打包", + "抽检采样", + "人工审核", + "MongoDB 写入", + "LeRobot 数据集概览", + "ROSBag 数据概览", + "结果上传", + "Shell 命令执行", + ] + + +def find_storage_connection(config: Config) -> dict[str, Any] | None: + items = request(config, "GET", "/api/storage/connections")["items"] + if config.storage_connection_id: + return next((item for item in items if item.get("id") == config.storage_connection_id), None) + if config.storage_connection_name: + return next((item for item in items if str(item.get("name", "")).strip() == config.storage_connection_name), None) + return None + + +def list_mongo_connections(config: Config) -> list[dict[str, Any]]: + return request(config, "GET", "/api/mongo/connections")["items"] + + +def find_mongo_connection(config: Config) -> dict[str, Any] | None: + items = list_mongo_connections(config) + if config.mongo_connection_id: + return next((item for item in items if item.get("id") == config.mongo_connection_id), None) + return items[0] if items else None + + +def ensure_storage_connection(config: Config) -> dict[str, Any]: + existing = find_storage_connection(config) + if existing is not None: + return existing + if not config.create_connection_if_missing: + raise RegressionError( + "missing storage connection. Set STORAGE_CONNECTION_ID / STORAGE_CONNECTION_NAME, " + "or rerun with --create-connection-if-missing plus BOS_ACCESS_KEY_ID / BOS_SECRET_ACCESS_KEY." + ) + missing = [] + if not config.bos_access_key_id: + missing.append("BOS_ACCESS_KEY_ID") + if not config.bos_secret_access_key: + missing.append("BOS_SECRET_ACCESS_KEY") + if missing: + raise RegressionError(f"cannot create storage connection; missing env: {', '.join(missing)}") + payload = { + "name": config.storage_connection_name or DEFAULT_CONNECTION_NAME, + "description": "Created by scripts/wjzh_operator_regression.py", + "provider": "BOS", + "endpoint": config.bos_endpoint, + "bucket": config.bos_bucket, + "accessKeyId": config.bos_access_key_id, + "secretAccessKey": config.bos_secret_access_key, + } + return request(config, "POST", "/api/storage/connections", payload, expected=(201,)) + + +def settings_has_workspace(settings: dict[str, Any]) -> bool: + k8s = settings.get("k8s") if isinstance(settings, dict) else None + if not isinstance(k8s, dict): + return False + value = str(k8s.get("workspacePVCName", "") or "").strip() + return value != "" + + +def validate_preflight( + config: Config, + connection: dict[str, Any], + operators: list[dict[str, Any]], + settings: dict[str, Any], + mongo_connection: dict[str, Any] | None, +) -> None: + source_bucket, _ = parse_bos_path(config.source_path) + upload_bucket, _ = parse_bos_path(config.upload_root_prefix) + issues: list[str] = [] + + if not settings_has_workspace(settings): + issues.append( + "settings.k8s.workspacePVCName is missing. Deploy the current api/worker version and configure WORKSPACE_PVC_NAME / workspacePVCName first." + ) + if "enabled" in (settings.get("k8s") or {}): + issues.append( + "live /api/settings still returns k8s.enabled, which indicates the cluster is likely running an older server build than this repository." + ) + if str(connection.get("provider", "")).strip().upper() != "BOS": + issues.append("storage connection provider must be BOS") + connection_bucket = str(connection.get("bucket", "")).strip() + if connection_bucket and connection_bucket != source_bucket: + issues.append( + f"storage connection bucket mismatch: connection bucket={connection_bucket}, source bucket={source_bucket}" + ) + if connection_bucket and connection_bucket != upload_bucket: + issues.append( + f"storage connection bucket mismatch: connection bucket={connection_bucket}, upload bucket={upload_bucket}" + ) + if mongo_connection is None: + issues.append("no Mongo connection available for MongoDB 写入 real regression") + + by_name = {item.get("name"): item for item in operators} + missing_ops = [name for name in required_operators() if name not in by_name] + if missing_ops: + issues.append(f"missing operators: {', '.join(missing_ops)}") + + if issues: + message = { + "summary": "preflight failed", + "issues": issues, + "requiredConfiguration": [ + "deploy current api and worker images from this repository", + "configure shared workspace PVC via WORKSPACE_PVC_NAME / settings.k8s.workspacePVCName", + "ensure a BOS storage connection exists for bucket wjzh-data-migration", + ], + } + raise RegressionError(json.dumps(message, ensure_ascii=False, indent=2)) + + +def build_node(operator: dict[str, Any], node_id: str, x: int, y: int, overrides: dict[str, str] | None = None, *, connection_id: str = "", label: str | None = None) -> dict[str, Any]: + hyperparams = [dict(item) for item in operator.get("hyperparams", [])] + overrides = overrides or {} + seen: set[str] = set() + for item in hyperparams: + key = str(item.get("key", "")).strip() + if key in overrides: + item["value"] = overrides[key] + seen.add(key) + for key, value in overrides.items(): + if key in seen: + continue + hyperparams.append({"key": key, "value": value}) + return { + "id": node_id, + "type": "operator", + "data": { + "label": label or operator["name"], + "category": operator["category"], + "description": operator["description"], + "inputs": operator.get("inputs", []), + "outputs": operator.get("outputs", []), + "image": operator.get("image", ""), + "skillDefinition": operator.get("skillDefinition", ""), + "hyperparams": hyperparams, + "connectionId": connection_id, + }, + "position": {"x": x, "y": y}, + } + + +def upload_node(operator: dict[str, Any], connection_id: str, node_id: str, x: int, y: int, output_path: str) -> dict[str, Any]: + return build_node( + operator, + node_id, + x, + y, + { + "DATA_SOURCE": "BOS", + "OUTPUT_PATH": output_path, + "SOURCE_PATH": output_path, + }, + connection_id=connection_id, + ) + + +def create_project(config: Config, name: str, description: str) -> dict[str, Any]: + return request(config, "POST", "/api/projects", {"name": name, "description": description}, expected=(201,)) + + +def create_workflow( + config: Config, + project_id: str, + name: str, + description: str, + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]], + *, + workflow_type: str = "batch", + kind: str = "", + schedule: dict[str, Any] | None = None, + scan_config: dict[str, Any] | None = None, + max_concurrent_runs: int | None = None, + node_timeout_seconds: int | None = None, + run_timeout_seconds: int | None = None, + max_retry_attempts: int | None = None, + retry_backoff_seconds: int | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "name": name, + "description": description, + "type": workflow_type, + "nodes": nodes, + "edges": edges, + } + if kind.strip(): + payload["kind"] = kind.strip() + if schedule is not None: + payload["schedule"] = schedule + if scan_config is not None: + payload["scanConfig"] = scan_config + if max_concurrent_runs is not None: + payload["maxConcurrentRuns"] = int(max_concurrent_runs) + if node_timeout_seconds is not None: + payload["nodeTimeoutSeconds"] = int(node_timeout_seconds) + if run_timeout_seconds is not None: + payload["runTimeoutSeconds"] = int(run_timeout_seconds) + if max_retry_attempts is not None: + payload["maxRetryAttempts"] = int(max_retry_attempts) + if retry_backoff_seconds is not None: + payload["retryBackoffSeconds"] = int(retry_backoff_seconds) + return request( + config, + "POST", + f"/api/projects/{project_id}/workflows", + payload, + expected=(201,), + ) + + +def get_workflow(config: Config, workflow_id: str) -> dict[str, Any]: + return request(config, "GET", f"/api/workflows/{workflow_id}") + + +def update_workflow(config: Config, workflow_id: str, payload: dict[str, Any]) -> dict[str, Any]: + return request(config, "PUT", f"/api/workflows/{workflow_id}", payload, expected=(200,)) + + +def clone_workflow(config: Config, project_id: str, workflow_id: str, name: str) -> dict[str, Any]: + workflow = get_workflow(config, workflow_id) + return create_workflow( + config, + project_id, + name, + str(workflow.get("description", "")).strip(), + workflow.get("nodes", []), + workflow.get("edges", []), + workflow_type=str(workflow.get("type", "batch") or "batch"), + kind=str(workflow.get("kind", "") or ""), + schedule=workflow.get("schedule"), + scan_config=workflow.get("scanConfig"), + max_concurrent_runs=workflow.get("maxConcurrentRuns"), + node_timeout_seconds=workflow.get("nodeTimeoutSeconds"), + run_timeout_seconds=workflow.get("runTimeoutSeconds"), + max_retry_attempts=workflow.get("maxRetryAttempts"), + retry_backoff_seconds=workflow.get("retryBackoffSeconds"), + ) + + +def run_workflow(config: Config, workflow_id: str) -> dict[str, Any]: + return request(config, "POST", f"/api/workflows/{workflow_id}/run", {"trigger": "manual"}, expected=(201, 202)) + + +def list_runs(config: Config, status: str = "") -> list[dict[str, Any]]: + query = f"?status={urllib.parse.quote(status)}" if status else "" + return request(config, "GET", f"/api/runs{query}")["items"] + + +def get_run(config: Config, run_id: str) -> dict[str, Any]: + return request(config, "GET", f"/api/runs/{run_id}") + + +def get_run_logs(config: Config, run_id: str) -> list[dict[str, Any]]: + return request(config, "GET", f"/api/runs/{run_id}/logs")["items"] + + +def wait_for_run(config: Config, run_id: str, timeout_seconds: int) -> dict[str, Any]: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + item = get_run(config, run_id) + if item["status"] in {"completed", "failed", "cancelled", "waiting_for_audit"}: + return item + time.sleep(5) + raise RegressionError(f"timeout waiting for run {run_id}") + + +def wait_for_next_workflow_run( + config: Config, + workflow_id: str, + previous_run_id: str, + timeout_seconds: int, +) -> dict[str, Any]: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + workflow = get_workflow(config, workflow_id) + current_run_id = str(workflow.get("lastRunId", "")).strip() + if current_run_id and current_run_id != previous_run_id: + return wait_for_run(config, current_run_id, timeout_seconds) + time.sleep(3) + raise RegressionError(f"timeout waiting for next scheduled run of workflow {workflow_id}") + + +def delete_projects(config: Config) -> None: + projects = request(config, "GET", "/api/projects")["items"] + for project in projects: + name = str(project.get("name", "")).strip() + if config.delete_all_projects: + request(config, "DELETE", f"/api/projects/{project['id']}", expected=(204,)) + continue + if config.delete_regression_projects and name.startswith(config.project_prefix): + request(config, "DELETE", f"/api/projects/{project['id']}", expected=(204,)) + + +def upload_edge(source: str, target: str, edge_id: str | None = None, source_handle: str = "out-1", target_handle: str = "in-1") -> dict[str, str]: + return { + "id": edge_id or f"{source}-{target}", + "source": source, + "sourceHandle": source_handle, + "target": target, + "targetHandle": target_handle, + } + + +def list_audit_tasks(config: Config, status: str = "all") -> list[dict[str, Any]]: + return request(config, "GET", f"/api/audit-tasks?status={urllib.parse.quote(status)}")["items"] + + +def wait_for_audit_task(config: Config, run_id: str, timeout_seconds: int) -> dict[str, Any]: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + tasks = list_audit_tasks(config, "all") + for task in tasks: + if task.get("runId") == run_id and task.get("status") == "pending": + return task + time.sleep(2) + raise RegressionError(f"timeout waiting for audit task of run {run_id}") + + +def resolve_audit_task(config: Config, task_id: str, decision: str) -> dict[str, Any]: + decision = decision.strip().lower() + if decision not in {"approve", "reject"}: + raise RegressionError(f"unsupported audit decision: {decision}") + return request(config, "POST", f"/api/audit-tasks/{task_id}/{decision}", expected=(200,)) + + +def query_business_collection(config: Config, connection_id: str, collection: str, limit: int = 20) -> dict[str, Any]: + query = urllib.parse.urlencode( + { + "target": "business", + "connectionId": connection_id, + "limit": str(limit), + "skip": "0", + } + ) + return request(config, "GET", f"/api/database/collections/{urllib.parse.quote(collection)}/documents?{query}") + + +def cap_duration_command(seconds: int, *, max_items: int | None = None, max_duration: float | None = None) -> str: + return f""" +python3 - <<'PY' +import json +import os +import pathlib + +input_path = pathlib.Path(os.environ['INPUT_ARTIFACTS_PATH']) +output_dir = pathlib.Path(os.environ['OUTPUT_DIR']) +artifacts = json.loads(input_path.read_text(encoding='utf-8')) + + +def load_payload(path_value: str): + path = pathlib.Path(str(path_value or '').strip()) + if not path_value or path.suffix.lower() != '.json' or not path.exists(): + return None + try: + return json.loads(path.read_text(encoding='utf-8')) + except Exception: + return None + + +payload = None +for artifact in artifacts: + data = artifact.get('data') + if isinstance(data, dict) and isinstance(data.get('items'), list): + payload = json.loads(json.dumps(data, ensure_ascii=False)) + break + if isinstance(data, dict): + data_ref = data.get('dataRef') if isinstance(data.get('dataRef'), dict) else {{}} + payload_path = str(data_ref.get('payloadPath', '')).strip() + loaded = load_payload(payload_path) + if isinstance(loaded, dict) and isinstance(loaded.get('items'), list): + payload = loaded + break + loaded = load_payload(str(artifact.get('path', '')).strip()) + if isinstance(loaded, dict) and isinstance(loaded.get('items'), list): + payload = loaded + break +if payload is None: + payload = {{}} +payload = json.loads(json.dumps(payload, ensure_ascii=False)) +items = payload.get('items') +if not isinstance(items, list): + items = [] + payload['items'] = items +if {max_items if max_items is not None else 'None'} is not None: + items = items[:{max_items if max_items is not None else 0}] + payload['items'] = items +if {repr(max_duration if max_duration is not None else None)} is not None: + items = [item for item in items if float(item.get('duration') or 0.0) <= float({max_duration if max_duration is not None else 0.0})] + payload['items'] = items +for item in items: + if not isinstance(item, dict): + continue + duration = float(item.get('duration') or 0.0) + item['duration'] = min(duration, {seconds}) if duration > 0 else {seconds} +payload['datasetType'] = 'x-platform-windowed' +payload['windowSeconds'] = {seconds} +payload['count'] = len(items) +payload['itemsCount'] = len(items) +payload['videoCount'] = len(items) +(output_dir / 'windowed-videos.json').write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8') +PY +""".strip() + + +def model_inference_command() -> str: + return """ +python3 - <<'PY' +import json +import os +import pathlib + +inputs = json.loads(pathlib.Path(os.environ["INPUT_ARTIFACTS_PATH"]).read_text(encoding="utf-8")) +summary = { + "operator": "模型推理", + "inputArtifactCount": len(inputs), + "artifactNames": [str(item.get("name", "")) for item in inputs][:20], +} +pathlib.Path(os.environ["OUTPUT_DIR"]).joinpath("model-inference-summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2), + encoding="utf-8", +) +PY +""".strip() + + +def robotics_fixture_command() -> str: + return """ +python3 - <<'PY' +import json +import os +import pathlib +import shutil +import sqlite3 +import tempfile + +inputs = json.loads(pathlib.Path(os.environ["INPUT_ARTIFACTS_PATH"]).read_text(encoding="utf-8")) +payload = None +for artifact in inputs: + data = artifact.get("data") + if isinstance(data, dict) and isinstance(data.get("collections"), list): + payload = data + break +if payload is None: + payload = {"collections": [], "items": []} +root = pathlib.Path(os.environ["OUTPUT_DIR"]) +lerobot_root = root / "lerobot-fixture" +(lerobot_root / "meta").mkdir(parents=True, exist_ok=True) +(lerobot_root / "data").mkdir(parents=True, exist_ok=True) +(lerobot_root / "videos").mkdir(parents=True, exist_ok=True) +(lerobot_root / "meta" / "info.json").write_text(json.dumps({"robot_type": "aloha", "fps": 30, "sourceCollections": len(payload.get("collections", []))}, ensure_ascii=False, indent=2), encoding="utf-8") +(lerobot_root / "meta" / "stats.json").write_text(json.dumps({"episodes": len(payload.get("items", []))}, ensure_ascii=False, indent=2), encoding="utf-8") +(lerobot_root / "meta" / "tasks.jsonl").write_text("{\\"task\\":\\"pick\\"}\\n{\\"task\\":\\"place\\"}\\n", encoding="utf-8") +(lerobot_root / "data" / "episode_000.parquet").write_bytes(b"PAR1") +(lerobot_root / "videos" / "episode_000.mp4").write_bytes(b"video") + +rosbag_root = root / "rosbag-fixture" +rosbag_root.mkdir(parents=True, exist_ok=True) +with tempfile.TemporaryDirectory() as temp_dir: + db_path = pathlib.Path(temp_dir) / "bag_0.db3" + connection = sqlite3.connect(db_path) + cursor = connection.cursor() + cursor.execute("create table topics(id integer primary key, name text, type text, serialization_format text, offered_qos_profiles text)") + cursor.execute("create table messages(id integer primary key, topic_id integer, timestamp integer, data blob)") + cursor.execute("insert into topics(id, name, type, serialization_format, offered_qos_profiles) values (1, '/camera', 'sensor_msgs/msg/Image', 'cdr', '')") + cursor.execute("insert into messages(id, topic_id, timestamp, data) values (1, 1, 123456789, X'00')") + connection.commit() + connection.close() + shutil.copy2(db_path, rosbag_root / "bag_0.db3") +(rosbag_root / "metadata.yaml").write_text("rosbag2_bagfile_information:\\n version: 5\\n", encoding="utf-8") +PY +""".strip() + + +def ffprobe_video_metadata(payload: bytes, filename: str) -> dict[str, Any]: + suffix = os.path.splitext(filename)[1] or ".mp4" + try: + with tempfile.NamedTemporaryFile(suffix=suffix) as handle: + handle.write(payload) + handle.flush() + probe = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=codec_name,width,height,avg_frame_rate:format=duration,bit_rate", + "-of", + "json", + handle.name, + ], + check=True, + capture_output=True, + text=True, + ) + data = json.loads(probe.stdout or "{}") + except (FileNotFoundError, subprocess.CalledProcessError, json.JSONDecodeError): + return { + "filename": filename, + "codec": "", + "resolution": "", + "frameRate": 0.0, + "bitrate": 0, + "duration": 0.0, + "startTime": int(time.time()), + } + streams = data.get("streams") or [] + stream = streams[0] if streams else {} + fmt = data.get("format") or {} + frame_rate = 0.0 + avg_frame_rate = str(stream.get("avg_frame_rate", "0/1")).strip() + if "/" in avg_frame_rate: + numerator, denominator = avg_frame_rate.split("/", 1) + try: + denominator_value = float(denominator or 0.0) + if denominator_value: + frame_rate = float(numerator or 0.0) / denominator_value + except ValueError: + frame_rate = 0.0 + return { + "filename": filename, + "codec": stream.get("codec_name") or "", + "resolution": f"{int(stream.get('width') or 0)}x{int(stream.get('height') or 0)}" if stream.get("width") and stream.get("height") else "", + "frameRate": round(frame_rate, 3) if frame_rate else 0.0, + "bitrate": int(float(fmt.get("bit_rate") or 0)), + "duration": round(float(fmt.get("duration") or 0.0), 3), + "startTime": int(time.time()), + } + + +def ensure_incremental_seed_directories( + config: Config, + connection_id: str, + seed_prefix: str, + source_clip_keys: list[str], + *, + bucket: str, + round_name: str, + directory_count: int, +) -> list[str]: + created: list[str] = [] + if directory_count <= 0: + return created + normalized_seed_prefix = seed_prefix.strip().strip("/") + for index in range(directory_count): + clip_key = source_clip_keys[index % len(source_clip_keys)] + clip_payload = object_storage_read(config, connection_id, clip_key, bucket=bucket) + clip_name = os.path.basename(clip_key) + dir_name = f"{round_name}_{index+1:02d}_OsmoNano_{time.strftime('%Y-%m-%d_%H-%M-%S')}" + base_key = f"{normalized_seed_prefix}/{dir_name}" + video_key = f"{base_key}/DJI_001/{clip_name}" + object_storage_upload(config, connection_id, video_key, clip_payload, bucket=bucket, filename=clip_name, content_type="video/mp4") + video_meta = [ffprobe_video_metadata(clip_payload, clip_name)] + meta = { + "type": "工厂", + "place": "绿控传动", + "mission": f"增量回归-{round_name}", + "project_id": dir_name, + "date": time.strftime("%Y%m%d"), + "device_id": f"REG{index+1:04d}", + } + intrinsics = { + "camera_model": "OsmoNano", + "fx": 1.0, + "fy": 1.0, + "cx": 0.5, + "cy": 0.5, + } + object_storage_upload( + config, + connection_id, + f"{base_key}/meta.json", + json.dumps(meta, ensure_ascii=False, indent=2).encode("utf-8"), + bucket=bucket, + filename="meta.json", + content_type="application/json", + ) + object_storage_upload( + config, + connection_id, + f"{base_key}/video_metadata.json", + json.dumps(video_meta, ensure_ascii=False, indent=2).encode("utf-8"), + bucket=bucket, + filename="video_metadata.json", + content_type="application/json", + ) + object_storage_upload( + config, + connection_id, + f"{base_key}/intrinsics.json", + json.dumps(intrinsics, ensure_ascii=False, indent=2).encode("utf-8"), + bucket=bucket, + filename="intrinsics.json", + content_type="application/json", + ) + created.append(dir_name) + return created + + +def list_uploaded_mp4_keys(config: Config, connection_id: str, output_root: str, suffix: str) -> list[str]: + bucket, root_prefix = parse_bos_path(output_root) + prefix = f"{root_prefix.strip('/')}/{suffix.strip('/')}".strip("/") + items = object_storage_list(config, connection_id, prefix, bucket=bucket) + return [str(item.get("key", "")).strip() for item in items if str(item.get("kind", "")).strip() == "video" and str(item.get("key", "")).strip().lower().endswith(".mp4")] + + +def create_runtime_smoke(config: Config, connection_id: str, operators_by_name: dict[str, dict[str, Any]]) -> tuple[str, str]: + timestamp = time.strftime("%Y%m%d%H%M%S") + project = create_project(config, f"{config.project_prefix} smoke {timestamp}", "Temporary production-style storage connection smoke test") + nodes = [ + build_node( + operators_by_name["BOS 增量扫描"], + "source", + 0, + 0, + {"SOURCE_PATH": config.source_path, "SCAN_MODE": "FULL"}, + connection_id=connection_id, + ), + build_node(operators_by_name["X 平台数据集索引"], "index", 220, 0), + build_node(operators_by_name["采集目录校验"], "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + upload_node( + operators_by_name["结果上传"], + connection_id, + "upload", + 660, + 0, + f"{config.upload_root_prefix}/{timestamp}/smoke-connid", + ), + ] + edges = [ + {"id": "source-index", "source": "source", "sourceHandle": "out-1", "target": "index", "targetHandle": "in-1"}, + {"id": "index-validate", "source": "index", "sourceHandle": "out-1", "target": "validate", "targetHandle": "in-1"}, + {"id": "validate-upload", "source": "validate", "sourceHandle": "out-1", "target": "upload", "targetHandle": "in-1"}, + ] + workflow = create_workflow(config, project["id"], "connid-smoke", "Validate storage connection runtime path", nodes, edges) + return project["id"], workflow["id"] + + +def create_regression_workflows( + config: Config, + connection_id: str, + mongo_connection_id: str, + operators_by_name: dict[str, dict[str, Any]], +) -> tuple[str, list[dict[str, Any]], str, str, str]: + timestamp = time.strftime("%Y%m%d%H%M%S") + output_root = f"{config.upload_root_prefix}/{timestamp}" + mongo_collection = f"{config.mongo_collection_prefix}_{timestamp}".replace("-", "_") + source_bucket, source_prefix = parse_bos_path(config.source_path) + incremental_prefix = f"{source_prefix}/__codex_incremental_regression__/{timestamp}".strip("/") + incremental_source_path = f"bos://{source_bucket}/{incremental_prefix}" + project = create_project(config, f"{config.project_prefix} {timestamp}", "使用真实 WJZH 数据做全算子批量回归") + + source_operator = operators_by_name["BOS 增量扫描"] + real_source_operator = operators_by_name["真实数据源"] + directory_scan_operator = operators_by_name["目录扫描"] + model_operator = operators_by_name["模型推理"] + index_operator = operators_by_name["X 平台数据集索引"] + validate_operator = operators_by_name["采集目录校验"] + upload_operator = operators_by_name["结果上传"] + shell_operator = operators_by_name["Shell 命令执行"] + json_filter_operator = operators_by_name["JSON 过滤"] + + def scan_source(node_id: str = "source", x: int = 0, y: int = 0) -> dict[str, Any]: + return build_node( + source_operator, + node_id, + x, + y, + {"SOURCE_PATH": config.source_path, "SCAN_MODE": "FULL"}, + connection_id=connection_id, + ) + + def scan_incremental(node_id: str = "source", x: int = 0, y: int = 0) -> dict[str, Any]: + return build_node( + source_operator, + node_id, + x, + y, + {"SOURCE_PATH": incremental_source_path, "SCAN_MODE": "INCREMENTAL"}, + connection_id=connection_id, + ) + + def real_source(node_id: str, x: int, y: int) -> dict[str, Any]: + return build_node( + real_source_operator, + node_id, + x, + y, + {"SOURCE_PATH": config.source_path, "DATA_SOURCE": "BOS"}, + connection_id=connection_id, + ) + + def named_upload(node_id: str, x: int, y: int, suffix: str) -> dict[str, Any]: + return upload_node(upload_operator, connection_id, node_id, x, y, f"{output_root}/{suffix}") + + workflows: list[dict[str, Any]] = [] + + wf0_nodes = [ + real_source("source", 0, 0), + build_node(directory_scan_operator, "scan_dir", 240, -90), + build_node(model_operator, "infer", 240, 90, {"COMMAND": model_inference_command()}), + named_upload("dir_upload", 500, -90, "00-source-directory/scan"), + named_upload("infer_upload", 500, 90, "00-source-directory/infer"), + ] + wf0_edges = [ + upload_edge("source", "scan_dir", "source-dir"), + upload_edge("source", "infer", "source-infer"), + upload_edge("scan_dir", "dir_upload", "dir-upload"), + upload_edge("infer", "infer_upload", "infer-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "00-源数据目录推理", "真实数据源、目录扫描和模型推理回归", wf0_nodes, wf0_edges), + } + ) + + wf1_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, -70), + build_node(validate_operator, "validate", 440, -70, {"REQUIRE_INTRINSICS": "false"}), + named_upload("index_upload", 680, -140, "01-index-validate/index"), + named_upload("validate_upload", 680, 0, "01-index-validate/validate"), + ] + wf1_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("index", "index_upload", "index-upload"), + upload_edge("validate", "validate_upload", "validate-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "01-索引校验双上传", "扫描全量真实数据并分别上传索引与校验结果", wf1_nodes, wf1_edges), + } + ) + + wf2_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(operators_by_name["JSON 字段提取"], "extract", 460, -180, {"FIELD_PATH": "collections.0.meta.place"}), + build_node(operators_by_name["文本模板渲染"], "render", 680, -180, {"TEMPLATE_TEXT": "place={{value}}", "OUTPUT_NAME": "place.txt"}), + build_node(json_filter_operator, "duration_filter", 460, -20, {"FIELD_PATH": "duration", "OPERATOR": "lte", "EXPECTED_JSON": "700"}), + build_node( + operators_by_name["JSON 映射重命名"], + "rename", + 680, + -20, + {"FIELD_MAP_JSON": "{\"collectionId\":\"collection_id\",\"filename\":\"video_name\",\"duration\":\"video_duration\"}"}, + ), + build_node(operators_by_name["JSON 转 CSV"], "to_csv", 900, -20), + build_node(operators_by_name["CSV 转 JSON"], "to_json", 1120, -20), + build_node(operators_by_name["条件分流"], "route", 460, 150, {"FIELD_PATH": "meta.device_id", "OPERATOR": "eq", "EXPECTED": "C00009"}), + build_node(operators_by_name["数据合并"], "merge", 1340, 40), + named_upload("render_upload", 1560, -180, "02-json-toolchain/render"), + named_upload("table_upload", 1560, -20, "02-json-toolchain/table"), + named_upload("matched_upload", 900, 130, "02-json-toolchain/route-matched"), + named_upload("unmatched_upload", 900, 250, "02-json-toolchain/route-unmatched"), + named_upload("merge_upload", 1560, 120, "02-json-toolchain/merged"), + ] + wf2_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "extract", "index-extract"), + upload_edge("extract", "render", "extract-render"), + upload_edge("render", "render_upload", "render-upload"), + upload_edge("index", "duration_filter", "index-duration-filter"), + upload_edge("duration_filter", "rename", "filter-rename"), + upload_edge("rename", "to_csv", "rename-csv"), + upload_edge("to_csv", "to_json", "csv-json"), + upload_edge("to_json", "table_upload", "table-upload"), + upload_edge("index", "route", "index-route"), + upload_edge("route", "matched_upload", "route-matched-upload", source_handle="matched"), + upload_edge("route", "unmatched_upload", "route-unmatched-upload", source_handle="unmatched"), + upload_edge("route", "merge", "route-merge", source_handle="matched", target_handle="in-1"), + upload_edge("to_json", "merge", "json-merge", target_handle="in-2"), + upload_edge("merge", "merge_upload", "merge-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "02-JSON工具链多分支", "真实清单数据上的 JSON/CSV/分流/合并回归", wf2_nodes, wf2_edges), + } + ) + + def audit_workflow(name: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + build_node(json_filter_operator, "pick_one", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": "DJI_20260327132547_0003_D.MP4"}), + build_node(operators_by_name["人工审核"], "review", 880, 0), + named_upload("approved_upload", 1120, -80, f"03-audit/{name}/approved"), + named_upload("rejected_upload", 1120, 80, f"03-audit/{name}/rejected"), + ] + edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("validate", "pick_one", "validate-pick"), + upload_edge("pick_one", "review", "pick-review"), + upload_edge("review", "approved_upload", "review-approved", source_handle="approved"), + upload_edge("review", "rejected_upload", "review-rejected", source_handle="rejected"), + ] + return nodes, edges + + audit_approve_nodes, audit_approve_edges = audit_workflow("approve") + workflows.append( + { + "workflow": create_workflow(config, project["id"], "03-人工审核通过", "真实数据人工审核 approve 分支回归", audit_approve_nodes, audit_approve_edges), + "auditDecision": "approve", + } + ) + audit_reject_nodes, audit_reject_edges = audit_workflow("reject") + workflows.append( + { + "workflow": create_workflow(config, project["id"], "04-人工审核拒绝", "真实数据人工审核 reject 分支回归", audit_reject_nodes, audit_reject_edges), + "auditDecision": "reject", + } + ) + + wf5_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + build_node(json_filter_operator, "pick_medium", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": MEDIUM_VIDEO}), + build_node( + operators_by_name["MongoDB 写入"], + "mongo_write", + 880, + 0, + { + "MONGO_CONNECTION_ID": mongo_connection_id, + "MONGO_COLLECTION": mongo_collection, + "MONGO_MODE": "insert_many", + "DOCUMENT_PATH": "items", + }, + ), + named_upload("mongo_upload", 1120, 0, "05-mongo-write"), + ] + wf5_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("validate", "pick_medium", "validate-pick"), + upload_edge("pick_medium", "mongo_write", "pick-mongo"), + upload_edge("mongo_write", "mongo_upload", "mongo-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "05-Mongo写入", "真实视频清单写入业务 Mongo 并上传写入结果", wf5_nodes, wf5_edges), + "mongoCollection": mongo_collection, + } + ) + + wf6_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + build_node(json_filter_operator, "pick_medium", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": MEDIUM_VIDEO}), + build_node(operators_by_name["视频质检"], "quality", 880, -60, {"MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}), + build_node(shell_operator, "window_medium", 1100, -60, {"COMMAND": cap_duration_command(45, max_items=1, max_duration=120)}, label="中视频窗口采样"), + build_node(operators_by_name["视频切片"], "slice", 1320, -60, {"SEGMENT_DURATION_SECONDS": "45"}), + named_upload("quality_upload", 1540, -140, "06-medium-video/quality"), + named_upload("slice_upload", 1540, 20, "06-medium-video/slice"), + ] + wf6_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("validate", "pick_medium", "validate-pick"), + upload_edge("pick_medium", "quality", "pick-quality"), + upload_edge("quality", "window_medium", "quality-window"), + upload_edge("window_medium", "slice", "window-slice"), + upload_edge("quality", "quality_upload", "quality-upload"), + upload_edge("slice", "slice_upload", "slice-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "06-中大型视频切片", "真实中大型视频的质检和切片回归", wf6_nodes, wf6_edges), + } + ) + + wf7_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + build_node(json_filter_operator, "pick_huge", 660, 0, {"FIELD_PATH": "filename", "OPERATOR": "eq", "EXPECTED": HUGE_VIDEO}), + build_node(operators_by_name["视频质检"], "quality", 880, 0, {"MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}), + build_node( + operators_by_name["数据过滤"], + "filter", + 1100, + 0, + {"DATE_START": "2026-03-01", "DATE_END": "2026-04-30", "MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}, + ), + build_node(shell_operator, "window", 1320, 0, {"COMMAND": cap_duration_command(HUGE_WINDOW_SECONDS)}, label="大文件窗口采样"), + build_node(operators_by_name["视频切片"], "slice", 1540, 0, {"SEGMENT_DURATION_SECONDS": "120"}), + build_node(operators_by_name["视频标注"], "annotate", 1760, 0, {"FRAME_INTERVAL_SECONDS": "1.0"}), + build_node(operators_by_name["成品打包"], "package", 1980, 0), + build_node( + operators_by_name["抽检采样"], + "sample", + 2200, + 0, + { + "RULES_JSON": json.dumps( + [ + {"id": "low-clarity", "field": "quality.clarity", "operator": "lt", "value": 0.5, "samplePercent": 10}, + {"id": "low-confidence", "field": "annotation.minConfidence", "operator": "lt", "value": 0.7, "samplePercent": 10}, + {"id": "high-shake", "field": "quality.shake", "operator": "gt", "value": 0.15, "samplePercent": 10}, + {"id": "high-hand-occlusion", "field": "quality.handOcclusion", "operator": "gt", "value": 0.15, "samplePercent": 10}, + ], + ensure_ascii=False, + ) + }, + ), + named_upload("quality_upload", 2460, -240, "07-full-chain/quality"), + named_upload("filter_upload", 2460, -160, "07-full-chain/filter"), + named_upload("window_upload", 2460, -80, "07-full-chain/window"), + named_upload("slice_upload", 2460, 0, "07-full-chain/slice"), + named_upload("annotate_upload", 2460, 80, "07-full-chain/annotate"), + named_upload("package_upload", 2460, 160, "07-full-chain/package"), + named_upload("sample_upload", 2460, 240, "07-full-chain/sample"), + ] + wf7_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("validate", "pick_huge", "validate-pick"), + upload_edge("pick_huge", "quality", "pick-quality"), + upload_edge("quality", "filter", "quality-filter"), + upload_edge("filter", "window", "filter-window"), + upload_edge("window", "slice", "window-slice"), + upload_edge("slice", "annotate", "slice-annotate"), + upload_edge("annotate", "package", "annotate-package"), + upload_edge("package", "sample", "package-sample"), + upload_edge("quality", "quality_upload", "quality-upload"), + upload_edge("filter", "filter_upload", "filter-upload"), + upload_edge("window", "window_upload", "window-upload"), + upload_edge("slice", "slice_upload", "slice-upload"), + upload_edge("annotate", "annotate_upload", "annotate-upload"), + upload_edge("package", "package_upload", "package-upload"), + upload_edge("sample", "sample_upload", "sample-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "07-超大视频全链路", "17GB 级真实视频的完整处理链路回归", wf7_nodes, wf7_edges), + } + ) + + wf8_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + build_node(json_filter_operator, "pick_batch", 660, 0, {"FIELD_PATH": "duration", "OPERATOR": "lte", "EXPECTED_JSON": "400"}), + build_node(shell_operator, "window_batch", 880, 0, {"COMMAND": cap_duration_command(45, max_items=4)}, label="批量窗口采样"), + build_node(operators_by_name["视频质检"], "quality", 1100, 0, {"MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}), + build_node(operators_by_name["视频切片"], "slice", 1320, 0, {"SEGMENT_DURATION_SECONDS": "30"}), + build_node(operators_by_name["视频标注"], "annotate", 1540, 0, {"FRAME_INTERVAL_SECONDS": "1.0"}), + build_node(operators_by_name["成品打包"], "package", 1760, 0), + named_upload("batch_package_upload", 1980, -60, "08-batch-windowed/package"), + named_upload("batch_annotation_upload", 1980, 60, "08-batch-windowed/annotate"), + ] + wf8_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("validate", "pick_batch", "validate-pick"), + upload_edge("pick_batch", "window_batch", "pick-window"), + upload_edge("window_batch", "quality", "window-quality"), + upload_edge("quality", "slice", "quality-slice"), + upload_edge("slice", "annotate", "slice-annotate"), + upload_edge("annotate", "package", "annotate-package"), + upload_edge("annotate", "batch_annotation_upload", "annotate-upload"), + upload_edge("package", "batch_package_upload", "package-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "08-多目录批量窗口化", "跨目录批量视频窗口化、切片、标注和打包回归", wf8_nodes, wf8_edges), + } + ) + + wf9_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(shell_operator, "fixtures", 440, 0, {"COMMAND": robotics_fixture_command()}, label="派生机器人数据集"), + build_node(operators_by_name["LeRobot 数据集概览"], "lerobot", 680, -80), + build_node(operators_by_name["ROSBag 数据概览"], "rosbag", 680, 80), + named_upload("lerobot_upload", 920, -80, "09-robotics/lerobot"), + named_upload("rosbag_upload", 920, 80, "09-robotics/rosbag"), + ] + wf9_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "fixtures", "index-fixtures"), + upload_edge("fixtures", "lerobot", "fixtures-lerobot"), + upload_edge("fixtures", "rosbag", "fixtures-rosbag"), + upload_edge("lerobot", "lerobot_upload", "lerobot-upload"), + upload_edge("rosbag", "rosbag_upload", "rosbag-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "09-Robotics派生概览", "从 WJZH 数据派生 LeRobot 与 ROSBag 最小数据集并回归概览算子", wf9_nodes, wf9_edges), + } + ) + + wf10_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + build_node(shell_operator, "batch_pick", 660, 0, {"COMMAND": cap_duration_command(90, max_items=8, max_duration=500)}, label="多目录多视频抽样"), + build_node(operators_by_name["视频质检"], "quality", 900, 0, {"MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}), + named_upload("batch_pick_upload", 1140, -60, "10-batch-quality/pick"), + named_upload("quality_upload", 1140, 60, "10-batch-quality/quality"), + ] + wf10_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("validate", "batch_pick", "validate-batch-pick"), + upload_edge("batch_pick", "quality", "batch-pick-quality"), + upload_edge("batch_pick", "batch_pick_upload", "batch-pick-upload"), + upload_edge("quality", "quality_upload", "quality-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "10-多目录多视频批量质检", "跨目录抽样多个真实视频,批量跑视频质检并上传结果", wf10_nodes, wf10_edges), + "rounds": 2, + } + ) + + wf11_nodes = [ + scan_source(), + build_node(index_operator, "index", 220, 0), + build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + build_node(shell_operator, "batch_pick", 660, 0, {"COMMAND": cap_duration_command(45, max_items=6, max_duration=500)}, label="多目录全链路抽样"), + build_node(operators_by_name["视频质检"], "quality", 900, -80, {"MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}), + build_node(operators_by_name["数据过滤"], "filter", 1120, -80, {"DATE_START": "2026-01-01", "DATE_END": "2026-12-31", "MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}), + build_node(operators_by_name["视频切片"], "slice", 1340, -80, {"SEGMENT_DURATION_SECONDS": "30"}), + build_node(operators_by_name["视频标注"], "annotate", 1560, -80, {"FRAME_INTERVAL_SECONDS": "1.0"}), + build_node(operators_by_name["成品打包"], "package", 1780, -80), + build_node( + operators_by_name["抽检采样"], + "sample", + 2000, + -80, + { + "RULES_JSON": json.dumps( + [ + {"id": "low-clarity", "field": "quality.clarity", "operator": "lt", "value": 0.5, "samplePercent": 10}, + {"id": "high-shake", "field": "quality.shake", "operator": "gt", "value": 0.15, "samplePercent": 10}, + ], + ensure_ascii=False, + ) + }, + ), + named_upload("pick_upload", 2240, -220, "11-batch-fullchain/pick"), + named_upload("quality_upload", 2240, -140, "11-batch-fullchain/quality"), + named_upload("slice_upload", 2240, -60, "11-batch-fullchain/slice"), + named_upload("annotate_upload", 2240, 20, "11-batch-fullchain/annotate"), + named_upload("package_upload", 2240, 100, "11-batch-fullchain/package"), + named_upload("sample_upload", 2240, 180, "11-batch-fullchain/sample"), + ] + wf11_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("validate", "batch_pick", "validate-batch-pick"), + upload_edge("batch_pick", "quality", "batch-pick-quality"), + upload_edge("quality", "filter", "quality-filter"), + upload_edge("filter", "slice", "filter-slice"), + upload_edge("slice", "annotate", "slice-annotate"), + upload_edge("annotate", "package", "annotate-package"), + upload_edge("package", "sample", "package-sample"), + upload_edge("batch_pick", "pick_upload", "pick-upload"), + upload_edge("quality", "quality_upload", "quality-upload"), + upload_edge("slice", "slice_upload", "slice-upload"), + upload_edge("annotate", "annotate_upload", "annotate-upload"), + upload_edge("package", "package_upload", "package-upload"), + upload_edge("sample", "sample_upload", "sample-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "11-多目录多视频批量全链路", "跨目录抽样多个真实视频,批量跑过滤/切片/标注/打包/抽检", wf11_nodes, wf11_edges), + "rounds": 2, + } + ) + + wf12_nodes = [ + scan_incremental(), + build_node(index_operator, "index", 220, 0), + build_node(validate_operator, "validate", 440, 0, {"REQUIRE_INTRINSICS": "false"}), + build_node(operators_by_name["视频质检"], "quality", 660, -80, {"MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}), + build_node(operators_by_name["数据过滤"], "filter", 880, -80, {"DATE_START": "2026-01-01", "DATE_END": "2026-12-31", "MIN_CLARITY": "0.0", "MAX_SHAKE": "1.0", "MAX_HAND_OCCLUSION": "1.0"}), + build_node(shell_operator, "window", 1100, -80, {"COMMAND": cap_duration_command(30, max_items=8, max_duration=120)}, label="增量视频窗口抽样"), + build_node(operators_by_name["视频切片"], "slice", 1320, -80, {"SEGMENT_DURATION_SECONDS": "30"}), + build_node(operators_by_name["视频标注"], "annotate", 1540, -80, {"FRAME_INTERVAL_SECONDS": "1.0"}), + build_node(operators_by_name["成品打包"], "package", 1760, -80), + build_node( + operators_by_name["抽检采样"], + "sample", + 1980, + -80, + { + "RULES_JSON": json.dumps( + [ + {"id": "low-clarity", "field": "quality.clarity", "operator": "lt", "value": 0.5, "samplePercent": 10}, + {"id": "low-confidence", "field": "annotation.minConfidence", "operator": "lt", "value": 0.7, "samplePercent": 10}, + ], + ensure_ascii=False, + ) + }, + ), + named_upload("quality_upload", 2220, -220, "12-incremental-loop/quality"), + named_upload("filter_upload", 2220, -140, "12-incremental-loop/filter"), + named_upload("slice_upload", 2220, -60, "12-incremental-loop/slice"), + named_upload("annotate_upload", 2220, 20, "12-incremental-loop/annotate"), + named_upload("package_upload", 2220, 100, "12-incremental-loop/package"), + named_upload("sample_upload", 2220, 180, "12-incremental-loop/sample"), + ] + wf12_edges = [ + upload_edge("source", "index", "source-index"), + upload_edge("index", "validate", "index-validate"), + upload_edge("validate", "quality", "validate-quality"), + upload_edge("quality", "filter", "quality-filter"), + upload_edge("filter", "window", "filter-window"), + upload_edge("window", "slice", "window-slice"), + upload_edge("slice", "annotate", "slice-annotate"), + upload_edge("annotate", "package", "annotate-package"), + upload_edge("package", "sample", "package-sample"), + upload_edge("quality", "quality_upload", "quality-upload"), + upload_edge("filter", "filter_upload", "filter-upload"), + upload_edge("slice", "slice_upload", "slice-upload"), + upload_edge("annotate", "annotate_upload", "annotate-upload"), + upload_edge("package", "package_upload", "package-upload"), + upload_edge("sample", "sample_upload", "sample-upload"), + ] + workflows.append( + { + "workflow": create_workflow(config, project["id"], "12-增量嗅探循环处理", "在源前缀下持续嗅探新增目录,并对新增目录执行完整处理链", wf12_nodes, wf12_edges), + "incrementalRounds": [ + {"name": "round-1", "directoryCount": 2}, + {"name": "round-2", "directoryCount": 2}, + {"name": "round-3-no-new", "directoryCount": 0}, + ], + "incrementalSourcePath": incremental_source_path, + "streamingIntervalSeconds": 20, + } + ) + + return project["id"], workflows, mongo_collection, output_root, incremental_source_path + + +def summarize_run(run: dict[str, Any]) -> dict[str, Any]: + result = { + "runId": run["id"], + "status": run["status"], + "workflowId": run["workflowId"], + "workflowName": run["workflowName"], + "currentNodeId": run.get("currentNodeId"), + "durationMs": run.get("durationMs"), + } + if run["status"] != "completed": + failed = next((item for item in run.get("nodeExecutions", []) if item.get("status") == "failed"), None) + if failed: + result["failedNode"] = { + "nodeId": failed.get("nodeId"), + "nodeLabel": failed.get("nodeLabel"), + "error": failed.get("error"), + } + result["logsTail"] = get_logs_tail(failed) + return result + + +def get_logs_tail(node_execution: dict[str, Any]) -> list[dict[str, Any]]: + logs = node_execution.get("logs") + if not isinstance(logs, list): + return [] + return logs[-20:] + + +def run_workflow_until_complete( + config: Config, + workflow_id: str, + timeout_seconds: int, + *, + audit_decision: str = "", +) -> dict[str, Any]: + started = run_workflow(config, workflow_id) + if not audit_decision: + return wait_for_run(config, started["id"], timeout_seconds) + + deadline = time.time() + timeout_seconds + while time.time() < deadline: + current = get_run(config, started["id"]) + if current["status"] in {"completed", "failed", "cancelled"}: + return current + if current["status"] == "waiting_for_audit": + task = wait_for_audit_task(config, started["id"], min(300, max(int(deadline - time.time()), 5))) + resolve_audit_task(config, task["id"], audit_decision) + time.sleep(3) + raise RegressionError(f"timeout waiting for run {started['id']}") + + +def main() -> int: + config = parse_args() + settings = request(config, "GET", "/api/settings") + operators = request(config, "GET", "/api/operators")["items"] + connection = ensure_storage_connection(config) + mongo_connection = find_mongo_connection(config) + validate_preflight(config, connection, operators, settings, mongo_connection) + operators_by_name = {item["name"]: item for item in operators} + + if config.preflight_only: + print( + json.dumps( + { + "status": "ok", + "mode": "preflight-only", + "connectionId": connection["id"], + "mongoConnectionId": mongo_connection["id"] if mongo_connection else "", + "sourcePath": config.source_path, + "uploadRootPrefix": config.upload_root_prefix, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + if not config.skip_runtime_smoke: + smoke_project_id, smoke_workflow_id = create_runtime_smoke(config, connection["id"], operators_by_name) + smoke_run = run_workflow(config, smoke_workflow_id) + smoke_result = wait_for_run(config, smoke_run["id"], min(config.max_wait_seconds, 1800)) + if smoke_result["status"] != "completed": + print( + json.dumps( + { + "status": "runtime-smoke-failed", + "projectId": smoke_project_id, + "workflowId": smoke_workflow_id, + "run": summarize_run(smoke_result), + "runLogsTail": get_run_logs(config, smoke_result["id"])[-30:], + }, + ensure_ascii=False, + indent=2, + ), + file=sys.stderr, + ) + return 2 + + delete_projects(config) + project_id, workflows, mongo_collection, output_root, incremental_source_path = create_regression_workflows( + config, + connection["id"], + mongo_connection["id"] if mongo_connection else "", + operators_by_name, + ) + + results = [] + mongo_verification: dict[str, Any] | None = None + incremental_workflow: dict[str, Any] | None = None + for item in workflows: + if item.get("incrementalRounds"): + incremental_workflow = item + continue + workflow = item["workflow"] + for round_index in range(max(int(item.get("rounds", 1) or 1), 1)): + current_workflow = workflow + if round_index > 0: + current_workflow = clone_workflow( + config, + project_id, + workflow["id"], + f"{workflow['name']} 第{round_index + 1}轮", + ) + finished = run_workflow_until_complete( + config, + current_workflow["id"], + config.max_wait_seconds, + audit_decision=str(item.get("auditDecision", "")).strip(), + ) + summary = summarize_run(finished) + if item.get("auditDecision"): + summary["auditDecision"] = item["auditDecision"] + if int(item.get("rounds", 1) or 1) > 1: + summary["round"] = round_index + 1 + results.append(summary) + if finished["status"] != "completed": + print( + json.dumps( + { + "status": "workflow-failed", + "projectId": project_id, + "workflowId": current_workflow["id"], + "workflowName": current_workflow["name"], + "run": summary, + "runLogsTail": get_run_logs(config, finished["id"])[-30:], + }, + ensure_ascii=False, + indent=2, + ), + file=sys.stderr, + ) + return 3 + if item.get("mongoCollection"): + documents = query_business_collection(config, mongo_connection["id"], str(item["mongoCollection"]), limit=50) + mongo_verification = { + "collection": item["mongoCollection"], + "documentCount": documents.get("total", 0), + "sampleDocumentIds": [doc.get("_id") for doc in documents.get("documents", [])[:10]], + } + if int(mongo_verification["documentCount"] or 0) <= 0: + raise RegressionError(f"mongo verification failed for collection {item['mongoCollection']}") + + incremental_verification: dict[str, Any] | None = None + if incremental_workflow is not None: + seed_clip_keys: list[str] = [] + for suffix in ["06-medium-video/slice", "07-full-chain/slice", "11-batch-fullchain/slice"]: + seed_clip_keys.extend(list_uploaded_mp4_keys(config, connection["id"], output_root, suffix)) + deduped_seed_keys = list(dict.fromkeys([key for key in seed_clip_keys if key])) + if not deduped_seed_keys: + raise RegressionError(f"failed to find seed clips under {output_root}") + + incremental_bucket, incremental_prefix = parse_bos_path(incremental_source_path) + round_summaries: list[dict[str, Any]] = [] + incremental_workflow_id = incremental_workflow["workflow"]["id"] + previous_run_id = "" + for round_item in incremental_workflow.get("incrementalRounds", []): + created_directories = ensure_incremental_seed_directories( + config, + connection["id"], + incremental_prefix, + deduped_seed_keys, + bucket=incremental_bucket, + round_name=str(round_item.get("name", "round")).strip() or "round", + directory_count=int(round_item.get("directoryCount", 0) or 0), + ) + if round_summaries: + time.sleep(2) + if not round_summaries: + scan_bucket, scan_prefix = parse_bos_path(incremental_source_path) + update_payload: dict[str, Any] = { + "kind": "scan", + "maxConcurrentRuns": 1, + "incrementalUnit": "DIRECTORY", + "directoryDepth": 1, + "scanConfig": { + "enabled": True, + "connectionId": connection["id"], + "dataSource": "BOS", + "sourcePath": incremental_source_path, + "bucket": scan_bucket, + "prefix": scan_prefix, + "scanMode": "INCREMENTAL", + }, + "type": "streaming", + "schedule": { + "interval": int(incremental_workflow.get("streamingIntervalSeconds", 20) or 20), + }, + } + update_workflow( + config, + incremental_workflow_id, + update_payload, + ) + finished = wait_for_next_workflow_run( + config, + incremental_workflow_id, + previous_run_id, + config.max_wait_seconds, + ) + summary = summarize_run(finished) + summary["incrementalRound"] = round_item.get("name") + summary["createdDirectories"] = created_directories + round_summaries.append(summary) + results.append(summary) + previous_run_id = str(finished.get("id", "")).strip() + if finished["status"] != "completed": + print( + json.dumps( + { + "status": "incremental-workflow-failed", + "projectId": project_id, + "workflowId": incremental_workflow_id, + "workflowName": incremental_workflow["workflow"]["name"], + "run": summary, + "runLogsTail": get_run_logs(config, finished["id"])[-30:], + }, + ensure_ascii=False, + indent=2, + ), + file=sys.stderr, + ) + return 4 + incremental_verification = { + "sourcePath": incremental_source_path, + "seedClipKeys": deduped_seed_keys, + "rounds": round_summaries, + } + + print( + json.dumps( + { + "status": "completed", + "projectId": project_id, + "connectionId": connection["id"], + "mongoConnectionId": mongo_connection["id"] if mongo_connection else "", + "mongoCollection": mongo_collection, + "mongoVerification": mongo_verification, + "incrementalVerification": incremental_verification, + "sourcePath": config.source_path, + "uploadRootPrefix": config.upload_root_prefix, + "workflows": results, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except RegressionError as err: + print(str(err), file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/xplatform-live-acceptance.py b/scripts/xplatform-live-acceptance.py new file mode 100644 index 00000000..da2c19db --- /dev/null +++ b/scripts/xplatform-live-acceptance.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +import json +import sys +import textwrap +import time +import urllib.error +import urllib.request + + +BASE = "http://114.111.24.223:30080" + + +def request(method, path, payload=None, expected=(200, 201, 202, 204)): + data = None + headers = {"Content-Type": "application/json"} + if payload is not None: + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + req = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + body = resp.read().decode("utf-8") + if resp.getcode() not in expected: + raise RuntimeError(f"{method} {path} unexpected {resp.getcode()}: {body}") + return json.loads(body) if body else None + except urllib.error.HTTPError as err: + body = err.read().decode("utf-8", errors="replace") + raise RuntimeError(f"{method} {path} failed {err.code}: {body}") from err + + +def build_dataset_command() -> str: + return textwrap.dedent( + r""" + python - <<'PY' + import json + import pathlib + import cv2 + import numpy as np + + root = pathlib.Path("dataset/algo/X-platform-test") + + def write_video(path: pathlib.Path, seconds: float, fps: int = 10, blur: bool = False, seed: int = 0): + path.parent.mkdir(parents=True, exist_ok=True) + width, height = 96, 72 + writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) + if not writer.isOpened(): + raise RuntimeError(f"failed to create video: {path}") + frames = max(1, int(seconds * fps)) + rng = np.random.default_rng(seed) + try: + for idx in range(frames): + frame = np.zeros((height, width, 3), dtype=np.uint8) + for y in range(0, height, 8): + color = 220 if (y // 8) % 2 == 0 else 40 + frame[y : y + 4, :, :] = color + cv2.rectangle(frame, (8, 8), (40, 28), (255, 255, 255), 2) + cv2.circle(frame, (72, 22), 10, (0, 255, 0), -1) + cv2.putText(frame, f"F{idx:02d}", (8, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (255, 255, 255), 1, cv2.LINE_AA) + noise = rng.integers(0, 15, size=frame.shape, dtype=np.uint8) + frame = cv2.add(frame, noise) + if blur: + frame = cv2.GaussianBlur(frame, (13, 13), 3.5) + writer.write(frame) + finally: + writer.release() + + def write_collection(name: str, meta: dict, videos: list[dict], with_intrinsics: bool = False): + collection = root / name + video_dir = collection / "DJI_001" + video_dir.mkdir(parents=True, exist_ok=True) + rows = [] + for item in videos: + filename = item["filename"] + path = video_dir / filename + write_video(path, seconds=item["duration"], blur=item.get("blur", False), seed=item.get("seed", 0)) + rows.append( + { + "filename": filename, + "codec": "mp4v", + "resolution": "96x72", + "frameRate": 10.0, + "bitrate": int(path.stat().st_size * 8 / max(item["duration"], 0.1)), + "duration": item["duration"], + "startTime": item.get("startTime", 1773904218), + } + ) + (collection / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") + (collection / "video_meta.json").write_text(json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8") + (video_dir / "DJI_meta.json").write_text( + json.dumps({"camera": "OsmoNano", "folder": "DJI_001"}, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + if with_intrinsics: + (collection / "intrinsics.json").write_text( + json.dumps({"fx": 510.2, "fy": 509.8, "cx": 48.0, "cy": 36.0}, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + write_collection( + "BJ_001_0001_OsmoNano_2026-03-19_14-51-43", + { + "type": "家庭", + "place": "蓟门里小区", + "mission": "无", + "project_id": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43", + "date": "20260319", + "device_id": "C00000", + }, + [ + {"filename": "DJI_0001_D.mp4", "duration": 3.2, "seed": 1}, + {"filename": "DJI_0002_D.mp4", "duration": 1.6, "blur": True, "seed": 2, "startTime": 1773904265}, + ], + with_intrinsics=True, + ) + write_collection( + "BJ_002_0002_OsmoNano_2026-03-20_10-30-00", + { + "type": "办公", + "place": "中关村", + "mission": "测试", + "project_id": "BJ_002_0002_OsmoNano_2026-03-20_10-30-00", + "date": "20260320", + "device_id": "C00001", + }, + [ + {"filename": "DJI_0001_D.mp4", "duration": 2.6, "seed": 3}, + ], + ) + archive_path = pathlib.Path("xplatform-dataset.tgz") + import tarfile + import shutil + with tarfile.open(archive_path, "w:gz") as archive: + for child in sorted(root.iterdir()): + archive.add(child, arcname=child.name) + shutil.rmtree(pathlib.Path("dataset")) + print(archive_path) + PY + """ + ).strip() + + +def build_node(operator: dict, node_id: str, x: int, y: int, hyper_overrides=None) -> dict: + hyper = [dict(item) for item in operator.get("hyperparams", [])] + overrides = hyper_overrides or {} + seen = set() + for item in hyper: + key = item.get("key") + if key in overrides: + item["value"] = overrides[key] + seen.add(key) + for key, value in overrides.items(): + if key in seen: + continue + hyper.append({"key": key, "value": value}) + return { + "id": node_id, + "type": "operator", + "data": { + "label": operator["name"], + "category": operator["category"], + "description": operator["description"], + "inputs": operator.get("inputs", []), + "outputs": operator.get("outputs", []), + "image": operator.get("image", ""), + "skillDefinition": operator.get("skillDefinition", ""), + "hyperparams": hyper, + }, + "position": {"x": x, "y": y}, + } + + +def main() -> int: + timestamp = time.strftime("%Y%m%d%H%M%S") + project_name = f"XPlatform Acceptance {timestamp}" + workflow_name = f"XPlatform Pipeline {timestamp}" + + operators = request("GET", "/api/operators")["items"] + by_name = {item["name"]: item for item in operators} + needed = [ + "Shell 命令执行", + "X 平台数据集索引", + "采集目录校验", + "视频质检", + "数据过滤", + "视频切片", + "视频标注", + "成品打包", + "抽检采样", + "结果上传", + ] + missing = [name for name in needed if name not in by_name] + if missing: + raise SystemExit(f"missing operators: {missing}") + + project = request("POST", "/api/projects", {"name": project_name, "description": "Created by X-platform live acceptance"}) + project_id = project["id"] + + nodes = [ + build_node(by_name["Shell 命令执行"], "setup", 0, 0, {"COMMAND": build_dataset_command()}), + build_node(by_name["X 平台数据集索引"], "index", 220, 0), + build_node(by_name["采集目录校验"], "validate", 440, 0), + build_node( + by_name["视频质检"], + "quality", + 660, + 0, + {"MIN_CLARITY": "0.8", "MAX_SHAKE": "0.1", "MAX_HAND_OCCLUSION": "0.1", "CLARITY_SCALE": "35"}, + ), + build_node( + by_name["数据过滤"], + "filter", + 880, + 0, + { + "DATE_START": "2026-02-01", + "DATE_END": "2026-04-05", + "PLACE": "蓟门里小区", + "TYPE": "家庭", + "MIN_CLARITY": "0.8", + "MAX_SHAKE": "0.1", + "MAX_HAND_OCCLUSION": "0.1", + }, + ), + build_node(by_name["视频切片"], "slice", 1100, 0, {"SEGMENT_DURATION_SECONDS": "2"}), + build_node(by_name["视频标注"], "annotate", 1320, 0, {"FRAME_INTERVAL_SECONDS": "0.5"}), + build_node(by_name["成品打包"], "package", 1540, 0), + build_node( + by_name["抽检采样"], + "sample", + 1760, + -80, + { + "RULES_JSON": json.dumps( + [ + { + "id": "clarity-check", + "field": "quality.clarity", + "operator": "lt", + "value": 1, + "samplePercent": 100, + }, + { + "id": "confidence-check", + "field": "annotation.minConfidence", + "operator": "lt", + "value": 1, + "samplePercent": 100, + }, + ], + ensure_ascii=False, + ) + }, + ), + build_node(by_name["结果上传"], "upload", 1760, 100), + ] + + edges = [ + {"id": "setup-index", "source": "setup", "sourceHandle": "out-1", "target": "index", "targetHandle": "in-1"}, + {"id": "index-validate", "source": "index", "sourceHandle": "out-1", "target": "validate", "targetHandle": "in-1"}, + {"id": "validate-quality", "source": "validate", "sourceHandle": "out-1", "target": "quality", "targetHandle": "in-1"}, + {"id": "quality-filter", "source": "quality", "sourceHandle": "out-1", "target": "filter", "targetHandle": "in-1"}, + {"id": "filter-slice", "source": "filter", "sourceHandle": "out-1", "target": "slice", "targetHandle": "in-1"}, + {"id": "slice-annotate", "source": "slice", "sourceHandle": "out-1", "target": "annotate", "targetHandle": "in-1"}, + {"id": "annotate-package", "source": "annotate", "sourceHandle": "out-1", "target": "package", "targetHandle": "in-1"}, + {"id": "package-sample", "source": "package", "sourceHandle": "out-1", "target": "sample", "targetHandle": "in-1"}, + {"id": "package-upload", "source": "package", "sourceHandle": "out-1", "target": "upload", "targetHandle": "in-1"}, + ] + + workflow = request( + "POST", + f"/api/projects/{project_id}/workflows", + { + "name": workflow_name, + "description": "Live acceptance for X-platform operator chain", + "type": "batch", + "nodes": nodes, + "edges": edges, + }, + ) + workflow_id = workflow["id"] + run = request("POST", f"/api/workflows/{workflow_id}/run", {"trigger": "manual"}) + run_id = run["id"] + + terminal = None + for _ in range(360): + current = request("GET", f"/api/runs/{run_id}") + if current["status"] in {"completed", "failed", "cancelled", "waiting_for_audit"}: + terminal = current + break + time.sleep(2) + if terminal is None: + raise SystemExit(f"timeout waiting for run {run_id}") + + artifacts = request("GET", f"/api/runs/{run_id}/artifacts")["items"] + logs = request("GET", f"/api/runs/{run_id}/logs")["items"] + + interesting = {} + interesting_names = { + "dataset-index.json", + "dataset-validation.json", + "quality-report.json", + "filtered-videos.json", + "clips-manifest.json", + "annotations-manifest.json", + "packages-manifest.json", + "audit-samples.json", + "uploaded-results.json", + } + for item in artifacts: + if item["name"] in interesting_names: + interesting[item["name"]] = item.get("data") + + summary_artifacts = {} + for name, data in interesting.items(): + if not isinstance(data, dict): + summary_artifacts[name] = data + continue + summary_artifacts[name] = { + key: data.get(key) + for key in ( + "collectionCount", + "validCollectionCount", + "videoCount", + "passedCount", + "count", + "clipCount", + "annotationCount", + "packageCount", + "sampledCount", + "uploadedCount", + ) + if key in data + } + + summary = { + "projectId": project_id, + "workflowId": workflow_id, + "runId": run_id, + "status": terminal["status"], + "artifactCount": len(artifacts), + "nodeStatuses": {item["nodeId"]: item["status"] for item in terminal.get("nodeExecutions", [])}, + "interestingArtifacts": summary_artifacts, + } + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + if terminal["status"] != "completed": + print(json.dumps({"logs": logs[-20:]}, ensure_ascii=False, indent=2), file=sys.stderr) + return 1 + + checks = { + "dataset-index.json": ("collectionCount", 2), + "dataset-validation.json": ("validCollectionCount", 2), + "quality-report.json": ("videoCount", 3), + "filtered-videos.json": ("count", 1), + "clips-manifest.json": ("clipCount", 2), + "annotations-manifest.json": ("annotationCount", 2), + "packages-manifest.json": ("packageCount", 2), + "audit-samples.json": ("sampledCount", 2), + } + for artifact_name, (field, expected) in checks.items(): + data = interesting.get(artifact_name) + if not isinstance(data, dict) or data.get(field) != expected: + raise SystemExit(f"{artifact_name} expected {field}={expected}, got {data}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/App.tsx b/src/App.tsx index 5b3fdc81..530cbc7b 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -19,6 +19,13 @@ const Settings = lazy(() => import("./pages/Settings")); const Templates = lazy(() => import("./pages/Templates")); const Users = lazy(() => import("./pages/Users")); const Audit = lazy(() => import("./pages/Audit")); +const DataAssets = lazy(() => import("./pages/DataAssets")); +const DatabaseBrowser = lazy(() => import("./pages/DatabaseBrowser")); +const ObjectStorageBrowser = lazy(() => import("./pages/ObjectStorageBrowser")); +const LegacyDataAssetsRedirect = lazy( + () => import("./pages/LegacyDataAssetsRedirect"), +); +const Sampling = lazy(() => import("./pages/Sampling")); function RouteFallback() { return ( @@ -41,6 +48,21 @@ export default function App() { } /> } /> } /> + } /> + } /> + } + /> + } + /> + } + /> + } /> } /> } /> } /> diff --git a/src/components/ArtifactPreviewPanel.tsx b/src/components/ArtifactPreviewPanel.tsx new file mode 100644 index 00000000..46f75c7a --- /dev/null +++ b/src/components/ArtifactPreviewPanel.tsx @@ -0,0 +1,197 @@ +import React from "react"; +import type { ExecutionData } from "../types"; +import { buildBrowserPlayableVideoURL } from "../lib/video-preview.js"; + +const artifactExtension = (item: ExecutionData) => { + const source = item.name || item.path || ""; + const parts = source.split("."); + return parts.length > 1 ? parts[parts.length - 1].toLowerCase() : ""; +}; + +const rawPreviewPayload = (item: ExecutionData) => { + const preview = item.data?.preview; + return preview && typeof preview === "object" + ? (preview as Record) + : null; +}; + +const derivedPreview = (item: ExecutionData) => { + const preview = rawPreviewPayload(item); + if (preview) { + return preview; + } + if (item.type === "json" && item.data) { + return { kind: "json", value: item.data }; + } + const mimeType = item.metadata?.mimeType || ""; + const ext = artifactExtension(item); + if (mimeType.startsWith("image/")) return { kind: "image" }; + if (item.type === "video" || mimeType.startsWith("video/")) return { kind: "video" }; + if (mimeType.startsWith("audio/")) return { kind: "audio" }; + if (ext === "pdf") return { kind: "document", format: "pdf" }; + if (["csv", "tsv", "jsonl"].includes(ext)) return { kind: "table", format: ext }; + if (["txt", "md", "log", "yaml", "yml", "xml", "html", "htm"].includes(ext)) return { kind: "text", format: ext }; + return { kind: "file" }; +}; + +const artifactPreviewURL = (item: ExecutionData, preview: Record) => { + if (typeof preview.url === "string" && preview.url) { + return preview.url; + } + const source = item.data?.source; + if (source && typeof source === "object") { + if (typeof source.previewURL === "string" && source.previewURL) { + return source.previewURL; + } + if (typeof source.path === "string" && /^https?:\/\//i.test(source.path)) { + return source.path; + } + } + return /^https?:\/\//i.test(item.path) ? item.path : ""; +}; + +export function ArtifactPreviewPanel({ item }: { item: ExecutionData | null }) { + const [mediaError, setMediaError] = React.useState(false); + const preview = item ? derivedPreview(item) : null; + const previewURL = item && preview ? artifactPreviewURL(item, preview) : ""; + const playablePreviewURL = + preview?.kind === "video" + ? buildBrowserPlayableVideoURL(previewURL) + : previewURL; + + React.useEffect(() => { + setMediaError(false); + }, [item?.id, playablePreviewURL]); + + if (!item) { + return ( +
+ 请选择文件查看预览 +
+ ); + } + + if (preview.kind === "image" && previewURL) { + return ( +
+ {item.name} +
+ ); + } + + if (preview.kind === "video" && playablePreviewURL) { + return ( +
+ {mediaError ? ( +
+
浏览器预览生成失败
+
+ 系统已经尝试自动将原始视频转为浏览器可播放的 H.264 预览,但本次预览仍然失败。你仍可直接打开原始文件,或稍后重试。 +
+ + 打开原始视频 + +
+ ) : ( +
+ ); + } + + if (preview.kind === "audio" && previewURL) { + return ( +
+
+ ); + } + + if (preview.kind === "document" && preview.format === "pdf" && previewURL) { + return ( +
+