diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ef2943d6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +node_modules +dist +build +coverage +.git +.gitignore +.env +.env.local +*.log diff --git a/.env.example b/.env.example index 7a550fe7..18e5d42c 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,15 @@ -# GEMINI_API_KEY: Required for Gemini AI API calls. -# AI Studio automatically injects this at runtime from user secrets. -# Users configure this via the Secrets panel in the AI Studio UI. -GEMINI_API_KEY="MY_GEMINI_API_KEY" - -# APP_URL: The URL where this applet is hosted. -# AI Studio automatically injects this at runtime with the Cloud Run service URL. -# Used for self-referential links, OAuth callbacks, and API endpoints. -APP_URL="MY_APP_URL" +PORT=3001 +WORKER_PORT=3002 +MONGO_URI=mongodb://127.0.0.1:27017 +MONGO_DB=droboticflow +MONGO_CONNECT_RETRIES=30 +MONGO_CONNECT_RETRY_DELAY_MS=2000 +WORKER_POLL_INTERVAL_MS=2000 +K8S_ENABLED=false +K8S_NAMESPACE=droboticflow +K8S_IMAGE_PULL_SECRET_NAME=droboticflow-registry +OPERATOR_RUNNER_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:latest +VITE_API_BASE_URL= +AI_API_BASE_URL=https://coding.dashscope.aliyuncs.com/v1 +AI_MODEL=glm-5 +AI_API_KEY= diff --git a/Dockerfile.api b/Dockerfile.api new file mode 100644 index 00000000..9fdf0219 --- /dev/null +++ b/Dockerfile.api @@ -0,0 +1,32 @@ +FROM node:22-alpine AS web-build +WORKDIR /app + +COPY package*.json ./ +RUN npm ci + +COPY index.html metadata.json package.json tsconfig.json vite.config.ts ./ +COPY src ./src +RUN npm run build:web + +FROM golang:1.24-bookworm AS go-build +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd ./cmd +COPY internal ./internal +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-api ./cmd/api + +FROM debian:bookworm-slim AS runtime +WORKDIR /app +ENV PORT=3001 +ENV DIST_DIR=/app/dist + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* + +COPY --from=web-build /app/dist ./dist +COPY --from=go-build /out/droboticflow-api /usr/local/bin/droboticflow-api + +EXPOSE 3001 +CMD ["droboticflow-api"] diff --git a/Dockerfile.operator-runner b/Dockerfile.operator-runner new file mode 100644 index 00000000..8f9fd242 --- /dev/null +++ b/Dockerfile.operator-runner @@ -0,0 +1,7 @@ +FROM python:3.11-slim AS runtime +WORKDIR /app +ENV PYTHONUNBUFFERED=1 + +COPY backend/operator-runner/main.py ./main.py + +CMD ["python", "/app/main.py"] diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 00000000..c8efc514 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,20 @@ +FROM golang:1.24-bookworm AS build +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd ./cmd +COPY internal ./internal +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-worker ./cmd/worker + +FROM debian:bookworm-slim AS runtime +WORKDIR /app +ENV WORKER_PORT=3002 + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* + +COPY --from=build /out/droboticflow-worker /usr/local/bin/droboticflow-worker + +EXPOSE 3002 +CMD ["droboticflow-worker"] diff --git a/README.md b/README.md index 87d6a68e..74509c38 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,174 @@ -
-GHBanner -
+# DRoboticFlow -# Run and deploy your AI Studio app +DRoboticFlow is a workflow orchestration platform for embodied AI and vision pipelines. This repository now contains: -This contains everything you need to run your app locally. +- a React/Vite frontend +- a Go-based MongoDB-backed REST API +- a Go-based worker service that executes workflow DAGs +- a Python-based Kubernetes Job operator runner +- deployment assets for MongoDB, Kubernetes, and Baidu Cloud Container Registry -View your app in AI Studio: https://ai.studio/apps/0006ee57-f2f7-45cb-b325-2830a8f19364 +## Architecture -## Run Locally +- `cmd/api`: Go API entrypoint and SPA host +- `cmd/worker`: Go worker entrypoint and worker control API +- `internal/api`: REST handlers and request validation +- `internal/worker`: run scheduler, DAG executor, audit pause/resume, worker control API +- `internal/store`: Mongo-backed repositories plus in-memory test store +- `internal/k8s`: Kubernetes Job launcher for operator nodes +- `backend/operator-runner`: Python entrypoint used by Kubernetes Jobs +- `src`: existing frontend, now wired to real APIs +- `k8s`: Kubernetes manifest template +- `scripts`: build, render, deploy, and smoke test scripts +- `docs`: API, deployment, operations, and design documents -**Prerequisites:** Node.js +## Local Development +### Prerequisites + +- Node.js 22+ +- Go 1.24+ +- Python 3.9+ +- MongoDB 7+ reachable at `MONGO_URI` + +### Setup 1. Install dependencies: - `npm install` -2. Set the `GEMINI_API_KEY` in [.env.local](.env.local) to your Gemini API key -3. Run the app: - `npm run dev` + + ```bash + npm install + ``` + +2. Copy the example environment file and adjust values: + + ```bash + cp .env.example .env.local + ``` + +3. Start the Go API and frontend: + + ```bash + npm run dev + ``` + +4. Start the full stack including Go worker: + + ```bash + npm run dev:full + ``` + +### Useful Scripts + +- `npm run lint`: type-check frontend and verify Go formatting +- `npm test`: Go unit and integration tests +- `npm run build`: build frontend assets and Go binaries +- `npm run verify`: lint + test + build +- `npm run smoke`: local smoke test against the running API and worker +- `npm run reconcile:data -- `: remove legacy demo data, reset settings, and bootstrap built-in operators through the real API + +## Runtime Configuration + +Key environment variables: + +- `PORT`: API port, default `3001` +- `WORKER_PORT`: worker port, default `3002` +- `MONGO_URI`: MongoDB connection string +- `MONGO_DB`: Mongo database name +- `MONGO_CONNECT_RETRIES`: startup retry count for API and worker Mongo connections +- `MONGO_CONNECT_RETRY_DELAY_MS`: delay between Mongo connection retries +- `WORKER_POLL_INTERVAL_MS`: worker polling interval +- `K8S_ENABLED`: enable Kubernetes Job execution in the worker +- `K8S_NAMESPACE`: namespace used for Jobs +- `K8S_IMAGE_PULL_SECRET_NAME`: image pull secret injected into operator-runner Jobs +- `OPERATOR_RUNNER_IMAGE`: container image used for node execution Jobs +- `AI_API_BASE_URL`: OpenAI-compatible DashScope endpoint used by server-side AI generation, default `https://coding.dashscope.aliyuncs.com/v1` +- `AI_MODEL`: server-side model name, default `glm-5` +- `AI_API_KEY`: server-side DashScope-compatible API key used for workflow/operator generation +- `VITE_API_BASE_URL`: optional frontend API base URL override + +The frontend never reads the AI key directly. Workflow generation and operator generation are proxied through the Go API so secrets stay on the server side. +The settings page can update K8s access details, the worker defaults, the global `timezone`, `provider`, `baseURL`, `model`, and the server-side AI key. `GET /api/settings` only returns whether secrets exist via `k8s.hasKubeconfig` / `ai.hasAPIKey`; it does not echo plaintext kubeconfig or API keys back to the browser. Timestamps in the UI are rendered in the configured IANA timezone with compact minute-level formatting for cards, lists, and summaries, while logs keep finer granularity when needed. OSS-related bucket and path settings belong on OSS operators in workflows, not in the global system settings page. The OSS browser APIs also now require the current node `SOURCE_PATH` and only list content inside that node-scoped root. + +Workflow execution concurrency now has two levels: + +- system default: `settings.worker.maxConcurrentRuns` +- workflow override: `workflow.maxConcurrentRuns` + +The worker enforces both. A workflow value of `0` means "inherit the global default". Dispatch only starts queued runs while the global active run count is below the system limit and the active count for that workflow is below its effective limit. + +Workflow-level execution safeguards are configured alongside the other top-bar workflow settings in the editor: + +- `workflow.nodeTimeoutSeconds` + - `0`: disable workflow-level per-node timeout + - `1+`: fail an operator execution once a single node exceeds the configured seconds +- `workflow.runTimeoutSeconds` + - `0`: disable workflow-level timeout + - `1+`: fail the current run once total execution time exceeds the configured seconds +- `workflow.maxRetryAttempts` + - `0`: disable automatic retry + - `1+`: automatically enqueue a new retry run after a failure, up to the configured count +- `workflow.retryBackoffSeconds` + - `0`: retry immediately + - `1+`: keep the retry run queued until its backoff window expires + +Automatic retries create real follow-up runs with `trigger=retry`, `attempt=2+`, `parentRunId` pointing at the failed attempt, and `availableAt` marking the first dispatchable time after backoff. Monitoring and history stay auditable because the original run is never mutated into the retry attempt. + +Workflow templates now carry the same workflow-level defaults as project workflows. When you save a workflow as a template, duplicate a template, edit a template, or instantiate a template into a project, the workflow `type`, `schedule`, and execution-safeguard fields move across with it instead of being dropped or reset. + +Artifact previews are persisted with the run artifacts themselves. JSON artifacts keep their structured payload in `artifact.data`, while non-JSON artifacts can attach a compact `artifact.data.preview` block for text, CSV/TSV, JSONL, images, audio/video metadata, PDF/document hints, SQLite/rosbag summaries, and LeRobot dataset summaries. Artifacts also carry `portId` / `portName`, which lets the worker route multi-output operators by edge handle instead of mixing all outputs together. + +The built-in operator catalog now includes real data-processing and robotics helpers on top of the original source/merge/upload/audit flow: `目录扫描`, `CSV 转 JSON`, `JSON 转 CSV`, `JSON 字段提取`, `JSON 过滤`, `JSON 映射重命名`, `文本模板渲染`, `条件分流`, `LeRobot 数据集概览`, `ROSBag 数据概览`, and `Shell 命令执行`. + +## Deployment + +The repository includes three container images: + +- `droboticflow-api` +- `droboticflow-worker` +- `droboticflow-operator-runner` + +The API image builds the Vite frontend and a Go binary into one container. The worker image contains a Go binary. The operator-runner image is Python-only and is launched by the worker as a Kubernetes Job. + +Build and push images: + +```bash +./scripts/build-and-push-images.sh +``` + +If the target cluster cannot pull the upstream `mongo:7` image from Docker Hub, mirror it into the same Baidu registry and set `MONGO_IMAGE` during deployment. + +Render Kubernetes manifests: + +```bash +./scripts/render-manifests.sh +``` + +Deploy to Kubernetes: + +```bash +./scripts/deploy-k8s.sh +``` + +Reconcile persistent API data after deployment: + +```bash +./scripts/reconcile-system-data.sh http://:30080 +``` + +Run a smoke test locally: + +```bash +npm run smoke +``` + +Run a smoke test against a deployed cluster: + +```bash +./scripts/smoke-test.sh http://:30080 +``` + +See the detailed guides: + +- [`docs/api.md`](/Users/longtaowu/workspace/droboticflow/docs/api.md) +- [`docs/deployment.md`](/Users/longtaowu/workspace/droboticflow/docs/deployment.md) +- [`docs/operations.md`](/Users/longtaowu/workspace/droboticflow/docs/operations.md) diff --git a/backend/operator-runner/main.py b/backend/operator-runner/main.py new file mode 100644 index 00000000..24198b4e --- /dev/null +++ b/backend/operator-runner/main.py @@ -0,0 +1,1470 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import datetime as dt +import fnmatch +import hashlib +import json +import mimetypes +import os +import pathlib +import base64 +import csv +import re +import resource +import shutil +import sqlite3 +import subprocess +import sys +import time +from typing import Any + + +RESULT_PREFIX = "DROBOTICFLOW_RESULT=" +TEXT_FILE_SUFFIXES = { + ".txt", + ".log", + ".md", + ".yaml", + ".yml", + ".xml", + ".html", + ".htm", +} +TABULAR_FILE_SUFFIXES = {".csv": ",", ".tsv": "\t"} +IMAGE_FILE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"} +VIDEO_FILE_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"} +AUDIO_FILE_SUFFIXES = {".wav", ".mp3", ".aac", ".m4a", ".ogg", ".flac"} +ROSBAG_FILE_SUFFIXES = {".bag", ".db3", ".mcap"} +PREVIEW_TEXT_LIMIT = 12_000 +PREVIEW_TEXT_LINE_LIMIT = 240 +PREVIEW_TABLE_ROW_LIMIT = 24 +PREVIEW_FILE_LIST_LIMIT = 40 +PREVIEW_INLINE_BINARY_LIMIT = 512 * 1024 + + +def now_iso() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") + + +def log(message: str) -> None: + print(message, flush=True) + + +def load_json_env(name: str, default: Any) -> Any: + value = os.getenv(name, "").strip() + if not value: + return default + return json.loads(value) + + +def resource_snapshot() -> tuple[float, int]: + usage = resource.getrusage(resource.RUSAGE_SELF) + cpu_seconds = usage.ru_utime + usage.ru_stime + if sys.platform == "darwin": + memory_mib = max(1, int(usage.ru_maxrss / (1024 * 1024))) + else: + memory_mib = max(1, int(usage.ru_maxrss / 1024)) + return cpu_seconds, memory_mib + + +def resource_series(start_cpu: float, start_memory: int, started_at: str) -> list[dict[str, Any]]: + end_cpu, end_memory = resource_snapshot() + ended_at = now_iso() + started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00")) + ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00")) + elapsed = max((ended - started).total_seconds(), 0.001) + cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100))) + return [ + {"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0}, + {"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, "gpu": 0}, + ] + + +def format_size(num_bytes: int) -> str: + if num_bytes < 1024: + return f"{num_bytes} B" + if num_bytes < 1024 * 1024: + return f"{num_bytes / 1024:.1f} KiB" + if num_bytes < 1024 * 1024 * 1024: + return f"{num_bytes / (1024 * 1024):.1f} MiB" + return f"{num_bytes / (1024 * 1024 * 1024):.2f} GiB" + + +def sha256_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + digest.update(chunk) + return digest.hexdigest() + + +def artifact_slug(value: str) -> str: + normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value).strip("-") + return normalized or "artifact" + + +def infer_port_type(path: pathlib.Path, fallback: str = "file") -> str: + suffix = path.suffix.lower() + if suffix == ".json": + return "json" + if suffix in VIDEO_FILE_SUFFIXES: + return "video" + if suffix in IMAGE_FILE_SUFFIXES: + return "image" + return fallback + + +def clean_metadata(metadata: dict[str, Any]) -> dict[str, Any] | None: + cleaned = {key: value for key, value in metadata.items() if value not in ("", None, [], {})} + return cleaned or None + + +def load_payload_from_file(path: pathlib.Path) -> dict[str, Any] | None: + if not path.exists() or path.suffix.lower() != ".json": + return None + try: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle) + except Exception: + return None + if isinstance(value, dict): + return value + if isinstance(value, list): + return {"items": value, "itemCount": len(value)} + return {"value": value} + + +def output_binding(node_data: dict[str, Any], index: int = 0, name: str = "") -> dict[str, Any] | None: + outputs = node_data.get("outputs", []) or [] + if name: + for output in outputs: + if str(output.get("name", "")).strip() == name or str(output.get("id", "")).strip() == name: + return output + if 0 <= index < len(outputs): + return outputs[index] + return None + + +def read_text_sample(path: pathlib.Path) -> tuple[str, bool, int]: + with path.open("rb") as handle: + raw = handle.read(PREVIEW_TEXT_LIMIT + 1) + truncated = len(raw) > PREVIEW_TEXT_LIMIT + text = raw[:PREVIEW_TEXT_LIMIT].decode("utf-8", errors="replace") + lines = text.splitlines() + if len(lines) > PREVIEW_TEXT_LINE_LIMIT: + text = "\n".join(lines[:PREVIEW_TEXT_LINE_LIMIT]) + truncated = True + line_count = max(1, text.count("\n") + 1) if text else 0 + return text, truncated, line_count + + +def preview_text_payload(path: pathlib.Path, format_name: str) -> dict[str, Any]: + text, truncated, line_count = read_text_sample(path) + return { + "preview": { + "kind": "text", + "format": format_name, + "text": text, + "truncated": truncated, + "lineCount": line_count, + } + } + + +def preview_tabular_payload(path: pathlib.Path, delimiter: str, format_name: str) -> dict[str, Any]: + columns: list[str] = [] + rows: list[dict[str, Any]] = [] + total_rows = 0 + with path.open("r", encoding="utf-8", errors="replace", newline="") as handle: + reader = csv.reader(handle, delimiter=delimiter) + for index, row in enumerate(reader): + if index == 0: + columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)] + continue + total_rows += 1 + if len(rows) >= PREVIEW_TABLE_ROW_LIMIT: + continue + width = max(len(columns), len(row)) + row_payload: dict[str, Any] = {} + for column_index in range(width): + column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}" + row_payload[column_name] = row[column_index] if column_index < len(row) else "" + rows.append(row_payload) + return { + "preview": { + "kind": "table", + "format": format_name, + "columns": columns, + "rows": rows, + "rowCount": total_rows, + "truncated": total_rows > len(rows), + } + } + + +def load_tabular_rows(path: pathlib.Path, delimiter: str) -> tuple[list[str], list[dict[str, Any]]]: + columns: list[str] = [] + rows: list[dict[str, Any]] = [] + with path.open("r", encoding="utf-8", errors="replace", newline="") as handle: + reader = csv.reader(handle, delimiter=delimiter) + for index, row in enumerate(reader): + if index == 0: + columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)] + continue + width = max(len(columns), len(row)) + row_payload: dict[str, Any] = {} + for column_index in range(width): + column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}" + row_payload[column_name] = row[column_index] if column_index < len(row) else "" + rows.append(row_payload) + return columns, rows + + +def preview_jsonl_payload(path: pathlib.Path) -> dict[str, Any]: + rows: list[Any] = [] + line_count = 0 + with path.open("r", encoding="utf-8", errors="replace") as handle: + for raw_line in handle: + line = raw_line.strip() + if not line: + continue + line_count += 1 + if len(rows) >= PREVIEW_TABLE_ROW_LIMIT: + continue + try: + rows.append(json.loads(line)) + except Exception: + rows.append({"value": line}) + columns: list[str] = [] + for row in rows: + if isinstance(row, dict): + for key in row.keys(): + if key not in columns: + columns.append(key) + return { + "preview": { + "kind": "table", + "format": "jsonl", + "columns": columns, + "rows": rows, + "rowCount": line_count, + "truncated": line_count > len(rows), + } + } + + +def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]: + payload: dict[str, Any] = { + "preview": { + "kind": "image", + "format": path.suffix.lower().lstrip(".") or "image", + } + } + size_bytes = path.stat().st_size if path.exists() else 0 + if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type: + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + payload["preview"]["url"] = f"data:{mime_type};base64,{encoded}" + return payload + + +def sample_file_entries(paths: list[pathlib.Path], root: pathlib.Path) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + for candidate in paths[:PREVIEW_FILE_LIST_LIMIT]: + try: + relative = candidate.relative_to(root).as_posix() + except ValueError: + relative = candidate.name + items.append({"path": relative, "sizeBytes": candidate.stat().st_size}) + return items + + +def summarize_directory_tree(root: pathlib.Path) -> dict[str, Any]: + files = [candidate for candidate in sorted(root.rglob("*")) if candidate.is_file()] + extensions: dict[str, int] = {} + total_bytes = 0 + for candidate in files: + extension = candidate.suffix.lower() or "" + extensions[extension] = extensions.get(extension, 0) + 1 + total_bytes += candidate.stat().st_size + ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0])) + return { + "root": str(root), + "fileCount": len(files), + "totalBytes": total_bytes, + "topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]], + "files": sample_file_entries(files, root), + "truncated": len(files) > PREVIEW_FILE_LIST_LIMIT, + } + + +def optional_json_file(path: pathlib.Path) -> Any: + if not path.exists(): + return None + try: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + except Exception: + return None + + +def optional_jsonl_rows(path: pathlib.Path) -> list[Any]: + rows: list[Any] = [] + if not path.exists(): + return rows + try: + with path.open("r", encoding="utf-8", errors="replace") as handle: + for raw_line in handle: + line = raw_line.strip() + if not line: + continue + rows.append(json.loads(line)) + if len(rows) >= PREVIEW_TABLE_ROW_LIMIT: + break + except Exception: + return [] + return rows + + +def find_lerobot_root(path: pathlib.Path) -> pathlib.Path | None: + candidates = [path] if path.is_dir() else [path.parent] + candidates.extend(list((path if path.is_dir() else path.parent).parents)) + for candidate in candidates: + meta_dir = candidate / "meta" + if meta_dir.is_dir() and any((meta_dir / name).exists() for name in ("info.json", "stats.json", "tasks.jsonl")): + return candidate + return None + + +def summarize_lerobot_root(root: pathlib.Path) -> dict[str, Any]: + tree = summarize_directory_tree(root) + meta_dir = root / "meta" + info = optional_json_file(meta_dir / "info.json") + stats = optional_json_file(meta_dir / "stats.json") + tasks = optional_jsonl_rows(meta_dir / "tasks.jsonl") + episodes = optional_jsonl_rows(meta_dir / "episodes.jsonl") + parquet_files = [candidate for candidate in root.rglob("*.parquet") if candidate.is_file()] + video_files = [candidate for candidate in root.rglob("*") if candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES] + return { + "preview": { + "kind": "dataset", + "format": "lerobot", + "root": str(root), + "summary": tree, + "info": info, + "stats": stats, + "tasks": tasks, + "episodes": episodes, + "parquetFiles": sample_file_entries(parquet_files, root), + "parquetCount": len(parquet_files), + "videoCount": len(video_files), + } + } + + +def sqlite_table_names(connection: sqlite3.Connection) -> list[str]: + rows = connection.execute( + "select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name" + ).fetchall() + return [str(row[0]) for row in rows] + + +def summarize_sqlite_payload(path: pathlib.Path) -> dict[str, Any]: + connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + try: + tables = sqlite_table_names(connection) + table_summaries: list[dict[str, Any]] = [] + for table_name in tables[:12]: + count = connection.execute(f"select count(*) from \"{table_name}\"").fetchone()[0] + table_summaries.append({"name": table_name, "rowCount": int(count)}) + payload: dict[str, Any] = { + "preview": { + "kind": "dataset", + "format": "sqlite", + "tables": table_summaries, + "tableCount": len(tables), + } + } + if "topics" in tables and "messages" in tables: + topic_rows = connection.execute( + "select id, name, type, serialization_format from topics order by id" + ).fetchall() + message_counts = { + int(topic_id): int(count) + for topic_id, count in connection.execute( + "select topic_id, count(*) from messages group by topic_id order by topic_id" + ).fetchall() + } + payload["preview"]["format"] = "rosbag2" + payload["preview"]["topics"] = [ + { + "id": int(topic_id), + "name": str(name), + "type": str(topic_type), + "serialization": str(serialization), + "messageCount": message_counts.get(int(topic_id), 0), + } + for topic_id, name, topic_type, serialization in topic_rows[:PREVIEW_TABLE_ROW_LIMIT] + ] + payload["preview"]["messageCount"] = int(sum(message_counts.values())) + return payload + finally: + connection.close() + + +def summarize_rosbag_file(path: pathlib.Path) -> dict[str, Any]: + suffix = path.suffix.lower() + if suffix == ".db3": + return summarize_sqlite_payload(path) + header = "" + with path.open("rb") as handle: + header = handle.read(32).decode("utf-8", errors="replace") + format_name = "mcap" if suffix == ".mcap" else "rosbag1" + return { + "preview": { + "kind": "dataset", + "format": format_name, + "header": header.strip(), + "note": "文件已识别,但深度解析依赖外部运行时;当前展示真实文件摘要和元数据。", + } + } + + +def resolve_source_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path | None: + direct = params.get("SOURCE_PATH", "").strip() + if direct and pathlib.Path(direct).exists(): + return pathlib.Path(direct) + for artifact in inputs: + payload = artifact.get("data") + if isinstance(payload, dict): + candidate = payload.get("sourcePath") + if isinstance(candidate, str) and candidate and pathlib.Path(candidate).exists(): + return pathlib.Path(candidate) + source_path = artifact.get("path") + if isinstance(source_path, str) and source_path and pathlib.Path(source_path).exists(): + return pathlib.Path(source_path) + return None + + +def artifact_preview_payload(artifact: dict[str, Any]) -> dict[str, Any] | None: + data = artifact.get("data") + if not isinstance(data, dict): + return None + preview = data.get("preview") + if isinstance(preview, dict): + return preview + return None + + +def summarize_input_artifacts(inputs: list[dict[str, Any]]) -> dict[str, Any]: + extensions: dict[str, int] = {} + files: list[dict[str, Any]] = [] + for artifact in inputs: + name = str(artifact.get("name", "") or artifact.get("path", "")).strip() + extension = pathlib.Path(name).suffix.lower() or "" + extensions[extension] = extensions.get(extension, 0) + 1 + files.append( + { + "path": str(artifact.get("path", "")).strip() or name, + "name": name, + "size": str(artifact.get("size", "")), + } + ) + ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0])) + return { + "fileCount": len(files), + "topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]], + "files": files[:PREVIEW_FILE_LIST_LIMIT], + "truncated": len(files) > PREVIEW_FILE_LIST_LIMIT, + } + + +def infer_lerobot_root_from_inputs(inputs: list[dict[str, Any]]) -> str: + for artifact in inputs: + raw_path = str(artifact.get("path", "")).strip() + if not raw_path: + continue + parts = pathlib.PurePosixPath(raw_path).parts + for marker in ("meta", "data", "videos"): + if marker in parts: + index = parts.index(marker) + if index > 0: + return str(pathlib.PurePosixPath(*parts[:index])) + return "" + + +def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any]: + info = None + stats = None + tasks: list[Any] = [] + episodes: list[Any] = [] + parquet_files: list[dict[str, Any]] = [] + video_count = 0 + for artifact in inputs: + name = str(artifact.get("name", "")).strip() + path = str(artifact.get("path", "")).strip() + data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} + preview = artifact_preview_payload(artifact) or {} + if name == "info.json" and isinstance(data, dict): + info = data + elif name == "stats.json" and isinstance(data, dict): + stats = data + elif name.endswith("tasks.jsonl") and isinstance(preview.get("rows"), list): + tasks = preview["rows"] + elif name.endswith("episodes.jsonl") and isinstance(preview.get("rows"), list): + episodes = preview["rows"] + if name.endswith(".parquet"): + parquet_files.append({"path": path or name, "size": str(artifact.get("size", ""))}) + if pathlib.Path(name).suffix.lower() in VIDEO_FILE_SUFFIXES or str(artifact.get("type", "")) == "video": + video_count += 1 + return { + "preview": { + "kind": "dataset", + "format": "lerobot", + "root": infer_lerobot_root_from_inputs(inputs), + "summary": summarize_input_artifacts(inputs), + "info": info, + "stats": stats, + "tasks": tasks, + "episodes": episodes, + "parquetFiles": parquet_files[:PREVIEW_FILE_LIST_LIMIT], + "parquetCount": len(parquet_files), + "videoCount": video_count, + } + } + + +def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) -> dict[str, Any] | None: + suffix = path.suffix.lower() + if suffix in TABULAR_FILE_SUFFIXES: + return preview_tabular_payload(path, TABULAR_FILE_SUFFIXES[suffix], suffix.lstrip(".")) + if suffix == ".jsonl": + return preview_jsonl_payload(path) + if suffix in TEXT_FILE_SUFFIXES: + format_name = "html" if suffix in {".html", ".htm"} else suffix.lstrip(".") + return preview_text_payload(path, format_name) + if suffix in IMAGE_FILE_SUFFIXES: + return preview_image_payload(path, mime_type) + if suffix in ROSBAG_FILE_SUFFIXES: + return summarize_rosbag_file(path) + if suffix == ".pdf": + return {"preview": {"kind": "document", "format": "pdf"}} + if artifact_type == "video" or suffix in VIDEO_FILE_SUFFIXES: + return {"preview": {"kind": "video", "format": suffix.lstrip(".") or "video"}} + if suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"): + return {"preview": {"kind": "audio", "format": suffix.lstrip(".") or "audio"}} + return None + + +def build_artifact( + run_id: str, + node_id: str, + path: pathlib.Path, + artifact_type: str, + data: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + output: dict[str, Any] | None = None, +) -> dict[str, Any]: + mime_type, _ = mimetypes.guess_type(path.name) + artifact_metadata = clean_metadata( + { + **(metadata or {}), + "sha256": sha256_file(path) if path.exists() else "", + "mimeType": mime_type or "", + } + ) + size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8")) + artifact = { + "id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else path.stem)}-{artifact_slug(path.name)}", + "runId": run_id, + "nodeId": node_id, + "name": path.name, + "type": artifact_type, + "size": format_size(size_bytes), + "path": str(path), + } + if output: + if str(output.get("id", "")).strip(): + artifact["portId"] = str(output.get("id", "")).strip() + if str(output.get("name", "")).strip(): + artifact["portName"] = str(output.get("name", "")).strip() + if artifact_metadata: + artifact["metadata"] = artifact_metadata + if data: + artifact["data"] = data + return artifact + + +def write_json_artifact( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + filename: str, + payload: dict[str, Any], + artifact_type: str = "json", + output: dict[str, Any] | None = None, +) -> dict[str, Any]: + path = output_dir / filename + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2) + return build_artifact(run_id, node_id, path, artifact_type, data=payload, output=output) + + +def summarize_json_like(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return {"kind": "object", "keys": sorted(value.keys())} + if isinstance(value, list): + return {"kind": "list", "length": len(value)} + return {"kind": type(value).__name__, "value": value} + + +def load_artifact_payload(artifact: dict[str, Any]) -> dict[str, Any]: + data = artifact.get("data") + if isinstance(data, dict): + return data + path_value = artifact.get("path") + if isinstance(path_value, str) and path_value: + payload = load_payload_from_file(pathlib.Path(path_value)) + if payload is not None: + return payload + summary = { + "name": artifact.get("name", ""), + "type": artifact.get("type", ""), + "path": artifact.get("path", ""), + "size": artifact.get("size", ""), + } + return {key: value for key, value in summary.items() if value} + + +def merge_dicts(values: list[dict[str, Any]]) -> dict[str, Any]: + merged: dict[str, Any] = {} + for value in values: + merged.update(value) + return merged + + +def list_source_entries(base_path: pathlib.Path, pattern: str) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + if not base_path.exists(): + return entries + if base_path.is_file(): + relative = base_path.name + if pattern and not fnmatch.fnmatch(relative, pattern): + return entries + return [ + { + "path": relative, + "sizeBytes": base_path.stat().st_size, + "sha256": sha256_file(base_path), + } + ] + for candidate in sorted(base_path.rglob("*")): + if candidate.is_dir(): + continue + relative = candidate.relative_to(base_path).as_posix() + if pattern and not fnmatch.fnmatch(relative, pattern): + continue + entries.append( + { + "path": relative, + "sizeBytes": candidate.stat().st_size, + "sha256": sha256_file(candidate), + } + ) + return entries + + +def collect_output_artifacts( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any] | None = None, + output: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + artifacts: list[dict[str, Any]] = [] + for path in sorted(output_dir.rglob("*")): + if path.is_dir(): + continue + artifact_type = infer_port_type(path) + mime_type, _ = mimetypes.guess_type(path.name) + if artifact_type == "json": + payload = load_payload_from_file(path) + else: + payload = extract_preview_data(path, artifact_type, mime_type or "") + binding = output + if binding is None and node_data is not None: + binding = output_binding(node_data) + artifacts.append(build_artifact(run_id, node_id, path, artifact_type, data=payload, output=binding)) + return artifacts + + +def hyperparams(node_data: dict[str, Any]) -> dict[str, str]: + values: dict[str, str] = {} + for item in node_data.get("hyperparams", []) or []: + key = str(item.get("key", "")).strip() + if not key: + continue + values[key] = str(item.get("value", "")) + return values + + +def parse_json_param(params: dict[str, str], key: str, default: Any) -> Any: + raw = params.get(key, "").strip() + if not raw: + return default + try: + return json.loads(raw) + except Exception: + return default + + +def payloads_from_inputs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [load_artifact_payload(item) for item in inputs] + + +def collect_items(payloads: list[dict[str, Any]]) -> list[Any]: + items: list[Any] = [] + for payload in payloads: + if isinstance(payload.get("items"), list): + items.extend(payload["items"]) + elif isinstance(payload.get("entries"), list): + items.extend(payload["entries"]) + elif isinstance(payload.get("rows"), list): + items.extend(payload["rows"]) + else: + items.append(payload) + return items + + +def get_field_value(value: Any, field_path: str) -> Any: + current = value + for token in [part for part in field_path.split(".") if part]: + if isinstance(current, dict): + current = current.get(token) + continue + if isinstance(current, list) and token.isdigit(): + index = int(token) + if 0 <= index < len(current): + current = current[index] + continue + return None + return current + + +def compare_values(actual: Any, operator: str, expected: Any) -> bool: + if operator == "exists": + return actual is not None + if operator == "missing": + return actual is None + if operator == "contains": + if isinstance(actual, list): + return expected in actual + return str(expected) in str(actual) + if operator == "in": + if isinstance(expected, list): + return actual in expected + return str(actual) in str(expected) + if operator in {"gt", "gte", "lt", "lte"}: + try: + actual_number = float(actual) + expected_number = float(expected) + except Exception: + return False + if operator == "gt": + return actual_number > expected_number + if operator == "gte": + return actual_number >= expected_number + if operator == "lt": + return actual_number < expected_number + return actual_number <= expected_number + if operator == "ne": + return actual != expected + return actual == expected + + +def flatten_context(value: Any, prefix: str = "") -> dict[str, str]: + context: dict[str, str] = {} + if isinstance(value, dict): + for key, item in value.items(): + next_prefix = f"{prefix}.{key}" if prefix else str(key) + context.update(flatten_context(item, next_prefix)) + if prefix: + context[prefix] = json.dumps(value, ensure_ascii=False) + return context + if isinstance(value, list): + for index, item in enumerate(value[:8]): + next_prefix = f"{prefix}.{index}" if prefix else str(index) + context.update(flatten_context(item, next_prefix)) + if prefix: + context[prefix] = json.dumps(value, ensure_ascii=False) + return context + if prefix: + context[prefix] = "" if value is None else str(value) + return context + + +def render_template_text(template_text: str, context: dict[str, str]) -> str: + rendered = template_text + for key, value in context.items(): + rendered = rendered.replace("{{" + key + "}}", value) + safe_context = {key: value for key, value in context.items()} + + class SafeDict(dict): + def __missing__(self, key: str) -> str: + return "{" + key + "}" + + try: + rendered = rendered.format_map(SafeDict(safe_context)) + except Exception: + pass + return rendered + + +def rows_from_payloads(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for item in collect_items(payloads): + if isinstance(item, dict): + rows.append(item) + else: + rows.append({"value": item}) + return rows + + +def write_text_artifact( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + filename: str, + content: str, + output: dict[str, Any] | None = None, +) -> dict[str, Any]: + path = output_dir / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + mime_type, _ = mimetypes.guess_type(path.name) + preview = extract_preview_data(path, infer_port_type(path), mime_type or "") + return build_artifact(run_id, node_id, path, infer_port_type(path), data=preview, output=output) + + +def write_csv_artifact( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + filename: str, + rows: list[dict[str, Any]], + output: dict[str, Any] | None = None, +) -> dict[str, Any]: + path = output_dir / filename + path.parent.mkdir(parents=True, exist_ok=True) + columns: list[str] = [] + for row in rows: + for key in row.keys(): + if key not in columns: + columns.append(key) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=columns or ["value"]) + writer.writeheader() + if rows: + writer.writerows(rows) + preview = preview_tabular_payload(path, ",", "csv") + return build_artifact(run_id, node_id, path, "file", data=preview, output=output) + + +def builtin_source( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + source_path = params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "").strip() + prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip() + bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip() + pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip() + + entries: list[dict[str, Any]] = [] + if source_path: + entries = list_source_entries(pathlib.Path(source_path), pattern) + elif prefix and pathlib.Path(prefix).exists(): + entries = list_source_entries(pathlib.Path(prefix), pattern) + elif params.get("SOURCE_JSON"): + parsed = json.loads(params["SOURCE_JSON"]) + if isinstance(parsed, list): + entries = [{"value": item} for item in parsed] + else: + entries = [{"value": parsed}] + elif params.get("SOURCE_TEXT"): + entries = [{"text": params["SOURCE_TEXT"]}] + elif inputs: + entries = [load_artifact_payload(item) for item in inputs] + + payload = { + "bucket": bucket, + "prefix": prefix, + "pattern": pattern, + "sourcePath": source_path, + "entries": entries, + } + return [write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data))] + + +def builtin_merge( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + payloads = payloads_from_inputs(inputs) + merged = merge_dicts(payloads) + merged["inputCount"] = len(payloads) + return [write_json_artifact(run_id, node_id, output_dir, "merged.json", merged, output=output_binding(node_data))] + + +def builtin_intersection( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + payloads = payloads_from_inputs(inputs) + if not payloads: + return [ + write_json_artifact( + run_id, + node_id, + output_dir, + "intersection.json", + {"keys": [], "inputCount": 0}, + output=output_binding(node_data), + ) + ] + key_sets = [set(payload.keys()) for payload in payloads] + common_keys = sorted(set.intersection(*key_sets)) if key_sets else [] + payload = {"keys": common_keys, "values": {key: payloads[0].get(key) for key in common_keys}} + return [write_json_artifact(run_id, node_id, output_dir, "intersection.json", payload, output=output_binding(node_data))] + + +def builtin_distribution( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + payloads = payloads_from_inputs(inputs) + items = collect_items(payloads) + outputs = node_data.get("outputs", []) or [{"name": "queue-a", "type": "json"}] + partitions: list[list[Any]] = [[] for _ in outputs] + for index, item in enumerate(items): + partitions[index % len(outputs)].append(item) + artifacts: list[dict[str, Any]] = [] + for index, output in enumerate(outputs): + filename = f"{output.get('name', f'queue-{index}')}.json" + payload = {"items": partitions[index], "count": len(partitions[index])} + artifacts.append( + write_json_artifact( + run_id, + node_id, + output_dir, + filename, + payload, + str(output.get("type", "json")), + output=output, + ) + ) + return artifacts + + +def builtin_upload( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + copied_paths: list[str] = [] + bundle_inputs: list[dict[str, Any]] = [] + staging_dir = output_dir / "uploaded" + staging_dir.mkdir(parents=True, exist_ok=True) + + for artifact in inputs: + payload = load_artifact_payload(artifact) + bundle_inputs.append(payload) + source_path = artifact.get("path") + if isinstance(source_path, str) and source_path and os.path.exists(source_path): + source = pathlib.Path(source_path) + destination = staging_dir / source.name + if source.resolve() != destination.resolve(): + shutil.copy2(source, destination) + copied_paths.append(str(destination)) + + payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)} + artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))] + artifacts.extend(collect_output_artifacts(run_id, node_id, staging_dir)) + return artifacts + + +def builtin_json_extract( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + field_path = params.get("FIELD_PATH", "").strip() + payloads = payloads_from_inputs(inputs) + payload = payloads[0] if payloads else {} + extracted = get_field_value(payload, field_path) if field_path else payload + result = {"fieldPath": field_path, "value": extracted} + return [write_json_artifact(run_id, node_id, output_dir, "extracted.json", result, output=output_binding(node_data))] + + +def builtin_json_filter( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + field_path = params.get("FIELD_PATH", "").strip() + operator = params.get("OPERATOR", "eq").strip().lower() or "eq" + expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", "")) + rows = collect_items(payloads_from_inputs(inputs)) + filtered = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)] + payload = { + "fieldPath": field_path, + "operator": operator, + "expected": expected, + "items": filtered, + "count": len(filtered), + } + return [write_json_artifact(run_id, node_id, output_dir, "filtered.json", payload, output=output_binding(node_data))] + + +def builtin_json_map( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + field_map = parse_json_param(params, "FIELD_MAP_JSON", {}) + rows = rows_from_payloads(payloads_from_inputs(inputs)) + mapped = [{str(field_map.get(key, key)): value for key, value in row.items()} for row in rows] + payload = {"fieldMap": field_map, "items": mapped, "count": len(mapped)} + return [write_json_artifact(run_id, node_id, output_dir, "mapped.json", payload, output=output_binding(node_data))] + + +def builtin_text_template( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + template_text = params.get("TEMPLATE_TEXT", "").strip() + output_name = params.get("OUTPUT_NAME", "").strip() or "rendered.txt" + payloads = payloads_from_inputs(inputs) + context = flatten_context(payloads[0] if payloads else {}) + rendered = render_template_text(template_text, context) + binding = output_binding(node_data) + manifest = write_json_artifact( + run_id, + node_id, + output_dir, + "rendered-template.json", + {"template": template_text, "context": context, "outputName": output_name}, + output=binding, + ) + return [manifest, write_text_artifact(run_id, node_id, output_dir, output_name, rendered, output=binding)] + + +def builtin_csv_to_json( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + source_path = resolve_source_path(params, inputs) + payload: dict[str, Any] + if source_path is not None and source_path.suffix.lower() in TABULAR_FILE_SUFFIXES: + delimiter = TABULAR_FILE_SUFFIXES[source_path.suffix.lower()] + columns, rows = load_tabular_rows(source_path, delimiter) + payload = { + "sourcePath": str(source_path), + "columns": columns, + "items": rows, + "rowCount": len(rows), + } + else: + for artifact in inputs: + preview = artifact_preview_payload(artifact) or {} + if preview.get("kind") == "table": + payload = { + "sourcePath": str(artifact.get("path", "")), + "columns": preview.get("columns", []), + "items": preview.get("rows", []), + "rowCount": preview.get("rowCount", len(preview.get("rows", []))), + "previewTruncated": preview.get("truncated", False), + } + return [ + write_json_artifact( + run_id, + node_id, + output_dir, + "table.json", + payload, + output=output_binding(node_data), + ) + ] + raise RuntimeError("CSV 转 JSON 需要 SOURCE_PATH 或上游 artifact 指向真实 csv/tsv 文件") + return [write_json_artifact(run_id, node_id, output_dir, "table.json", payload, output=output_binding(node_data))] + + +def builtin_json_to_csv( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + rows = rows_from_payloads(payloads_from_inputs(inputs)) + binding = output_binding(node_data) + summary = write_json_artifact( + run_id, + node_id, + output_dir, + "csv-export-summary.json", + {"rowCount": len(rows), "columns": sorted({key for row in rows for key in row.keys()})}, + output=binding, + ) + return [summary, write_csv_artifact(run_id, node_id, output_dir, "rows.csv", rows, output=binding)] + + +def builtin_directory_scan( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + source_path = resolve_source_path(params, inputs) + if source_path is not None: + root = source_path if source_path.is_dir() else source_path.parent + payload = { + "sourcePath": str(source_path), + "scanRoot": str(root), + "summary": summarize_directory_tree(root), + } + elif inputs: + payload = { + "sourcePath": "", + "scanRoot": "", + "summary": summarize_input_artifacts(inputs), + } + else: + raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录") + return [write_json_artifact(run_id, node_id, output_dir, "directory-scan.json", payload, output=output_binding(node_data))] + + +def builtin_condition_router( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + field_path = params.get("FIELD_PATH", "").strip() + operator = params.get("OPERATOR", "eq").strip().lower() or "eq" + expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", "")) + rows = collect_items(payloads_from_inputs(inputs)) + matched = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)] + unmatched = [row for row in rows if not compare_values(get_field_value(row, field_path), operator, expected)] + outputs = node_data.get("outputs", []) or [ + {"id": "matched", "name": "matched", "type": "json"}, + {"id": "unmatched", "name": "unmatched", "type": "json"}, + ] + return [ + write_json_artifact( + run_id, + node_id, + output_dir, + f"{str(outputs[0].get('name', 'matched')).strip() or 'matched'}.json", + {"fieldPath": field_path, "operator": operator, "expected": expected, "items": matched, "count": len(matched)}, + str(outputs[0].get("type", "json")), + output=outputs[0], + ), + write_json_artifact( + run_id, + node_id, + output_dir, + f"{str(outputs[1].get('name', 'unmatched')).strip() or 'unmatched'}.json", + {"fieldPath": field_path, "operator": operator, "expected": expected, "items": unmatched, "count": len(unmatched)}, + str(outputs[1].get("type", "json")), + output=outputs[1], + ), + ] + + +def builtin_lerobot_summary( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + source_path = resolve_source_path(params, inputs) + if source_path is not None: + root = find_lerobot_root(source_path) or (source_path if source_path.is_dir() else source_path.parent) + payload = summarize_lerobot_root(root) + payload["sourcePath"] = str(source_path) + elif inputs: + payload = summarize_lerobot_from_inputs(inputs) + payload["sourcePath"] = "" + else: + raise RuntimeError("LeRobot 数据集概览需要 SOURCE_PATH 或上游 artifact 指向真实数据集目录") + payload["datasetType"] = "lerobot" + return [write_json_artifact(run_id, node_id, output_dir, "lerobot-summary.json", payload, output=output_binding(node_data))] + + +def builtin_rosbag_summary( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + params = hyperparams(node_data) + source_path = resolve_source_path(params, inputs) + if source_path is not None: + if source_path.is_dir(): + payload = { + "datasetType": "rosbag2", + "sourcePath": str(source_path), + "summary": summarize_directory_tree(source_path), + } + metadata_yaml = source_path / "metadata.yaml" + if metadata_yaml.exists(): + payload["metadata"] = preview_text_payload(metadata_yaml, "yaml")["preview"] + db3_files = [candidate for candidate in source_path.rglob("*.db3") if candidate.is_file()] + if db3_files: + payload["databasePreview"] = summarize_sqlite_payload(db3_files[0])["preview"] + return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))] + payload = summarize_rosbag_file(source_path) + payload["datasetType"] = payload.get("preview", {}).get("format", "rosbag") + payload["sourcePath"] = str(source_path) + else: + for artifact in inputs: + preview = artifact_preview_payload(artifact) or {} + if preview.get("kind") == "dataset" and preview.get("format") in {"rosbag2", "rosbag1", "mcap", "sqlite"}: + payload = { + "datasetType": preview.get("format", "rosbag"), + "sourcePath": str(artifact.get("path", "")), + "preview": preview, + } + return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))] + raise RuntimeError("ROS bag 概览需要 SOURCE_PATH 或上游 artifact 指向真实 bag/db3/mcap 文件") + return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))] + + +def builtin_generic( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_label: str, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + payloads = [load_artifact_payload(item) for item in inputs] + summary = { + "nodeLabel": node_label, + "inputCount": len(inputs), + "inputs": [summarize_json_like(item) for item in payloads], + "config": { + "bucket": node_data.get("bucket", ""), + "prefix": node_data.get("prefix", ""), + "pattern": node_data.get("pattern", ""), + }, + } + outputs = node_data.get("outputs", []) or [{"name": "result", "type": "json"}] + artifacts: list[dict[str, Any]] = [] + for index, output in enumerate(outputs): + output_name = str(output.get("name", f"output-{index}")).strip() or f"output-{index}" + filename = f"{output_name}.json" + payload = {**summary, "outputName": output_name} + artifacts.append( + write_json_artifact( + run_id, + node_id, + output_dir, + filename, + payload, + str(output.get("type", "json")), + output=output, + ) + ) + return artifacts + + +def execute_command( + run_id: str, + node_id: str, + output_dir: pathlib.Path, + node_data: dict[str, Any], + command: str, +) -> list[dict[str, Any]]: + completed = subprocess.run( + command, + cwd=output_dir, + shell=True, + text=True, + capture_output=True, + env=os.environ.copy(), + check=False, + ) + for stream_name, content in (("stdout", completed.stdout), ("stderr", completed.stderr)): + for line in content.splitlines(): + if line.strip(): + log(f"[operator-runner][command][{stream_name}] {line}") + if completed.returncode != 0: + raise RuntimeError(f"command failed with exit code {completed.returncode}") + artifacts = collect_output_artifacts(run_id, node_id, output_dir, node_data=node_data) + if artifacts: + return artifacts + payload = {"stdout": completed.stdout, "stderr": completed.stderr} + return [write_json_artifact(run_id, node_id, output_dir, "command-output.json", payload, output=output_binding(node_data))] + + +def select_handler(node_label: str) -> str: + normalized = node_label.lower() + if "lerobot" in normalized: + return "lerobot" + if any(token in normalized for token in ("rosbag", "mcap", "ros bag")): + return "rosbag" + if any(token in normalized for token in ("csv 转 json", "csv转json", "csv to json", "tsv 转 json", "tsv转json", "tsv to json")): + return "csv_to_json" + if any(token in normalized for token in ("json 转 csv", "json转csv", "json to csv")): + return "json_to_csv" + if any(token in normalized for token in ("字段提取", "json提取", "json 提取", "extract")): + return "json_extract" + if any(token in normalized for token in ("json过滤", "json 过滤", "filter")): + return "json_filter" + if any(token in normalized for token in ("json映射", "json 映射", "rename", "映射重命名")): + return "json_map" + if any(token in normalized for token in ("模板渲染", "template")): + return "text_template" + if any(token in normalized for token in ("条件分流", "conditional", "router", "route")): + return "condition_router" + if any(token in normalized for token in ("目录扫描", "directory", "browse", "browse dir")): + return "directory_scan" + if any(token in normalized for token in ("merge", "合并")): + return "merge" + if any(token in normalized for token in ("intersect", "交集")): + return "intersection" + if any(token in normalized for token in ("distribution", "分配")): + return "distribution" + if any(token in normalized for token in ("upload", "export", "上传", "导出")): + return "upload" + if any(token in normalized for token in ("source", "scanner", "oss", "scan", "扫描")): + return "source" + return "generic" + + +def main() -> int: + run_id = os.getenv("RUN_ID", "").strip() + workflow_id = os.getenv("WORKFLOW_ID", "").strip() + node_id = os.getenv("NODE_ID", "").strip() + node_label = os.getenv("NODE_LABEL", "").strip() or node_id + output_dir = pathlib.Path(os.getenv("OUTPUT_DIR", tempfile_dir(run_id, node_id))) + output_dir.mkdir(parents=True, exist_ok=True) + + node_data = load_json_env("NODE_DATA_JSON", {}) + inputs = load_json_env("INPUT_ARTIFACTS_JSON", []) + started_at = now_iso() + start_cpu, start_memory = resource_snapshot() + + logs: list[dict[str, Any]] = [] + + def append_log(level: str, message: str) -> None: + logs.append({"timestamp": now_iso(), "level": level, "message": message, "operator": node_label}) + log(f"[operator-runner][{level.lower()}] {message}") + + append_log("INFO", f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir}") + append_log("INFO", "Preparing operator execution context") + + params = hyperparams(node_data) + command = params.get("COMMAND") or params.get("SCRIPT") + artifacts: list[dict[str, Any]] + if command: + append_log("INFO", f"Executing explicit command: {command}") + artifacts = execute_command(run_id, node_id, output_dir, node_data, command) + else: + handler = select_handler(node_label) + append_log("INFO", f"Selected built-in handler: {handler}") + if handler == "source": + artifacts = builtin_source(run_id, node_id, output_dir, node_data, inputs) + elif handler == "merge": + artifacts = builtin_merge(run_id, node_id, output_dir, node_data, inputs) + elif handler == "intersection": + artifacts = builtin_intersection(run_id, node_id, output_dir, node_data, inputs) + elif handler == "distribution": + artifacts = builtin_distribution(run_id, node_id, output_dir, node_data, inputs) + elif handler == "json_extract": + artifacts = builtin_json_extract(run_id, node_id, output_dir, node_data, inputs) + elif handler == "json_filter": + artifacts = builtin_json_filter(run_id, node_id, output_dir, node_data, inputs) + elif handler == "json_map": + artifacts = builtin_json_map(run_id, node_id, output_dir, node_data, inputs) + elif handler == "text_template": + artifacts = builtin_text_template(run_id, node_id, output_dir, node_data, inputs) + elif handler == "csv_to_json": + artifacts = builtin_csv_to_json(run_id, node_id, output_dir, node_data, inputs) + elif handler == "json_to_csv": + artifacts = builtin_json_to_csv(run_id, node_id, output_dir, node_data, inputs) + elif handler == "directory_scan": + artifacts = builtin_directory_scan(run_id, node_id, output_dir, node_data, inputs) + elif handler == "condition_router": + artifacts = builtin_condition_router(run_id, node_id, output_dir, node_data, inputs) + elif handler == "lerobot": + artifacts = builtin_lerobot_summary(run_id, node_id, output_dir, node_data, inputs) + elif handler == "rosbag": + artifacts = builtin_rosbag_summary(run_id, node_id, output_dir, node_data, inputs) + elif handler == "upload": + artifacts = builtin_upload(run_id, node_id, output_dir, node_data, inputs) + else: + artifacts = builtin_generic(run_id, node_id, output_dir, node_label, node_data, inputs) + + append_log("INFO", f"Produced {len(artifacts)} artifact(s)") + result = { + "logs": logs, + "artifacts": artifacts, + "resourceSeries": resource_series(start_cpu, start_memory, started_at), + } + print(RESULT_PREFIX + json.dumps(result, ensure_ascii=False), flush=True) + return 0 + + +def tempfile_dir(run_id: str, node_id: str) -> str: + base = pathlib.Path("/tmp/droboticflow") + return str(base / (run_id or "run") / (node_id or "node")) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as error: # pragma: no cover + log(f"[operator-runner][error] {error}") + sys.exit(1) diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 00000000..68fe2ef1 --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,86 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "droboticflow/internal/ai" + "droboticflow/internal/api" + "droboticflow/internal/config" + "droboticflow/internal/retry" + "droboticflow/internal/store" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) { + return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) { + client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI)) + if err != nil { + return nil, err + } + if err := client.Ping(ctx, nil); err != nil { + _ = client.Disconnect(ctx) + return nil, err + } + return client, nil + }) +} + +func spaHandler(apiHandler http.Handler, distDir string) http.Handler { + fileServer := http.FileServer(http.Dir(distDir)) + indexPath := filepath.Join(distDir, "index.html") + + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if strings.HasPrefix(request.URL.Path, "/api/") || strings.HasPrefix(request.URL.Path, "/worker-api/") { + apiHandler.ServeHTTP(writer, request) + return + } + if request.URL.Path == "/" { + fileServer.ServeHTTP(writer, request) + return + } + targetPath := filepath.Join(distDir, filepath.Clean(request.URL.Path)) + if _, err := os.Stat(targetPath); err == nil { + fileServer.ServeHTTP(writer, request) + return + } + http.ServeFile(writer, request, indexPath) + }) +} + +func main() { + ctx := context.Background() + cfg := config.Load() + client, err := connectMongo(ctx, cfg.MongoURI, cfg.MongoConnectRetries, time.Duration(cfg.MongoConnectRetryDelayMS)*time.Millisecond) + if err != nil { + fmt.Printf("[api] failed to connect mongo: %v\n", err) + os.Exit(1) + } + defer func() { + _ = client.Disconnect(context.Background()) + }() + + dataStore := store.NewMongoStore(client.Database(cfg.MongoDB)) + + aiService := ai.NewService(cfg.AIAPIBaseURL, cfg.AIAPIKey, cfg.AIModel) + handler := spaHandler(api.NewHandler(dataStore, aiService), cfg.DistDir) + server := &http.Server{ + Addr: fmt.Sprintf("0.0.0.0:%d", cfg.Port), + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, + } + + fmt.Printf("[api] listening on http://0.0.0.0:%d\n", cfg.Port) + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + fmt.Printf("[api] failed to start: %v\n", err) + os.Exit(1) + } +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..6579d5dc --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "time" + + "droboticflow/internal/config" + "droboticflow/internal/retry" + "droboticflow/internal/store" + "droboticflow/internal/worker" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) { + return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) { + client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI)) + if err != nil { + return nil, err + } + if err := client.Ping(ctx, nil); err != nil { + _ = client.Disconnect(ctx) + return nil, err + } + return client, nil + }) +} + +func main() { + ctx := context.Background() + cfg := config.Load() + client, err := connectMongo(ctx, cfg.MongoURI, cfg.MongoConnectRetries, time.Duration(cfg.MongoConnectRetryDelayMS)*time.Millisecond) + if err != nil { + fmt.Printf("[worker] failed to connect mongo: %v\n", err) + os.Exit(1) + } + defer func() { + _ = client.Disconnect(context.Background()) + }() + + dataStore := store.NewMongoStore(client.Database(cfg.MongoDB)) + mode := "local" + if cfg.K8sEnabled { + mode = "k8s" + } + runtime, err := worker.NewRuntime(dataStore, mode, cfg.K8sNamespace, cfg.K8sImagePullSecretName, cfg.OperatorRunnerImage, cfg.LocalOperatorRunnerCmd) + if err != nil { + fmt.Printf("[worker] failed to initialize runtime: %v\n", err) + os.Exit(1) + } + + go func() { + ticker := time.NewTicker(time.Duration(cfg.PollIntervalMS) * time.Millisecond) + defer ticker.Stop() + for range ticker.C { + if err := runtime.ProcessAvailable(context.Background()); err != nil { + fmt.Printf("[worker] processAvailable failed: %v\n", err) + } + } + }() + + server := &http.Server{ + Addr: fmt.Sprintf("0.0.0.0:%d", cfg.WorkerPort), + Handler: worker.NewHTTPHandler(dataStore, runtime), + ReadHeaderTimeout: 10 * time.Second, + } + + fmt.Printf("[worker] listening on http://0.0.0.0:%d\n", cfg.WorkerPort) + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + fmt.Printf("[worker] failed to start: %v\n", err) + os.Exit(1) + } +} diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 00000000..6b847a1c --- /dev/null +++ b/docs/api.md @@ -0,0 +1,261 @@ +# API Reference + +## REST API + +Base path: `/api` + +### Health and Dashboard + +- `GET /api/health`: service health check +- `GET /api/ai/status`: returns whether server-side AI generation is enabled and which model is configured +- `POST /api/ai/operators/generate`: generate a custom operator definition from natural language +- `POST /api/ai/workflows/generate`: generate a workflow graph from natural language and the current operator catalog +- `GET /api/dashboard`: aggregate counts for projects, operators, jobs, and audits + +`POST /api/ai/operators/generate` + +```json +{ + "description": "生成一个算子,用于接收视频输入,抽取关键帧并输出图片列表。" +} +``` + +`POST /api/ai/workflows/generate` + +```json +{ + "description": "从对象存储读取视频,抽帧后调用目标检测算子,再把检测结果回传到对象存储。" +} +``` + +### Projects + +- `GET /api/projects` +- `POST /api/projects` +- `GET /api/projects/:id` +- `PUT /api/projects/:id` +- `DELETE /api/projects/:id` +- `GET /api/projects/:id/workflows` +- `POST /api/projects/:id/workflows` + +`POST /api/projects` + +```json +{ + "name": "自动驾驶新项目", + "description": "处理多摄像头感知数据" +} +``` + +### Workflows + +- `GET /api/workflows/:id` +- `PUT /api/workflows/:id` +- `DELETE /api/workflows/:id` +- `POST /api/workflows/:id/run` + +`PUT /api/workflows/:id` + +```json +{ + "name": "视觉处理流程", + "description": "包含扫描、推理、上传", + "type": "batch", + "maxConcurrentRuns": 2, + "nodeTimeoutSeconds": 120, + "runTimeoutSeconds": 300, + "maxRetryAttempts": 1, + "retryBackoffSeconds": 30, + "nodes": [], + "edges": [] +} +``` + +`POST /api/workflows/:id/run` + +```json +{ + "trigger": "manual" +} +``` + +### Templates + +- `GET /api/templates` +- `POST /api/templates` +- `GET /api/templates/:id` +- `PUT /api/templates/:id` +- `DELETE /api/templates/:id` + +Templates now persist the same workflow-level defaults used by project workflows, including `type`, `schedule`, `maxConcurrentRuns`, `nodeTimeoutSeconds`, `runTimeoutSeconds`, `maxRetryAttempts`, and `retryBackoffSeconds`. + +### Operators + +- `GET /api/operators` +- `POST /api/operators` +- `GET /api/operators/:id` +- `PUT /api/operators/:id` +- `DELETE /api/operators/:id` + +The reconciled built-in catalog includes the original workflow primitives plus: + +- `目录扫描` +- `CSV 转 JSON` +- `JSON 转 CSV` +- `JSON 字段提取` +- `JSON 过滤` +- `JSON 映射重命名` +- `文本模板渲染` +- `条件分流` +- `LeRobot 数据集概览` +- `ROSBag 数据概览` +- `Shell 命令执行` + +`POST /api/operators` + +```json +{ + "name": "自定义分割算子", + "type": "custom", + "category": "AI", + "description": "执行图像分割", + "inputs": [{ "id": "in-1", "name": "图像", "type": "image" }], + "outputs": [{ "id": "out-1", "name": "结果", "type": "json" }], + "image": "ccr.example.com/dataflow/custom-seg:latest" +} +``` + +### Runs and Monitoring + +- `GET /api/runs` +- `GET /api/runs/:id` +- `GET /api/runs/:id/logs` +- `GET /api/runs/:id/resources` +- `GET /api/runs/:id/artifacts` + +`GET /api/runs/:id/artifacts` + +Each artifact item can now include: + +- `portId` / `portName`: the originating output port so downstream edges can route multi-output operators correctly +- `metadata.mimeType` / `metadata.sha256`: persisted file metadata +- `data`: the real JSON payload for JSON artifacts +- `data.preview`: a compact preview object for non-JSON artifacts such as text, CSV/TSV, JSONL, image thumbnails, audio/video metadata, PDF hints, SQLite/rosbag summaries, and LeRobot dataset summaries + +### Audit + +- `GET /api/audit-tasks` +- `POST /api/audit-tasks/:id/approve` +- `POST /api/audit-tasks/:id/reject` + +### Users and Settings + +- `GET /api/users` +- `PUT /api/users/:id` +- `GET /api/settings` +- `PUT /api/settings` + +`GET /api/settings` + +```json +{ + "id": "system", + "k8s": { + "enabled": true, + "endpoint": "https://120.48.161.230:6443", + "namespace": "droboticflow", + "hasKubeconfig": true + }, + "worker": { + "pollIntervalMs": 2000, + "maxConcurrentRuns": 2 + }, + "ai": { + "provider": "dashscope-openai-compatible", + "baseURL": "https://coding.dashscope.aliyuncs.com/v1", + "model": "glm-5", + "hasAPIKey": true + }, + "timezone": "Asia/Shanghai" +} +``` + +`PUT /api/settings` + +```json +{ + "timezone": "Asia/Shanghai", + "worker": { + "maxConcurrentRuns": 2 + }, + "ai": { + "provider": "dashscope-openai-compatible", + "baseURL": "https://coding.dashscope.aliyuncs.com/v1", + "model": "glm-5", + "apiKey": "sk-..." + } +} +``` + +Workflow create/update payloads also accept workflow-level execution settings: + +- `0`: inherit the global worker default +- `1+`: workflow-specific maximum active runs + +Dispatch enforces both the global worker limit and the workflow-specific limit. + +- `runTimeoutSeconds` + - `0`: disable workflow-level timeout + - `1+`: fail the current run if total execution time exceeds this many seconds +- `nodeTimeoutSeconds` + - `0`: disable workflow-level per-node timeout + - `1+`: fail the current node if a single operator execution exceeds this many seconds +- `maxRetryAttempts` + - `0`: disable automatic retry + - `1+`: automatically enqueue up to N retry runs after a failed attempt +- `retryBackoffSeconds` + - `0`: enqueue retry runs immediately + - `1+`: delay retry dispatch by the configured number of seconds + +Run responses may also include: + +- `attempt`: current attempt number, starting from `1` +- `parentRunId`: the failed run that spawned this retry run +- `availableAt`: when this queued run becomes eligible for dispatch; retry backoff uses this field + +### Storage + +- `GET /api/storage/buckets?sourcePath=` +- `GET /api/storage/objects?sourcePath=&bucket=&prefix=` + +## Worker API + +Base path: `/worker-api` + +- `GET /worker-api/health`: worker health check +- `GET /worker-api/stats`: current queue/running/audit counts +- `POST /worker-api/process-once`: process one queued run immediately +- `POST /worker-api/runs/:id/retry`: reset a run to `queued` and clear prior execution state + +## Response Shape Notes + +- list endpoints return `{ "items": [...] }` +- delete endpoints return `204 No Content` +- validation failures return `400` +- missing resources return `404` +- AI service unavailable returns `503` +- upstream AI gateway failures return `502` +- unhandled server errors return `500` + +## Implementation Notes + +- `/api/*` is served by the Go API binary in [`cmd/api`](/Users/longtaowu/workspace/droboticflow/cmd/api). +- `/worker-api/*` is served by the Go worker binary in [`cmd/worker`](/Users/longtaowu/workspace/droboticflow/cmd/worker). +- Workflow execution and storage models are defined in [`internal/domain`](/Users/longtaowu/workspace/droboticflow/internal/domain) and [`internal/store`](/Users/longtaowu/workspace/droboticflow/internal/store). +- AI generation is implemented by [`internal/ai`](/Users/longtaowu/workspace/droboticflow/internal/ai) and is called only from the Go API, never directly from the browser. +- `GET /api/settings` never returns the saved AI key in plaintext. It only exposes `ai.hasAPIKey` so the browser can show whether a server-side key is already configured. +- `timezone` uses IANA timezone identifiers such as `Asia/Shanghai` or `America/Los_Angeles`. The frontend formats timestamps with second precision in that timezone. +- Monitoring previews are rendered from the persisted artifact payload itself, so text/table/robotics previews keep working after the Kubernetes Job has exited and its local filesystem is gone. +- OSS 相关配置不通过 `/api/settings` 管理。系统设置页只维护 K8s、worker、AI 和默认资源;OSS bucket/prefix 等参数由具体工作流节点维护。 +- `/api/storage/*` 也不再读取系统级存储根目录。浏览请求必须显式传入当前 OSS 节点超参里的 `SOURCE_PATH`,API 只在这个节点上下文内列 bucket 和对象。 +- Kubernetes Jobs launched for operator nodes use the Python entrypoint at [`backend/operator-runner/main.py`](/Users/longtaowu/workspace/droboticflow/backend/operator-runner/main.py). diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 00000000..2d85f742 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,141 @@ +# Deployment Guide + +## Images + +This project deploys three images: + +- `droboticflow-api` +- `droboticflow-worker` +- `droboticflow-operator-runner` + +Default Baidu registry naming: + +- `ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-api:` +- `ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-worker:` +- `ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:` + +## 1. Log In to the Registry + +Use the public endpoint unless you are deploying from the same VPC as the registry: + +```bash +docker login ccr-29eug8s3-pub.cnc.bj.baidubce.com --username +``` + +## 2. Build and Push Images + +```bash +IMAGE_TAG=20260408 ./scripts/build-and-push-images.sh +``` + +Image contents: + +- `droboticflow-api`: Vite frontend assets + Go API binary +- `droboticflow-worker`: Go worker binary +- `droboticflow-operator-runner`: Python runtime + `backend/operator-runner/main.py` + +The script prints the exact image URLs. Reuse those values for deployment: + +```bash +API_IMAGE=... +WORKER_IMAGE=... +OPERATOR_RUNNER_IMAGE=... +``` + +If the target cluster cannot pull `mongo:7` directly from Docker Hub, mirror it to the same Baidu registry: + +```bash +docker buildx build \ + --platform linux/amd64 \ + -t ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-mongo:7-amd64 \ + --push \ + - <<'EOF' +FROM mongo:7 +EOF +``` + +## 3. Prepare Kubernetes Access + +Point `kubectl` at the target cluster before deploying. The deployment scripts do not store kubeconfig in the repository. + +Recommended environment variables: + +```bash +export K8S_NAMESPACE=droboticflow +export API_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-api:20260408 +export WORKER_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-worker:20260408 +export OPERATOR_RUNNER_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:20260408 +export MONGO_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-mongo:7-amd64 +export REGISTRY_SERVER=ccr-29eug8s3-pub.cnc.bj.baidubce.com +export REGISTRY_USERNAME= +export REGISTRY_PASSWORD= +export K8S_IMAGE_PULL_SECRET_NAME=droboticflow-registry +export AI_API_BASE_URL=https://coding.dashscope.aliyuncs.com/v1 +export AI_MODEL=glm-5 +export AI_API_KEY= +``` + +## 4. Deploy + +```bash +./scripts/deploy-k8s.sh +``` + +The deploy script performs these actions: + +1. creates the namespace if it does not exist +2. creates or updates the image pull secret +3. creates or updates MongoDB credentials, reusing the existing Secret values when present +4. creates or updates the `droboticflow-ai-secret` Secret for server-side AI generation +5. renders `k8s/droboticflow.template.yaml` +6. applies the manifest to the cluster +7. waits for MongoDB, API, and worker rollouts + +## 5. Services Created + +- `mongodb`: ClusterIP service for MongoDB +- `droboticflow-api`: NodePort service that exposes the frontend and API +- `droboticflow-worker`: ClusterIP service for worker health/debugging + +Default API NodePort: + +- `30080` + +If the cluster node IP is reachable, the UI and API are exposed at: + +```text +http://:30080 +``` + +## 6. Manifest Variables + +`scripts/render-manifests.sh` accepts: + +- `K8S_NAMESPACE` +- `MONGO_DB` +- `MONGO_CONNECT_RETRIES` +- `MONGO_CONNECT_RETRY_DELAY_MS` +- `WORKER_POLL_INTERVAL_MS` +- `API_NODE_PORT` +- `API_REPLICAS` +- `WORKER_REPLICAS` +- `K8S_IMAGE_PULL_SECRET_NAME` +- `AI_API_BASE_URL` +- `AI_MODEL` +- `API_IMAGE` +- `WORKER_IMAGE` +- `OPERATOR_RUNNER_IMAGE` +- `MONGO_IMAGE` + +## 7. Notes + +- The provided manifest uses a single-replica MongoDB Deployment with `emptyDir` storage so it can run on clusters that do not expose a default `StorageClass`. +- MongoDB should be mirrored to the Baidu registry as an `amd64` image when the cluster cannot pull from Docker Hub. +- For production, replace the `emptyDir` volume with a PVC-backed deployment or StatefulSet. +- API and worker entrypoints are Go binaries with Mongo startup retry, so they tolerate the database becoming ready after the Deployments start. +- The worker runs with Kubernetes RBAC that allows creating and watching Jobs in the namespace. +- The operator-runner Job template inherits the private registry pull secret name from `K8S_IMAGE_PULL_SECRET_NAME`. +- The operator-runner is not a standalone Deployment. It is launched on demand as a Python Job container. +- AI generation runs through the API Deployment. `AI_API_KEY` is mounted through the `droboticflow-ai-secret` Secret and must not be baked into frontend env vars. +- If you use DashScope-compatible models, prefer a backend-allowed API key. Do not deploy tool-only Coding Plan keys into the cluster. +- Production hardening still depends on the target cluster policy: TLS ingress, network policies, backup strategy, and secret rotation. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 00000000..788d9006 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,125 @@ +# Operations Guide + +## Service Responsibilities + +### API + +- validates incoming payloads +- persists projects, workflows, operators, templates, settings, runs, and audit tasks +- serves the built SPA in production +- runs as a single Go binary + +### Worker + +- polls MongoDB for queued runs +- auto-enqueues streaming workflows based on `schedule.interval` +- executes workflow nodes in topological order +- pauses runs on human-review nodes +- resumes approved or rejected audit branches +- creates Kubernetes Jobs when `K8S_ENABLED=true` +- runs as a single Go binary + +### Operator Runner + +- receives `RUN_ID`, `WORKFLOW_ID`, `NODE_ID`, and `NODE_LABEL` +- executes a generic Python operator contract +- emits logs for the worker to associate with the run + +## MongoDB Collections + +- `projects` +- `operators` +- `workflows` +- `templates` +- `runs` +- `auditTasks` +- `users` +- `settings` + +## Run Lifecycle + +1. frontend calls `POST /api/workflows/:id/run` +2. API inserts a `queued` run document +3. worker claims the run and marks it `running` +4. worker executes start nodes and records logs, resource samples, and artifacts +5. if a node requires review, the run moves to `waiting_for_audit` +6. audit action updates the task to `approved` or `rejected` +7. worker resumes the graph from the appropriate edge +8. run ends in `completed` or `failed` + +## Local Verification + +Run the full verification loop: + +```bash +npm run verify +``` + +Start the stack locally: + +```bash +npm run dev:full +``` + +Smoke test the API and worker: + +```bash +./scripts/smoke-test.sh +``` + +Reconcile persistent system data and remove legacy demo records: + +```bash +./scripts/reconcile-system-data.sh http://127.0.0.1:3001 +``` + +## Kubernetes Verification + +Check workloads: + +```bash +kubectl -n droboticflow get pods +kubectl -n droboticflow get jobs +kubectl -n droboticflow get svc +``` + +Inspect logs: + +```bash +kubectl -n droboticflow logs deploy/droboticflow-api +kubectl -n droboticflow logs deploy/droboticflow-worker +kubectl -n droboticflow logs job/ +``` + +## Common Failure Modes + +### API starts but returns Mongo errors + +- confirm `mongodb` pod is healthy +- confirm the app secret and database name match the Mongo init script +- inspect `kubectl logs deploy/mongodb` + +### Worker cannot launch Jobs + +- confirm the worker pod is running with service account `droboticflow-worker` +- confirm Role and RoleBinding exist in the namespace +- inspect worker logs for Kubernetes API errors +- confirm the operator-runner image pull secret name matches `K8S_IMAGE_PULL_SECRET_NAME` + +### Image pull failure + +- recreate `droboticflow-registry` secret with valid registry credentials +- confirm the image tags pushed to the registry match the deployment values +- confirm operator-runner Jobs inherit the same pull secret via the rendered Job manifest + +### UI cannot load data + +- confirm the API pod is healthy at `/api/health` +- confirm the NodePort is reachable from the target network +- inspect browser network traffic for failing endpoints + +### Old demo data still appears after an upgrade + +- run `./scripts/reconcile-system-data.sh http://:30080` +- confirm `/api/users`, `/api/projects`, and `/api/templates` no longer return legacy seeded records +- confirm `/api/operators` only returns the current built-in operator catalog with the active operator-runner image tag diff --git a/docs/plans/2026-04-08-droboticflow-design.md b/docs/plans/2026-04-08-droboticflow-design.md new file mode 100644 index 00000000..8b7f6d03 --- /dev/null +++ b/docs/plans/2026-04-08-droboticflow-design.md @@ -0,0 +1,301 @@ +# DRoboticFlow Full-Stack Design + +## Goal + +Turn the current frontend prototype into a deployable workflow platform with: + +- a real backend API for all existing pages +- a real worker API and execution engine +- MongoDB persistence +- Kubernetes-based operator execution +- Baidu Cloud Container Registry compatible image build and deployment flow + +## Current State + +The repository currently contains: + +- a Vite/React frontend that started as a local-state prototype +- a single `server.ts` demo file that mixed API placeholders, in-memory storage, and a non-production worker loop +- no persistent database model +- no real worker service +- no deployment manifests, registry flow, or operations documentation + +## Target Architecture + +### Services + +1. `api` + - Go HTTP service for frontend-facing REST APIs + - serves the built SPA in production + - owns validation, CRUD operations, run creation, audit actions, settings, and dashboard aggregation + +2. `worker` + - Go HTTP service for worker health and control APIs + - background polling loop for queued runs and scheduled streaming workflows + - executes workflow DAGs node-by-node + - integrates with Kubernetes Jobs when enabled + +3. `operator-runner` + - lightweight Python container entrypoint used by Kubernetes Jobs + - executes a generic operator task contract for built-in operators + - produces logs and artifacts back into Mongo-backed run records + +4. `mongodb` + - persistent document store for projects, operators, workflows, templates, runs, audit tasks, users, settings, and logs + +### Runtime Modes + +- `local` + - used for tests and local development + - worker executes the real Python operator runner or a real local Docker image without requiring a Kubernetes cluster + +- `k8s` + - worker creates Kubernetes Jobs for workflow nodes + - jobs run container images pulled from Baidu Cloud Container Registry + +## Data Model + +### `projects` + +- `id` +- `name` +- `description` +- `status`: `active | archived` +- `createdAt` +- `updatedAt` + +### `operators` + +- `id` +- `name` +- `type`: `builtin | custom` +- `category` +- `description` +- `inputs` +- `outputs` +- `image` +- `skillDefinition` +- `hyperparams` +- `resources`: cpu and memory defaults +- `createdAt` +- `updatedAt` + +### `workflows` + +- `id` +- `projectId` +- `name` +- `description` +- `type`: `batch | streaming` +- `nodes` +- `edges` +- `schedule` +- `status`: `draft | ready | running | completed | failed` +- `lastRunId` +- `lastRunAt` +- `createdAt` +- `updatedAt` + +### `templates` + +- `id` +- `name` +- `description` +- `nodes` +- `edges` +- `createdAt` +- `updatedAt` + +### `runs` + +- `id` +- `workflowId` +- `projectId` +- `workflowName` +- `projectName` +- `trigger`: `manual | scheduled | retry` +- `status`: `queued | running | waiting_for_audit | completed | failed | cancelled` +- `startedAt` +- `endedAt` +- `durationMs` +- `currentNodeId` +- `nodeExecutions` +- `resourceSeries` +- `artifacts` +- `k8sRefs` +- `error` +- `createdAt` +- `updatedAt` + +### `auditTasks` + +- `id` +- `runId` +- `workflowId` +- `projectId` +- `nodeId` +- `workflowName` +- `projectName` +- `operatorName` +- `status`: `pending | approved | rejected` +- `data` +- `createdAt` +- `updatedAt` + +### `users` + +- `id` +- `name` +- `email` +- `role`: `admin | auditor | operator` +- `createdAt` +- `updatedAt` + +### `settings` + +- singleton document keyed by `system` +- `k8s.namespace` +- `k8s.endpoint` +- `k8s.enabled` +- `worker.pollIntervalMs` +- `worker.maxConcurrentRuns` +- `storage.buckets` +- `defaults.cpuRequest` +- `defaults.cpuLimit` +- `defaults.memoryRequest` +- `defaults.memoryLimit` +- `updatedAt` + +## API Surface + +### Frontend API + +- `GET /api/health` +- `GET /api/dashboard` +- `GET /api/projects` +- `POST /api/projects` +- `GET /api/projects/:id` +- `PUT /api/projects/:id` +- `DELETE /api/projects/:id` +- `GET /api/projects/:id/workflows` +- `POST /api/projects/:id/workflows` +- `GET /api/workflows/:id` +- `PUT /api/workflows/:id` +- `DELETE /api/workflows/:id` +- `POST /api/workflows/:id/run` +- `GET /api/templates` +- `POST /api/templates` +- `GET /api/templates/:id` +- `PUT /api/templates/:id` +- `DELETE /api/templates/:id` +- `GET /api/operators` +- `POST /api/operators` +- `GET /api/operators/:id` +- `PUT /api/operators/:id` +- `DELETE /api/operators/:id` +- `GET /api/runs` +- `GET /api/runs/:id` +- `GET /api/runs/:id/logs` +- `GET /api/runs/:id/resources` +- `GET /api/runs/:id/artifacts` +- `GET /api/audit-tasks` +- `POST /api/audit-tasks/:id/approve` +- `POST /api/audit-tasks/:id/reject` +- `GET /api/users` +- `PUT /api/users/:id` +- `GET /api/settings` +- `PUT /api/settings` +- `GET /api/storage/buckets` +- `GET /api/storage/objects` + +### Worker API + +- `GET /worker-api/health` +- `GET /worker-api/stats` +- `POST /worker-api/process-once` +- `POST /worker-api/runs/:id/retry` + +## Workflow Execution Model + +### Queueing + +1. API validates a workflow. +2. API creates a `run` with `status=queued`. +3. Worker atomically claims the run. + +### Execution + +1. Worker builds a DAG from workflow `nodes` and `edges`. +2. Start nodes with no incoming edges are scheduled first. +3. Each node transitions through: + - `queued` + - `running` + - `completed` + - `failed` + - `waiting_for_audit` when the operator requires manual review +4. Downstream nodes run only after all dependencies complete. + +### Audit Gate + +If a node is a human review operator: + +1. Worker creates an `auditTask`. +2. Run status becomes `waiting_for_audit`. +3. Approve/reject API updates the task. +4. Worker resumes the run on the corresponding branch. + +### Kubernetes Launch + +When Kubernetes execution is enabled: + +1. Worker resolves the node image. +2. Worker creates a namespaced Job with run and node metadata as env vars. +3. Worker watches Job completion and collects pod/job references. +4. Worker stores status, logs, and basic resource samples in Mongo. + +When disabled, the same state machine runs through the local execution path for deterministic tests and development. + +## Frontend Integration Strategy + +The existing UI is kept, but local prototype state is replaced with: + +- shared API client utilities +- fetch-based page loading and mutation handlers +- run, monitoring, audit, settings, and storage views backed by real endpoints + +The visual design stays intact; only data flow changes. + +## Deployment Topology + +### Containers + +- `droboticflow-api` +- `droboticflow-worker` +- `droboticflow-operator-runner` + +### Registry + +Baidu Cloud Container Registry namespace: + +- `ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow` + +### Kubernetes Resources + +- namespace +- MongoDB secret, PVC, service, deployment/statefulset +- API configmap, secret, deployment, service +- Worker configmap, secret, deployment, service +- image pull secret for Baidu registry + +## Acceptance Criteria + +The project is considered aligned when: + +- frontend pages read and write real backend data +- workflow runs are persisted in MongoDB +- worker processes queued runs and updates monitoring data +- audit tasks can pause and resume runs +- settings are persisted and exposed via API +- Docker images can be built and tagged for Baidu registry +- Kubernetes manifests can deploy MongoDB, API, and worker +- repo docs explain local development, deployment, and operations diff --git a/docs/plans/2026-04-08-droboticflow-implementation.md b/docs/plans/2026-04-08-droboticflow-implementation.md new file mode 100644 index 00000000..0897b8b7 --- /dev/null +++ b/docs/plans/2026-04-08-droboticflow-implementation.md @@ -0,0 +1,122 @@ +# DRoboticFlow Full-Stack Implementation Plan + +**Goal:** Replace the prototype backend with a Go-based MongoDB-backed API and worker stack, wire the frontend to it, and add Kubernetes plus Baidu registry deployment assets. + +**Architecture:** Keep the existing React frontend, add shared Go domain modules plus separate API and worker entrypoints, and use a Kubernetes launcher abstraction with a real local runner for development plus Kubernetes Jobs in cluster mode. Use a Python operator-runner container for Kubernetes Jobs. Update docs and deployment assets in the same change set so runtime assumptions stay aligned. + +**Tech Stack:** React, Vite, Go, MongoDB, Go test, client-go, Python, Docker, Kubernetes YAML + +--- + +### Task 1: Scaffold backend structure and shared models + +**Files:** +- Create: `cmd/api/main.go` +- Create: `cmd/worker/main.go` +- Create: `backend/operator-runner/main.py` +- Create: `internal/domain/*.go` +- Create: `internal/config/*.go` +- Modify: `package.json` +- Modify: `tsconfig.json` +- Modify: `README.md` + +**Steps:** +1. Create Go entrypoints for the API and worker services. +2. Add shared domain types, config loading, and identifier helpers. +3. Add the Python operator-runner entrypoint used inside Kubernetes Jobs. +4. Verify the binaries compile and the runtime contract is stable. + +### Task 2: Add Mongo repositories and default settings + +**Files:** +- Create: `internal/store/*.go` +- Test: `internal/store/*_test.go` + +**Steps:** +1. Add a store interface plus in-memory and Mongo implementations. +2. Port CRUD behavior, dashboard aggregation, default settings, and queue-claim logic. +3. Keep runtime bootstrap explicit so projects, operators, templates, users, runs, and audits are created through real APIs. +4. Re-run store tests until green. + +### Task 3: Implement API routes + +**Files:** +- Create: `internal/api/*.go` +- Test: `internal/api/*_test.go` + +**Steps:** +1. Write failing tests for dashboard, CRUD, run creation, audit actions, and storage endpoints. +2. Implement REST handlers with validation and consistent JSON responses. +3. Register `/api/*` routes and SPA fallback handling. +4. Re-run API tests until green. + +### Task 4: Implement worker run lifecycle + +**Files:** +- Create: `internal/worker/*.go` +- Create: `internal/retry/*.go` +- Test: `internal/worker/*_test.go` +- Test: `internal/retry/*_test.go` + +**Steps:** +1. Write failing tests for run claiming, DAG ordering, audit pause/resume, and streaming scheduling. +2. Implement queue claiming, topological execution, node logging, artifact/resource updates, and run status transitions. +3. Add worker control endpoints and polling loop. +4. Re-run worker and retry tests until green. + +### Task 5: Implement Kubernetes launcher and operator runner + +**Files:** +- Create: `internal/k8s/*.go` +- Create: `backend/operator-runner/main.py` +- Test: `internal/k8s/*_test.go` + +**Steps:** +1. Write failing tests for job spec generation and local execution fallback. +2. Implement launcher abstraction for local and Kubernetes modes using `client-go`. +3. Implement the Python operator-runner contract for generic built-in operators. +4. Re-run launcher tests until green. + +### Task 6: Refactor frontend to real API clients + +**Files:** +- Create: `src/services/api/client.ts` +- Create: `src/services/api/*.ts` +- Modify: `src/pages/*.tsx` +- Modify: `src/context/UserContext.tsx` + +**Steps:** +1. Replace local prototype state with fetch-backed hooks or page effects. +2. Wire project, workflow, operator, template, monitoring, audit, users, settings, and dashboard pages to the backend. +3. Re-run the frontend type-check and build. +4. Verify the existing UI still loads against the real API. + +### Task 7: Add Docker and Kubernetes deployment assets + +**Files:** +- Create: `Dockerfile.api` +- Create: `Dockerfile.worker` +- Create: `Dockerfile.operator-runner` +- Create: `k8s/*.yaml` +- Create: `scripts/*.sh` + +**Steps:** +1. Add image build definitions, k8s manifests, and deploy scripts for MongoDB, API, worker, registry secret, and config. +2. Ensure manifest env vars match the Go/Python runtime settings. +3. Preserve cluster-specific fixes for Mongo startup retry, secret reuse, image tag propagation, `jobs/status` RBAC, and operator-runner `imagePullSecrets`. +4. Re-run manifest rendering and local container build commands. + +### Task 8: Update documentation and verification assets + +**Files:** +- Modify: `README.md` +- Create: `docs/api.md` +- Create: `docs/deployment.md` +- Create: `docs/operations.md` +- Modify: `.env.example` + +**Steps:** +1. Document local development, environment variables, worker behavior, deployment, and acceptance checks. +2. Add Baidu registry and Kubernetes setup instructions without committing secrets. +3. Run doc/code sync review and fix any drift. +4. Run final verification loop: build, type check, tests, security grep, diff review. diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..c9c591d0 --- /dev/null +++ b/go.mod @@ -0,0 +1,61 @@ +module droboticflow + +go 1.24.0 + +require ( + github.com/go-chi/chi/v5 v5.2.3 + go.mongodb.org/mongo-driver v1.17.6 + k8s.io/api v0.34.1 + k8s.io/apimachinery v0.34.1 + k8s.io/client-go v0.34.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..f7aa3606 --- /dev/null +++ b/go.sum @@ -0,0 +1,193 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= +github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/index.html b/index.html index 21dfe690..9742593b 100644 --- a/index.html +++ b/index.html @@ -3,11 +3,10 @@ - My Google AI Studio App + DroboticFlow
- diff --git a/internal/ai/client.go b/internal/ai/client.go new file mode 100644 index 00000000..5e7f2650 --- /dev/null +++ b/internal/ai/client.go @@ -0,0 +1,712 @@ +package ai + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" + + "droboticflow/internal/domain" +) + +const ( + defaultProvider = "dashscope-openai-compatible" + defaultRequestTimeout = 240 * time.Second + defaultMaxAttempts = 2 +) + +var ErrUnavailable = errors.New("AI service is not configured") + +type Status struct { + Enabled bool `json:"enabled"` + Provider string `json:"provider"` + BaseURL string `json:"baseURL,omitempty"` + Model string `json:"model,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type GeneratedWorkflow struct { + Name string `json:"name"` + Description string `json:"description"` + Nodes []domain.WorkflowNode `json:"nodes"` + Edges []domain.WorkflowEdge `json:"edges"` +} + +type GeneratedOperatorPort struct { + Name string `json:"name"` + Type string `json:"type"` + Transmission string `json:"transmission"` + Description string `json:"description,omitempty"` +} + +type GeneratedOperator struct { + Name string `json:"name"` + Category string `json:"category"` + Description string `json:"description"` + Inputs []GeneratedOperatorPort `json:"inputs"` + Outputs []GeneratedOperatorPort `json:"outputs"` + Hyperparams []domain.HyperParam `json:"hyperparams"` + SkillDefinition string `json:"skillDefinition"` +} + +type Service interface { + Status() Status + GenerateWorkflow(context.Context, string, []domain.OperatorRecord) (*GeneratedWorkflow, error) + GenerateOperator(context.Context, string) (*GeneratedOperator, error) +} + +type disabledService struct { + status Status +} + +func NewDisabledService(baseURL string, model string, reason string) Service { + if strings.TrimSpace(reason) == "" { + reason = "AI 服务未配置" + } + return &disabledService{ + status: Status{ + Enabled: false, + Provider: defaultProvider, + BaseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"), + Model: strings.TrimSpace(model), + Reason: reason, + }, + } +} + +func (service *disabledService) Status() Status { + return service.status +} + +func (service *disabledService) GenerateWorkflow(context.Context, string, []domain.OperatorRecord) (*GeneratedWorkflow, error) { + return nil, ErrUnavailable +} + +func (service *disabledService) GenerateOperator(context.Context, string) (*GeneratedOperator, error) { + return nil, ErrUnavailable +} + +type client struct { + httpClient *http.Client + baseURL string + apiKey string + model string + requestTimeout time.Duration + maxAttempts int +} + +func NewService(baseURL string, apiKey string, model string) Service { + normalizedBaseURL := strings.TrimRight(strings.TrimSpace(baseURL), "/") + normalizedKey := strings.TrimSpace(apiKey) + normalizedModel := strings.TrimSpace(model) + if normalizedModel == "" { + normalizedModel = "glm-5" + } + if normalizedBaseURL == "" { + normalizedBaseURL = "https://coding.dashscope.aliyuncs.com/v1" + } + if normalizedKey == "" { + return NewDisabledService(normalizedBaseURL, normalizedModel, "服务端未配置 AI_API_KEY") + } + return &client{ + httpClient: &http.Client{}, + baseURL: normalizedBaseURL, + apiKey: normalizedKey, + model: normalizedModel, + requestTimeout: defaultRequestTimeout, + maxAttempts: defaultMaxAttempts, + } +} + +func (client *client) Status() Status { + return Status{ + Enabled: true, + Provider: defaultProvider, + BaseURL: client.baseURL, + Model: client.model, + } +} + +type chatMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type chatCompletionRequest struct { + Model string `json:"model"` + Messages []chatMessage `json:"messages"` + Temperature float64 `json:"temperature,omitempty"` +} + +type chatCompletionResponse struct { + Choices []struct { + Message struct { + Content any `json:"content"` + } `json:"message"` + } `json:"choices"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +type workflowPlan struct { + Name string `json:"name"` + Description string `json:"description"` + Nodes []workflowPlanNode `json:"nodes"` + Edges []workflowPlanEdge `json:"edges"` +} + +type workflowPlanNode struct { + ID string `json:"id"` + Label string `json:"label,omitempty"` + OperatorID string `json:"operatorId"` + Position domain.Position `json:"position"` + HyperparamOverrides []domain.HyperParam `json:"hyperparamOverrides,omitempty"` + Bucket string `json:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty"` + Pattern string `json:"pattern,omitempty"` + CPU string `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` +} + +type workflowPlanEdge struct { + ID string `json:"id"` + Source string `json:"source"` + Target string `json:"target"` + SourcePort string `json:"sourcePort,omitempty"` + TargetPort string `json:"targetPort,omitempty"` +} + +func (client *client) GenerateWorkflow(ctx context.Context, description string, operators []domain.OperatorRecord) (*GeneratedWorkflow, error) { + if strings.TrimSpace(description) == "" { + return nil, fmt.Errorf("description is required") + } + if len(operators) == 0 { + return nil, fmt.Errorf("no operators available") + } + + type operatorContext struct { + ID string `json:"id"` + Name string `json:"name"` + Category string `json:"category"` + Description string `json:"description"` + Inputs []domain.PortDefinition `json:"inputs"` + Outputs []domain.PortDefinition `json:"outputs"` + Image string `json:"image,omitempty"` + SkillDefinition string `json:"skillDefinition,omitempty"` + Hyperparams []domain.HyperParam `json:"hyperparams,omitempty"` + } + + available := make([]operatorContext, 0, len(operators)) + for _, operator := range operators { + available = append(available, operatorContext{ + ID: operator.ID, + Name: operator.Name, + Category: operator.Category, + Description: operator.Description, + Inputs: operator.Inputs, + Outputs: operator.Outputs, + Image: operator.Image, + SkillDefinition: operator.SkillDefinition, + Hyperparams: operator.Hyperparams, + }) + } + + contextJSON, err := json.MarshalIndent(available, "", " ") + if err != nil { + return nil, err + } + + systemPrompt := strings.TrimSpace(` +You are an expert workflow architect for DroboticFlow. +Return a single JSON object only. Do not wrap it in markdown. + +Your task is to turn the user's requirement into a workflow PLAN using only the provided operators. + +Rules: +1. Use only operatorId values from the provided operator catalog. +2. Nodes must be valid for DroboticFlow execution and connected in a runnable order. +3. For each edge, sourcePort and targetPort must match the real port names from the chosen operators. +4. Prefer batch workflows unless the user explicitly asks for streaming behavior. +5. Keep names short and production-oriented. +6. If a node needs operator-specific configuration, put it in hyperparamOverrides using real hyperparameter keys. +7. Output schema: +{ + "name": "workflow short name", + "description": "short summary", + "nodes": [ + { + "id": "node-source", + "label": "optional custom label", + "operatorId": "operator-id", + "position": {"x": 120, "y": 80}, + "hyperparamOverrides": [{"key":"ENV_KEY","value":"ENV_VALUE","description":"optional"}], + "bucket": "optional storage bucket", + "prefix": "optional storage prefix", + "pattern": "optional filename pattern", + "cpu": "optional cpu limit", + "memory": "optional memory limit" + } + ], + "edges": [ + { + "id": "edge-1", + "source": "node-source", + "target": "node-next", + "sourcePort": "output port name", + "targetPort": "input port name" + } + ] +}`) + + userPrompt := fmt.Sprintf("User requirement:\n%s\n\nAvailable operators:\n%s", strings.TrimSpace(description), string(contextJSON)) + + var plan workflowPlan + if err := client.generateJSON(ctx, systemPrompt, userPrompt, &plan); err != nil { + return nil, err + } + if len(plan.Nodes) == 0 { + return nil, fmt.Errorf("AI did not return any workflow nodes") + } + + operatorByID := make(map[string]domain.OperatorRecord, len(operators)) + nodes := make([]domain.WorkflowNode, 0, len(plan.Nodes)) + for _, operator := range operators { + operatorByID[operator.ID] = operator + } + + nodeByID := make(map[string]domain.WorkflowNode, len(plan.Nodes)) + for _, item := range plan.Nodes { + operator, ok := operatorByID[item.OperatorID] + if !ok { + return nil, fmt.Errorf("AI selected unknown operator %q", item.OperatorID) + } + if strings.TrimSpace(item.ID) == "" { + return nil, fmt.Errorf("AI returned a workflow node without id") + } + node := domain.WorkflowNode{ + ID: strings.TrimSpace(item.ID), + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: chooseValue(item.Label, operator.Name), + Category: operator.Category, + Description: operator.Description, + Inputs: clonePorts(operator.Inputs), + Outputs: clonePorts(operator.Outputs), + Image: operator.Image, + SkillDefinition: operator.SkillDefinition, + Hyperparams: mergeHyperparams(operator.Hyperparams, item.HyperparamOverrides), + Bucket: strings.TrimSpace(item.Bucket), + Prefix: strings.TrimSpace(item.Prefix), + Pattern: strings.TrimSpace(item.Pattern), + CPU: strings.TrimSpace(item.CPU), + Memory: strings.TrimSpace(item.Memory), + }, + Position: item.Position, + } + nodes = append(nodes, node) + nodeByID[node.ID] = node + } + + edges := make([]domain.WorkflowEdge, 0, len(plan.Edges)) + for _, item := range plan.Edges { + if strings.TrimSpace(item.Source) == "" || strings.TrimSpace(item.Target) == "" { + return nil, fmt.Errorf("AI returned an edge without source/target") + } + sourceNode, sourceOK := nodeByID[item.Source] + targetNode, targetOK := nodeByID[item.Target] + if !sourceOK || !targetOK { + return nil, fmt.Errorf("AI returned an edge with unknown nodes: %s -> %s", item.Source, item.Target) + } + sourceHandle, err := resolvePortHandle(sourceNode.Data.Outputs, item.SourcePort) + if err != nil { + return nil, fmt.Errorf("resolve source port for edge %s: %w", chooseValue(item.ID, item.Source+"-"+item.Target), err) + } + targetHandle, err := resolvePortHandle(targetNode.Data.Inputs, item.TargetPort) + if err != nil { + return nil, fmt.Errorf("resolve target port for edge %s: %w", chooseValue(item.ID, item.Source+"-"+item.Target), err) + } + edges = append(edges, domain.WorkflowEdge{ + ID: chooseValue(strings.TrimSpace(item.ID), "edge-"+item.Source+"-"+item.Target), + Source: item.Source, + Target: item.Target, + SourceHandle: sourceHandle, + TargetHandle: targetHandle, + }) + } + + return &GeneratedWorkflow{ + Name: chooseValue(strings.TrimSpace(plan.Name), "AI 生成工作流"), + Description: chooseValue(strings.TrimSpace(plan.Description), strings.TrimSpace(description)), + Nodes: nodes, + Edges: edges, + }, nil +} + +func (client *client) GenerateOperator(ctx context.Context, description string) (*GeneratedOperator, error) { + if strings.TrimSpace(description) == "" { + return nil, fmt.Errorf("description is required") + } + + systemPrompt := strings.TrimSpace(` +You are an expert DroboticFlow operator designer. +Return one JSON object only. Do not wrap it in markdown. + +Allowed categories: IO, CV, AI, MATH, HUMAN, FLOW +Allowed port types: json, image, video, tensor, file +Allowed transmission values: bus, oss, shm, stream + +Return schema: +{ + "name": "short operator name", + "category": "AI", + "description": "1-2 sentence description", + "inputs": [{"name":"input","type":"json","transmission":"bus","description":"optional"}], + "outputs": [{"name":"result","type":"json","transmission":"bus","description":"optional"}], + "hyperparams": [{"key":"ENV_KEY","value":"default","description":"what it controls"}], + "skillDefinition": "detailed markdown instructions for future workflow generation and operator usage" +}`) + + var generated GeneratedOperator + if err := client.generateJSON(ctx, systemPrompt, strings.TrimSpace(description), &generated); err != nil { + return nil, err + } + if strings.TrimSpace(generated.Name) == "" { + return nil, fmt.Errorf("AI did not return operator name") + } + if !validCategory(generated.Category) { + return nil, fmt.Errorf("AI returned unsupported operator category %q", generated.Category) + } + if err := validateGeneratedPorts(generated.Inputs); err != nil { + return nil, fmt.Errorf("invalid generated operator inputs: %w", err) + } + if err := validateGeneratedPorts(generated.Outputs); err != nil { + return nil, fmt.Errorf("invalid generated operator outputs: %w", err) + } + if strings.TrimSpace(generated.Description) == "" { + generated.Description = strings.TrimSpace(description) + } + if strings.TrimSpace(generated.SkillDefinition) == "" { + generated.SkillDefinition = generated.Description + } + return &generated, nil +} + +func (client *client) generateJSON(ctx context.Context, systemPrompt string, userPrompt string, target any) error { + requestBody, err := json.Marshal(chatCompletionRequest{ + Model: client.model, + Messages: []chatMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }, + Temperature: 0.1, + }) + if err != nil { + return err + } + + payload, err := client.chatCompletion(ctx, requestBody) + if err != nil { + return err + } + + var completion chatCompletionResponse + if err := json.Unmarshal(payload, &completion); err != nil { + return fmt.Errorf("decode AI response: %w", err) + } + if completion.Error != nil && strings.TrimSpace(completion.Error.Message) != "" { + return errors.New(completion.Error.Message) + } + if len(completion.Choices) == 0 { + return fmt.Errorf("AI response does not contain choices") + } + + content := normalizeMessageContent(completion.Choices[0].Message.Content) + if strings.TrimSpace(content) == "" { + return fmt.Errorf("AI response content is empty") + } + parsedJSON, err := extractJSONObject(content) + if err != nil { + return err + } + if err := json.Unmarshal([]byte(parsedJSON), target); err != nil { + return fmt.Errorf("decode AI JSON payload: %w", err) + } + return nil +} + +type httpStatusError struct { + StatusCode int + Body string +} + +func (err *httpStatusError) Error() string { + body := strings.TrimSpace(err.Body) + if body == "" { + return fmt.Sprintf("AI request failed with status %d", err.StatusCode) + } + return fmt.Sprintf("AI request failed with status %d: %s", err.StatusCode, body) +} + +func (client *client) chatCompletion(ctx context.Context, requestBody []byte) ([]byte, error) { + attempts := client.maxAttempts + if attempts < 1 { + attempts = 1 + } + + var lastErr error + for attempt := 1; attempt <= attempts; attempt++ { + payload, err := client.chatCompletionOnce(ctx, requestBody) + if err == nil { + return payload, nil + } + lastErr = err + if !isRetriableAIError(err) || attempt == attempts || ctx.Err() != nil { + break + } + if err := sleepWithContext(ctx, time.Duration(attempt)*2*time.Second); err != nil { + lastErr = err + break + } + } + return nil, normalizeAIError(lastErr) +} + +func (client *client) chatCompletionOnce(ctx context.Context, requestBody []byte) ([]byte, error) { + requestCtx := ctx + cancel := func() {} + if _, hasDeadline := ctx.Deadline(); !hasDeadline && client.requestTimeout > 0 { + requestCtx, cancel = context.WithTimeout(ctx, client.requestTimeout) + } + defer cancel() + + request, err := http.NewRequestWithContext(requestCtx, http.MethodPost, client.baseURL+"/chat/completions", bytes.NewReader(requestBody)) + if err != nil { + return nil, err + } + request.Header.Set("Authorization", "Bearer "+client.apiKey) + request.Header.Set("Content-Type", "application/json") + + response, err := client.httpClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + payload, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + if response.StatusCode >= 400 { + return nil, &httpStatusError{ + StatusCode: response.StatusCode, + Body: string(payload), + } + } + return payload, nil +} + +func isRetriableAIError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err) { + return true + } + var statusErr *httpStatusError + if errors.As(err, &statusErr) { + return statusErr.StatusCode == http.StatusTooManyRequests || + statusErr.StatusCode == http.StatusRequestTimeout || + statusErr.StatusCode >= http.StatusInternalServerError + } + return false +} + +func isTimeoutError(err error) bool { + if err == nil { + return false + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } + return strings.Contains(strings.ToLower(err.Error()), "client.timeout") +} + +func normalizeAIError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err) { + return fmt.Errorf("AI 生成超时,请稍后重试,或缩短描述范围后再试") + } + var statusErr *httpStatusError + if errors.As(err, &statusErr) { + switch { + case statusErr.StatusCode == http.StatusTooManyRequests: + return fmt.Errorf("AI 服务当前限流,请稍后重试") + case statusErr.StatusCode == http.StatusRequestTimeout: + return fmt.Errorf("AI 服务响应超时,请稍后重试") + case statusErr.StatusCode >= http.StatusInternalServerError: + return fmt.Errorf("AI 服务暂时不可用,请稍后重试") + } + } + return err +} + +func sleepWithContext(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func normalizeMessageContent(content any) string { + switch value := content.(type) { + case string: + return value + case []any: + parts := make([]string, 0, len(value)) + for _, item := range value { + typed, ok := item.(map[string]any) + if !ok { + continue + } + if text, ok := typed["text"].(string); ok { + parts = append(parts, text) + } + } + return strings.Join(parts, "\n") + default: + return "" + } +} + +func extractJSONObject(raw string) (string, error) { + trimmed := strings.TrimSpace(raw) + trimmed = strings.TrimPrefix(trimmed, "```json") + trimmed = strings.TrimPrefix(trimmed, "```") + trimmed = strings.TrimSuffix(trimmed, "```") + trimmed = strings.TrimSpace(trimmed) + start := strings.Index(trimmed, "{") + end := strings.LastIndex(trimmed, "}") + if start < 0 || end < start { + return "", fmt.Errorf("AI response is not a JSON object") + } + return trimmed[start : end+1], nil +} + +func clonePorts(ports []domain.PortDefinition) []domain.PortDefinition { + items := make([]domain.PortDefinition, len(ports)) + copy(items, ports) + return items +} + +func mergeHyperparams(base []domain.HyperParam, overrides []domain.HyperParam) []domain.HyperParam { + if len(base) == 0 && len(overrides) == 0 { + return []domain.HyperParam{} + } + merged := make([]domain.HyperParam, len(base)) + copy(merged, base) + indexByKey := make(map[string]int, len(merged)) + for index, item := range merged { + indexByKey[strings.ToUpper(strings.TrimSpace(item.Key))] = index + } + for _, item := range overrides { + key := strings.ToUpper(strings.TrimSpace(item.Key)) + if key == "" { + continue + } + item.Key = strings.TrimSpace(item.Key) + if index, ok := indexByKey[key]; ok { + if strings.TrimSpace(item.Value) != "" { + merged[index].Value = item.Value + } + if strings.TrimSpace(item.Description) != "" { + merged[index].Description = item.Description + } + if strings.TrimSpace(item.Type) != "" { + merged[index].Type = item.Type + } + continue + } + merged = append(merged, item) + indexByKey[key] = len(merged) - 1 + } + return merged +} + +func resolvePortHandle(ports []domain.PortDefinition, name string) (string, error) { + normalizedName := strings.TrimSpace(strings.ToLower(name)) + if normalizedName == "" { + if len(ports) == 1 { + return ports[0].ID, nil + } + return "", fmt.Errorf("port name is required when multiple ports are available") + } + for _, port := range ports { + if strings.ToLower(strings.TrimSpace(port.Name)) == normalizedName || strings.ToLower(strings.TrimSpace(port.ID)) == normalizedName { + return port.ID, nil + } + } + return "", fmt.Errorf("port %q not found", name) +} + +func chooseValue(value string, fallback string) string { + if strings.TrimSpace(value) == "" { + return fallback + } + return strings.TrimSpace(value) +} + +func validCategory(value string) bool { + switch strings.ToUpper(strings.TrimSpace(value)) { + case "IO", "CV", "AI", "MATH", "HUMAN", "FLOW": + return true + default: + return false + } +} + +func validPortType(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "json", "image", "video", "tensor", "file": + return true + default: + return false + } +} + +func validTransmission(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "bus", "oss", "shm", "stream": + return true + default: + return false + } +} + +func validateGeneratedPorts(ports []GeneratedOperatorPort) error { + for _, port := range ports { + if strings.TrimSpace(port.Name) == "" { + return fmt.Errorf("port name is required") + } + if !validPortType(port.Type) { + return fmt.Errorf("unsupported port type %q", port.Type) + } + if !validTransmission(port.Transmission) { + return fmt.Errorf("unsupported transmission %q", port.Transmission) + } + } + return nil +} diff --git a/internal/ai/client_test.go b/internal/ai/client_test.go new file mode 100644 index 00000000..30934a22 --- /dev/null +++ b/internal/ai/client_test.go @@ -0,0 +1,67 @@ +package ai + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func TestGenerateOperatorRetriesTimeoutThenSucceeds(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + count := attempts.Add(1) + if count == 1 { + time.Sleep(80 * time.Millisecond) + return + } + writer.Header().Set("Content-Type", "application/json") + fmt.Fprint(writer, `{ + "choices": [ + { + "message": { + "content": "{\"name\":\"Timeout Retry Operator\",\"category\":\"AI\",\"description\":\"retry ok\",\"inputs\":[{\"name\":\"input\",\"type\":\"json\",\"transmission\":\"bus\"}],\"outputs\":[{\"name\":\"result\",\"type\":\"json\",\"transmission\":\"bus\"}],\"hyperparams\":[],\"skillDefinition\":\"retry ok\"}" + } + } + ] + }`) + })) + defer server.Close() + + service := NewService(server.URL, "test-key", "glm-5").(*client) + service.requestTimeout = 40 * time.Millisecond + service.maxAttempts = 2 + + item, err := service.GenerateOperator(context.Background(), "generate operator") + if err != nil { + t.Fatalf("expected retry to succeed, got error: %v", err) + } + if item.Name != "Timeout Retry Operator" { + t.Fatalf("unexpected operator name: %s", item.Name) + } + if attempts.Load() != 2 { + t.Fatalf("expected 2 attempts, got %d", attempts.Load()) + } +} + +func TestGenerateOperatorTimeoutReturnsFriendlyError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + time.Sleep(80 * time.Millisecond) + })) + defer server.Close() + + service := NewService(server.URL, "test-key", "glm-5").(*client) + service.requestTimeout = 30 * time.Millisecond + service.maxAttempts = 1 + + _, err := service.GenerateOperator(context.Background(), "generate operator") + if err == nil { + t.Fatalf("expected timeout error") + } + if err.Error() != "AI 生成超时,请稍后重试,或缩短描述范围后再试" { + t.Fatalf("unexpected timeout message: %v", err) + } +} diff --git a/internal/api/server.go b/internal/api/server.go new file mode 100644 index 00000000..c4021319 --- /dev/null +++ b/internal/api/server.go @@ -0,0 +1,1274 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/mail" + "strings" + "time" + + "droboticflow/internal/ai" + "droboticflow/internal/domain" + "droboticflow/internal/store" + + "github.com/go-chi/chi/v5" +) + +type validationIssue struct { + Field string `json:"field"` + Message string `json:"message"` +} + +type server struct { + store store.Store + ai ai.Service +} + +func maskedSecretPreview(value string) string { + compact := strings.Join(strings.Fields(strings.TrimSpace(value)), "") + if compact == "" { + compact = strings.TrimSpace(value) + } + if compact == "" { + return "" + } + + runes := []rune(compact) + switch length := len(runes); { + case length <= 2: + return strings.Repeat("*", length) + case length <= 6: + return string(runes[:1]) + "****" + string(runes[length-1:]) + case length <= 10: + return string(runes[:2]) + "****" + string(runes[length-2:]) + default: + return string(runes[:4]) + "****" + string(runes[length-4:]) + } +} + +func NewHandler(dataStore store.Store, aiService ai.Service) http.Handler { + if aiService == nil { + aiService = ai.NewDisabledService("", "", "AI 服务未配置") + } + instance := &server{store: dataStore, ai: aiService} + router := chi.NewRouter() + + router.Get("/api/health", instance.handleHealth) + router.Get("/api/ai/status", instance.handleAIStatus) + router.Post("/api/ai/operators/generate", instance.handleAIOperatorGenerate) + router.Post("/api/ai/workflows/generate", instance.handleAIWorkflowGenerate) + router.Get("/api/dashboard", instance.handleDashboard) + + router.Get("/api/projects", instance.handleProjectsList) + router.Post("/api/projects", instance.handleProjectsCreate) + router.Get("/api/projects/{id}", instance.handleProjectsGet) + router.Put("/api/projects/{id}", instance.handleProjectsUpdate) + router.Delete("/api/projects/{id}", instance.handleProjectsDelete) + router.Get("/api/projects/{id}/workflows", instance.handleProjectWorkflowsList) + router.Post("/api/projects/{id}/workflows", instance.handleProjectWorkflowsCreate) + + router.Get("/api/workflows/{id}", instance.handleWorkflowsGet) + router.Put("/api/workflows/{id}", instance.handleWorkflowsUpdate) + router.Delete("/api/workflows/{id}", instance.handleWorkflowsDelete) + router.Post("/api/workflows/{id}/run", instance.handleWorkflowRun) + + router.Get("/api/templates", instance.handleTemplatesList) + router.Post("/api/templates", instance.handleTemplatesCreate) + router.Get("/api/templates/{id}", instance.handleTemplatesGet) + router.Put("/api/templates/{id}", instance.handleTemplatesUpdate) + router.Delete("/api/templates/{id}", instance.handleTemplatesDelete) + + router.Get("/api/operators", instance.handleOperatorsList) + router.Post("/api/operators", instance.handleOperatorsCreate) + router.Get("/api/operators/{id}", instance.handleOperatorsGet) + router.Put("/api/operators/{id}", instance.handleOperatorsUpdate) + router.Delete("/api/operators/{id}", instance.handleOperatorsDelete) + + router.Get("/api/runs", instance.handleRunsList) + router.Get("/api/runs/{id}", instance.handleRunsGet) + router.Get("/api/runs/{id}/logs", instance.handleRunsLogs) + router.Get("/api/runs/{id}/resources", instance.handleRunsResources) + router.Get("/api/runs/{id}/artifacts", instance.handleRunsArtifacts) + + router.Get("/api/audit-tasks", instance.handleAuditTasksList) + router.Post("/api/audit-tasks/{id}/approve", instance.handleAuditTaskApprove) + router.Post("/api/audit-tasks/{id}/reject", instance.handleAuditTaskReject) + + router.Get("/api/users", instance.handleUsersList) + router.Post("/api/users", instance.handleUsersCreate) + router.Put("/api/users/{id}", instance.handleUsersUpdate) + router.Delete("/api/users/{id}", instance.handleUsersDelete) + + router.Get("/api/settings", instance.handleSettingsGet) + router.Put("/api/settings", instance.handleSettingsUpdate) + + router.Get("/api/storage/buckets", instance.handleStorageBuckets) + router.Get("/api/storage/objects", instance.handleStorageObjects) + + return router +} + +func (server *server) handleHealth(writer http.ResponseWriter, _ *http.Request) { + renderJSON(writer, http.StatusOK, map[string]string{"status": "ok", "service": "droboticflow-api"}) +} + +func (server *server) handleAIStatus(writer http.ResponseWriter, _ *http.Request) { + renderJSON(writer, http.StatusOK, server.resolveAIService(context.Background()).Status()) +} + +type aiGenerateRequest struct { + Description string `json:"description"` +} + +func (server *server) handleAIOperatorGenerate(writer http.ResponseWriter, request *http.Request) { + var payload aiGenerateRequest + if !decodeJSON(writer, request, &payload) { + return + } + if strings.TrimSpace(payload.Description) == "" { + renderValidationError(writer, validationIssue{Field: "description", Message: "description is required"}) + return + } + item, err := server.resolveAIService(request.Context()).GenerateOperator(request.Context(), payload.Description) + if err != nil { + renderAIError(writer, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleAIWorkflowGenerate(writer http.ResponseWriter, request *http.Request) { + var payload aiGenerateRequest + if !decodeJSON(writer, request, &payload) { + return + } + if strings.TrimSpace(payload.Description) == "" { + renderValidationError(writer, validationIssue{Field: "description", Message: "description is required"}) + return + } + operators, err := server.store.ListOperators(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if len(operators) == 0 { + renderValidationError(writer, validationIssue{Field: "operators", Message: "at least one operator is required to generate a workflow"}) + return + } + item, err := server.resolveAIService(request.Context()).GenerateWorkflow(request.Context(), payload.Description, operators) + if err != nil { + renderAIError(writer, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleDashboard(writer http.ResponseWriter, request *http.Request) { + summary, err := server.store.GetDashboardSummary(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, summary) +} + +func (server *server) populateProjectWorkflowCounts(ctx context.Context, items []domain.ProjectRecord) ([]domain.ProjectRecord, error) { + if len(items) == 0 { + return items, nil + } + + workflows, err := server.store.ListAllWorkflows(ctx) + if err != nil { + return nil, err + } + + counts := make(map[string]int, len(items)) + for _, workflow := range workflows { + if workflow.ProjectID == "" { + continue + } + counts[workflow.ProjectID]++ + } + + enriched := make([]domain.ProjectRecord, len(items)) + for index, item := range items { + item.WorkflowCount = counts[item.ID] + enriched[index] = item + } + return enriched, nil +} + +func (server *server) populateSingleProjectWorkflowCount(ctx context.Context, item *domain.ProjectRecord) (*domain.ProjectRecord, error) { + if item == nil { + return nil, nil + } + + workflows, err := server.store.ListWorkflowsByProject(ctx, item.ID) + if err != nil { + return nil, err + } + + enriched := *item + enriched.WorkflowCount = len(workflows) + return &enriched, nil +} + +func (server *server) handleProjectsList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListProjects(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + items, err = server.populateProjectWorkflowCounts(request.Context(), items) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleProjectsCreate(writer http.ResponseWriter, request *http.Request) { + var payload domain.CreateProjectInput + if !decodeJSON(writer, request, &payload) { + return + } + if strings.TrimSpace(payload.Name) == "" { + renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"}) + return + } + project, err := server.store.CreateProject(request.Context(), payload) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, project) +} + +func (server *server) handleProjectsGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"}) + return + } + item, err = server.populateSingleProjectWorkflowCount(request.Context(), item) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +type projectPatch struct { + Name *string `json:"name"` + Description *string `json:"description"` + Status *string `json:"status"` +} + +func (server *server) handleProjectsUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"}) + return + } + + var payload projectPatch + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Name != nil { + item.Name = strings.TrimSpace(*payload.Name) + } + if payload.Description != nil { + item.Description = *payload.Description + } + if payload.Status != nil { + item.Status = *payload.Status + } + item.UpdatedAt = nowISO() + if err := server.store.UpsertProject(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + item, err = server.populateSingleProjectWorkflowCount(request.Context(), item) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleProjectsDelete(writer http.ResponseWriter, request *http.Request) { + deleted, err := server.store.DeleteProject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + writer.WriteHeader(http.StatusNotFound) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (server *server) handleProjectWorkflowsList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListWorkflowsByProject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func validWorkflowType(value domain.WorkflowType) bool { + return value == domain.WorkflowTypeBatch || value == domain.WorkflowTypeStreaming +} + +func validateWorkflowExecutionConfig(maxConcurrentRuns int, nodeTimeoutSeconds int, runTimeoutSeconds int, maxRetryAttempts int, retryBackoffSeconds int) []validationIssue { + issues := make([]validationIssue, 0, 5) + if maxConcurrentRuns < 0 { + issues = append(issues, validationIssue{Field: "maxConcurrentRuns", Message: "maxConcurrentRuns must be zero or greater"}) + } + if nodeTimeoutSeconds < 0 { + issues = append(issues, validationIssue{Field: "nodeTimeoutSeconds", Message: "nodeTimeoutSeconds must be zero or greater"}) + } + if runTimeoutSeconds < 0 { + issues = append(issues, validationIssue{Field: "runTimeoutSeconds", Message: "runTimeoutSeconds must be zero or greater"}) + } + if maxRetryAttempts < 0 { + issues = append(issues, validationIssue{Field: "maxRetryAttempts", Message: "maxRetryAttempts must be zero or greater"}) + } + if retryBackoffSeconds < 0 { + issues = append(issues, validationIssue{Field: "retryBackoffSeconds", Message: "retryBackoffSeconds must be zero or greater"}) + } + return issues +} + +func (server *server) handleProjectWorkflowsCreate(writer http.ResponseWriter, request *http.Request) { + var payload domain.CreateWorkflowInput + if !decodeJSON(writer, request, &payload) { + return + } + payload.ProjectID = chi.URLParam(request, "id") + if strings.TrimSpace(payload.Name) == "" || !validWorkflowType(payload.Type) { + renderValidationError(writer, + validationIssue{Field: "name", Message: "name is required"}, + validationIssue{Field: "type", Message: "type must be batch or streaming"}, + ) + return + } + if issues := validateWorkflowExecutionConfig(payload.MaxConcurrentRuns, payload.NodeTimeoutSeconds, payload.RunTimeoutSeconds, payload.MaxRetryAttempts, payload.RetryBackoffSeconds); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item, err := server.store.CreateWorkflow(request.Context(), payload) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, item) +} + +func (server *server) handleWorkflowsGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"}) + return + } + renderJSON(writer, http.StatusOK, item) +} + +type workflowPatch struct { + Name *string `json:"name"` + Description *string `json:"description"` + Type *domain.WorkflowType `json:"type"` + MaxConcurrentRuns *int `json:"maxConcurrentRuns"` + NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"` + RunTimeoutSeconds *int `json:"runTimeoutSeconds"` + MaxRetryAttempts *int `json:"maxRetryAttempts"` + RetryBackoffSeconds *int `json:"retryBackoffSeconds"` + Schedule *domain.WorkflowSchedule `json:"schedule"` + Nodes *[]domain.WorkflowNode `json:"nodes"` + Edges *[]domain.WorkflowEdge `json:"edges"` + Status *domain.WorkflowStatus `json:"status"` +} + +func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"}) + return + } + + var payload workflowPatch + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Name != nil { + item.Name = *payload.Name + } + if payload.Description != nil { + item.Description = *payload.Description + } + if payload.Type != nil { + item.Type = *payload.Type + } + if payload.MaxConcurrentRuns != nil { + if issues := validateWorkflowExecutionConfig(*payload.MaxConcurrentRuns, 0, 0, 0, 0); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.MaxConcurrentRuns = *payload.MaxConcurrentRuns + } + if payload.NodeTimeoutSeconds != nil { + if issues := validateWorkflowExecutionConfig(0, *payload.NodeTimeoutSeconds, 0, 0, 0); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.NodeTimeoutSeconds = *payload.NodeTimeoutSeconds + } + if payload.RunTimeoutSeconds != nil { + if issues := validateWorkflowExecutionConfig(0, 0, *payload.RunTimeoutSeconds, 0, 0); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.RunTimeoutSeconds = *payload.RunTimeoutSeconds + } + if payload.MaxRetryAttempts != nil { + if issues := validateWorkflowExecutionConfig(0, 0, 0, *payload.MaxRetryAttempts, 0); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.MaxRetryAttempts = *payload.MaxRetryAttempts + } + if payload.RetryBackoffSeconds != nil { + if issues := validateWorkflowExecutionConfig(0, 0, 0, 0, *payload.RetryBackoffSeconds); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.RetryBackoffSeconds = *payload.RetryBackoffSeconds + } + if payload.Schedule != nil { + item.Schedule = payload.Schedule + } + if payload.Nodes != nil { + item.Nodes = *payload.Nodes + } + if payload.Edges != nil { + item.Edges = *payload.Edges + } + if payload.Status != nil { + item.Status = *payload.Status + } + item.UpdatedAt = nowISO() + if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleWorkflowsDelete(writer http.ResponseWriter, request *http.Request) { + deleted, err := server.store.DeleteWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + writer.WriteHeader(http.StatusNotFound) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +type runRequest struct { + Trigger domain.RunTrigger `json:"trigger"` +} + +func validRunTrigger(value domain.RunTrigger) bool { + return value == domain.RunTriggerManual || value == domain.RunTriggerScheduled || value == domain.RunTriggerRetry +} + +func (server *server) handleWorkflowRun(writer http.ResponseWriter, request *http.Request) { + payload := runRequest{Trigger: domain.RunTriggerManual} + if request.ContentLength != 0 { + if !decodeJSON(writer, request, &payload) { + return + } + } + if !validRunTrigger(payload.Trigger) { + renderValidationError(writer, validationIssue{Field: "trigger", Message: "trigger must be manual, scheduled, or retry"}) + return + } + run, err := server.store.CreateRun(request.Context(), chi.URLParam(request, "id"), payload.Trigger) + if err != nil { + if strings.Contains(err.Error(), "not found") { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()}) + return + } + if errors.Is(err, store.ErrWorkflowAlreadyCompleted) { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流已成功执行,不支持再次手动运行"}) + return + } + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusAccepted, run) +} + +func (server *server) handleTemplatesList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListTemplates(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleTemplatesCreate(writer http.ResponseWriter, request *http.Request) { + var payload domain.CreateTemplateInput + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Type == "" { + payload.Type = domain.WorkflowTypeBatch + } + issues := make([]validationIssue, 0) + if strings.TrimSpace(payload.Name) == "" { + issues = append(issues, validationIssue{Field: "name", Message: "name is required"}) + } + if !validWorkflowType(payload.Type) { + issues = append(issues, validationIssue{Field: "type", Message: "type must be batch or streaming"}) + } + if configIssues := validateWorkflowExecutionConfig(payload.MaxConcurrentRuns, payload.NodeTimeoutSeconds, payload.RunTimeoutSeconds, payload.MaxRetryAttempts, payload.RetryBackoffSeconds); len(configIssues) > 0 { + issues = append(issues, configIssues...) + } + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item, err := server.store.CreateTemplate(request.Context(), payload) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, item) +} + +func (server *server) handleTemplatesGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetTemplate(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Template not found"}) + return + } + renderJSON(writer, http.StatusOK, item) +} + +type templatePatch struct { + Name *string `json:"name"` + Description *string `json:"description"` + Type *domain.WorkflowType `json:"type"` + MaxConcurrentRuns *int `json:"maxConcurrentRuns"` + NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"` + RunTimeoutSeconds *int `json:"runTimeoutSeconds"` + MaxRetryAttempts *int `json:"maxRetryAttempts"` + RetryBackoffSeconds *int `json:"retryBackoffSeconds"` + Schedule *domain.WorkflowSchedule `json:"schedule"` + Nodes *[]domain.WorkflowNode `json:"nodes"` + Edges *[]domain.WorkflowEdge `json:"edges"` +} + +func (server *server) handleTemplatesUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetTemplate(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Template not found"}) + return + } + var payload templatePatch + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Name != nil { + item.Name = *payload.Name + } + if payload.Description != nil { + item.Description = *payload.Description + } + if payload.Type != nil { + if !validWorkflowType(*payload.Type) { + renderValidationError(writer, validationIssue{Field: "type", Message: "type must be batch or streaming"}) + return + } + item.Type = *payload.Type + } + if payload.MaxConcurrentRuns != nil { + if issues := validateWorkflowExecutionConfig(*payload.MaxConcurrentRuns, 0, 0, 0, 0); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.MaxConcurrentRuns = *payload.MaxConcurrentRuns + } + if payload.NodeTimeoutSeconds != nil { + if issues := validateWorkflowExecutionConfig(0, *payload.NodeTimeoutSeconds, 0, 0, 0); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.NodeTimeoutSeconds = *payload.NodeTimeoutSeconds + } + if payload.RunTimeoutSeconds != nil { + if issues := validateWorkflowExecutionConfig(0, 0, *payload.RunTimeoutSeconds, 0, 0); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.RunTimeoutSeconds = *payload.RunTimeoutSeconds + } + if payload.MaxRetryAttempts != nil { + if issues := validateWorkflowExecutionConfig(0, 0, 0, *payload.MaxRetryAttempts, 0); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.MaxRetryAttempts = *payload.MaxRetryAttempts + } + if payload.RetryBackoffSeconds != nil { + if issues := validateWorkflowExecutionConfig(0, 0, 0, 0, *payload.RetryBackoffSeconds); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.RetryBackoffSeconds = *payload.RetryBackoffSeconds + } + if payload.Schedule != nil { + item.Schedule = payload.Schedule + } + if payload.Nodes != nil { + item.Nodes = *payload.Nodes + } + if payload.Edges != nil { + item.Edges = *payload.Edges + } + item.UpdatedAt = nowISO() + if err := server.store.UpsertTemplate(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleTemplatesDelete(writer http.ResponseWriter, request *http.Request) { + deleted, err := server.store.DeleteTemplate(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + writer.WriteHeader(http.StatusNotFound) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (server *server) handleOperatorsList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListOperators(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleOperatorsCreate(writer http.ResponseWriter, request *http.Request) { + var payload domain.CreateOperatorInput + if !decodeJSON(writer, request, &payload) { + return + } + if strings.TrimSpace(payload.Name) == "" || strings.TrimSpace(payload.Category) == "" || strings.TrimSpace(payload.Description) == "" { + renderValidationError(writer, + validationIssue{Field: "name", Message: "name is required"}, + validationIssue{Field: "category", Message: "category is required"}, + validationIssue{Field: "description", Message: "description is required"}, + ) + return + } + item, err := server.store.CreateOperator(request.Context(), payload) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, item) +} + +func (server *server) handleOperatorsGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetOperator(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Operator not found"}) + return + } + renderJSON(writer, http.StatusOK, item) +} + +type operatorPatch struct { + Name *string `json:"name"` + Type *string `json:"type"` + Category *string `json:"category"` + Description *string `json:"description"` + Inputs *[]domain.PortDefinition `json:"inputs"` + Outputs *[]domain.PortDefinition `json:"outputs"` + ConfigSchema *map[string]any `json:"configSchema"` + Image *string `json:"image"` + SkillDefinition *string `json:"skillDefinition"` + Hyperparams *[]domain.HyperParam `json:"hyperparams"` + Resources *domain.OperatorResources `json:"resources"` +} + +func (server *server) handleOperatorsUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetOperator(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Operator not found"}) + return + } + var payload operatorPatch + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Name != nil { + item.Name = *payload.Name + } + if payload.Type != nil { + item.Type = *payload.Type + } + if payload.Category != nil { + item.Category = *payload.Category + } + if payload.Description != nil { + item.Description = *payload.Description + } + if payload.Inputs != nil { + item.Inputs = *payload.Inputs + } + if payload.Outputs != nil { + item.Outputs = *payload.Outputs + } + if payload.ConfigSchema != nil { + item.ConfigSchema = *payload.ConfigSchema + } + if payload.Image != nil { + item.Image = *payload.Image + } + if payload.SkillDefinition != nil { + item.SkillDefinition = *payload.SkillDefinition + } + if payload.Hyperparams != nil { + item.Hyperparams = *payload.Hyperparams + } + if payload.Resources != nil { + item.Resources = payload.Resources + } + item.UpdatedAt = nowISO() + if err := server.store.UpsertOperator(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleOperatorsDelete(writer http.ResponseWriter, request *http.Request) { + deleted, err := server.store.DeleteOperator(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + writer.WriteHeader(http.StatusNotFound) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (server *server) handleRunsList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListRuns(request.Context(), request.URL.Query().Get("status")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) loadRun(ctx context.Context, id string) (*domain.RunRecord, error) { + item, err := server.store.GetRun(ctx, id) + if err != nil || item == nil { + return item, err + } + return item, nil +} + +func (server *server) handleRunsGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.loadRun(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleRunsLogs(writer http.ResponseWriter, request *http.Request) { + item, err := server.loadRun(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": item.Logs}) +} + +func (server *server) handleRunsResources(writer http.ResponseWriter, request *http.Request) { + item, err := server.loadRun(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": item.ResourceSeries}) +} + +func (server *server) handleRunsArtifacts(writer http.ResponseWriter, request *http.Request) { + item, err := server.loadRun(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": item.Artifacts}) +} + +func (server *server) handleAuditTasksList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListAuditTasks(request.Context(), request.URL.Query().Get("status")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) resolveAuditTask(ctx context.Context, id string, decision domain.AuditStatus) (*domain.AuditTaskRecord, error) { + task, err := server.store.GetAuditTask(ctx, id) + if err != nil || task == nil { + return task, err + } + task.Status = decision + task.UpdatedAt = nowISO() + if err := server.store.UpsertAuditTask(ctx, *task); err != nil { + return nil, err + } + run, err := server.store.GetRun(ctx, task.RunID) + if err != nil { + return nil, err + } + if run != nil { + run.Status = domain.RunStatusQueued + run.UpdatedAt = nowISO() + if err := server.store.UpsertRun(ctx, *run); err != nil { + return nil, err + } + } + return task, nil +} + +func (server *server) handleAuditTaskApprove(writer http.ResponseWriter, request *http.Request) { + task, err := server.resolveAuditTask(request.Context(), chi.URLParam(request, "id"), domain.AuditStatusApproved) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if task == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Audit task not found"}) + return + } + renderJSON(writer, http.StatusOK, task) +} + +func (server *server) handleAuditTaskReject(writer http.ResponseWriter, request *http.Request) { + task, err := server.resolveAuditTask(request.Context(), chi.URLParam(request, "id"), domain.AuditStatusRejected) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if task == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Audit task not found"}) + return + } + renderJSON(writer, http.StatusOK, task) +} + +func (server *server) handleUsersList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListUsers(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func normalizeUserInput(name string, email string) (string, string) { + return strings.TrimSpace(name), strings.TrimSpace(strings.ToLower(email)) +} + +func (server *server) handleUsersCreate(writer http.ResponseWriter, request *http.Request) { + var payload domain.CreateUserInput + if !decodeJSON(writer, request, &payload) { + return + } + + payload.Name, payload.Email = normalizeUserInput(payload.Name, payload.Email) + if payload.Name == "" { + renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"}) + return + } + if _, err := mail.ParseAddress(payload.Email); err != nil { + renderValidationError(writer, validationIssue{Field: "email", Message: "email must be valid"}) + return + } + if !validUserRole(payload.Role) { + renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, auditor, or operator"}) + return + } + + item, err := server.store.CreateUser(request.Context(), payload) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, item) +} + +type userPatch struct { + Name *string `json:"name"` + Email *string `json:"email"` + Role *domain.UserRole `json:"role"` +} + +func validUserRole(value domain.UserRole) bool { + return value == domain.UserRoleAdmin || value == domain.UserRoleAuditor || value == domain.UserRoleOperator +} + +func (server *server) handleUsersUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetUser(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "User not found"}) + return + } + var payload userPatch + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Name != nil { + item.Name = strings.TrimSpace(*payload.Name) + } + if payload.Email != nil { + normalizedEmail := strings.TrimSpace(strings.ToLower(*payload.Email)) + if _, err := mail.ParseAddress(normalizedEmail); err != nil { + renderValidationError(writer, validationIssue{Field: "email", Message: "email must be valid"}) + return + } + item.Email = normalizedEmail + } + if payload.Role != nil { + if !validUserRole(*payload.Role) { + renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, auditor, or operator"}) + return + } + item.Role = *payload.Role + } + item.UpdatedAt = nowISO() + if err := server.store.UpsertUser(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleUsersDelete(writer http.ResponseWriter, request *http.Request) { + deleted, err := server.store.DeleteUser(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + writer.WriteHeader(http.StatusNotFound) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (server *server) handleSettingsGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, sanitizeSettings(item)) +} + +type settingsPatch struct { + K8s *struct { + Enabled *bool `json:"enabled"` + Endpoint *string `json:"endpoint"` + Namespace *string `json:"namespace"` + Kubeconfig *string `json:"kubeconfig"` + } `json:"k8s"` + Worker *struct { + PollIntervalMS *int `json:"pollIntervalMs"` + MaxConcurrentRuns *int `json:"maxConcurrentRuns"` + } `json:"worker"` + AI *struct { + Provider *string `json:"provider"` + BaseURL *string `json:"baseURL"` + Model *string `json:"model"` + APIKey *string `json:"apiKey"` + } `json:"ai"` + Defaults *struct { + CPURequest *string `json:"cpuRequest"` + CPULimit *string `json:"cpuLimit"` + MemoryRequest *string `json:"memoryRequest"` + MemoryLimit *string `json:"memoryLimit"` + } `json:"defaults"` + Timezone *string `json:"timezone"` +} + +func (server *server) handleSettingsUpdate(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + var payload settingsPatch + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Timezone != nil { + timezone := strings.TrimSpace(*payload.Timezone) + if timezone == "" { + renderValidationError(writer, validationIssue{Field: "timezone", Message: "timezone is required"}) + return + } + if _, err := time.LoadLocation(timezone); err != nil { + renderValidationError(writer, validationIssue{Field: "timezone", Message: "timezone must be a valid IANA timezone"}) + return + } + item.Timezone = timezone + } + if payload.K8s != nil { + if payload.K8s.Enabled != nil { + item.K8s.Enabled = *payload.K8s.Enabled + } + if payload.K8s.Endpoint != nil { + item.K8s.Endpoint = *payload.K8s.Endpoint + } + if payload.K8s.Namespace != nil { + item.K8s.Namespace = *payload.K8s.Namespace + } + if payload.K8s.Kubeconfig != nil { + item.K8s.Kubeconfig = strings.TrimSpace(*payload.K8s.Kubeconfig) + } + } + if payload.Worker != nil { + if payload.Worker.PollIntervalMS != nil && *payload.Worker.PollIntervalMS > 0 { + item.Worker.PollIntervalMS = *payload.Worker.PollIntervalMS + } + if payload.Worker.MaxConcurrentRuns != nil && *payload.Worker.MaxConcurrentRuns > 0 { + item.Worker.MaxConcurrentRuns = *payload.Worker.MaxConcurrentRuns + } + } + if payload.AI != nil { + if payload.AI.Provider != nil { + item.AI.Provider = strings.TrimSpace(*payload.AI.Provider) + } + if payload.AI.BaseURL != nil { + item.AI.BaseURL = strings.TrimSpace(*payload.AI.BaseURL) + } + if payload.AI.Model != nil { + item.AI.Model = strings.TrimSpace(*payload.AI.Model) + } + if payload.AI.APIKey != nil { + item.AI.APIKey = strings.TrimSpace(*payload.AI.APIKey) + } + } + if payload.Defaults != nil { + if payload.Defaults.CPURequest != nil { + item.Defaults.CPURequest = *payload.Defaults.CPURequest + } + if payload.Defaults.CPULimit != nil { + item.Defaults.CPULimit = *payload.Defaults.CPULimit + } + if payload.Defaults.MemoryRequest != nil { + item.Defaults.MemoryRequest = *payload.Defaults.MemoryRequest + } + if payload.Defaults.MemoryLimit != nil { + item.Defaults.MemoryLimit = *payload.Defaults.MemoryLimit + } + } + item.UpdatedAt = nowISO() + if err := server.store.UpsertSettings(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, sanitizeSettings(item)) +} + +func (server *server) resolveAIService(ctx context.Context) ai.Service { + settings, err := server.store.GetSettings(ctx) + if err != nil || settings == nil { + return server.ai + } + if strings.TrimSpace(settings.AI.APIKey) != "" { + return ai.NewService(settings.AI.BaseURL, settings.AI.APIKey, settings.AI.Model) + } + if server.ai != nil && server.ai.Status().Enabled { + return server.ai + } + return ai.NewDisabledService(settings.AI.BaseURL, settings.AI.Model, "服务端未配置 AI_API_KEY") +} + +func sanitizeSettings(item *domain.SettingsRecord) *domain.SettingsView { + if item == nil { + return nil + } + view := &domain.SettingsView{ + ID: item.ID, + K8s: item.K8s, + Worker: item.Worker, + AI: item.AI, + Defaults: item.Defaults, + Timezone: item.Timezone, + UpdatedAt: item.UpdatedAt, + } + view.K8s.HasKubeconfig = strings.TrimSpace(view.K8s.Kubeconfig) != "" + view.K8s.KubeconfigPreview = maskedSecretPreview(view.K8s.Kubeconfig) + view.K8s.Kubeconfig = "" + view.AI.HasAPIKey = strings.TrimSpace(view.AI.APIKey) != "" + view.AI.APIKeyPreview = maskedSecretPreview(view.AI.APIKey) + view.AI.APIKey = "" + return view +} + +func (server *server) handleStorageBuckets(writer http.ResponseWriter, request *http.Request) { + sourcePath := strings.TrimSpace(request.URL.Query().Get("sourcePath")) + if sourcePath == "" { + renderValidationError(writer, validationIssue{Field: "sourcePath", Message: "sourcePath is required"}) + return + } + items, err := server.store.ListStorageBuckets(request.Context(), sourcePath) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleStorageObjects(writer http.ResponseWriter, request *http.Request) { + sourcePath := strings.TrimSpace(request.URL.Query().Get("sourcePath")) + if sourcePath == "" { + renderValidationError(writer, validationIssue{Field: "sourcePath", Message: "sourcePath is required"}) + return + } + items, err := server.store.ListStorageObjects( + request.Context(), + sourcePath, + request.URL.Query().Get("bucket"), + request.URL.Query().Get("prefix"), + ) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func decodeJSON(writer http.ResponseWriter, request *http.Request, target any) bool { + defer request.Body.Close() + decoder := json.NewDecoder(request.Body) + if err := decoder.Decode(target); err != nil { + renderJSON(writer, http.StatusBadRequest, map[string]any{"error": "Validation failed", "issues": []validationIssue{{Field: "body", Message: "invalid JSON payload"}}}) + return false + } + return true +} + +func renderValidationError(writer http.ResponseWriter, issues ...validationIssue) { + renderJSON(writer, http.StatusBadRequest, map[string]any{"error": "Validation failed", "issues": issues}) +} + +func renderError(writer http.ResponseWriter, status int, err error) { + renderJSON(writer, status, map[string]string{"error": err.Error()}) +} + +func renderAIError(writer http.ResponseWriter, err error) { + switch { + case errors.Is(err, ai.ErrUnavailable): + renderJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "AI 服务未配置,请在服务端配置 AI_API_KEY 后再试"}) + case strings.Contains(err.Error(), "AI 生成超时") || strings.Contains(err.Error(), "AI 服务响应超时"): + renderJSON(writer, http.StatusGatewayTimeout, map[string]string{"error": err.Error()}) + case strings.Contains(strings.ToLower(err.Error()), "description is required"): + renderValidationError(writer, validationIssue{Field: "description", Message: "description is required"}) + default: + renderJSON(writer, http.StatusBadGateway, map[string]string{"error": err.Error()}) + } +} + +func renderJSON(writer http.ResponseWriter, status int, payload any) { + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(payload) +} + +func nowISO() string { + return time.Now().UTC().Format(time.RFC3339Nano) +} diff --git a/internal/api/server_test.go b/internal/api/server_test.go new file mode 100644 index 00000000..d0eb2188 --- /dev/null +++ b/internal/api/server_test.go @@ -0,0 +1,579 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func TestAPIHandlerSupportsDashboardCRUDAndRunQueueing(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + storageRoot := t.TempDir() + for _, bucket := range []string{"incoming", "processed"} { + if err := os.Mkdir(filepath.Join(storageRoot, bucket), 0o755); err != nil { + t.Fatalf("create bucket %s: %v", bucket, err) + } + } + settings, err := dataStore.GetSettings(ctx) + if err != nil { + t.Fatalf("get settings: %v", err) + } + settings.K8s.Kubeconfig = "apiVersion: v1" + if err := dataStore.UpsertSettings(ctx, *settings); err != nil { + t.Fatalf("update settings: %v", err) + } + if _, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "Dashboard 项目", + Description: "用于 dashboard 验证", + }); err != nil { + t.Fatalf("create dashboard project: %v", err) + } + + handler := NewHandler(dataStore, nil) + + dashboardRecorder := httptest.NewRecorder() + handler.ServeHTTP(dashboardRecorder, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil)) + if dashboardRecorder.Code != http.StatusOK { + t.Fatalf("dashboard status: %d", dashboardRecorder.Code) + } + var dashboardBody map[string]any + if err := json.Unmarshal(dashboardRecorder.Body.Bytes(), &dashboardBody); err != nil { + t.Fatalf("decode dashboard: %v", err) + } + projects := dashboardBody["projects"].(map[string]any) + if projects["active"].(float64) <= 0 { + t.Fatalf("expected active projects") + } + + projectPayload := []byte(`{"name":"API 项目","description":"通过 API 创建"}`) + projectRecorder := httptest.NewRecorder() + handler.ServeHTTP(projectRecorder, httptest.NewRequest(http.MethodPost, "/api/projects", bytes.NewReader(projectPayload))) + if projectRecorder.Code != http.StatusCreated { + t.Fatalf("project create status: %d body=%s", projectRecorder.Code, projectRecorder.Body.String()) + } + var createdProject map[string]any + if err := json.Unmarshal(projectRecorder.Body.Bytes(), &createdProject); err != nil { + t.Fatalf("decode created project: %v", err) + } + projectID := createdProject["id"].(string) + if projectID == "" { + t.Fatalf("expected project id") + } + + workflowPayload := []byte(`{"name":"API 工作流","description":"workflow","type":"batch","nodes":[{"id":"node-source","type":"operator","data":{"label":"真实数据源","category":"IO","description":"source","inputs":[],"outputs":[{"id":"out-1","name":"file","type":"file"}],"hyperparams":[{"key":"SOURCE_TEXT","value":"api-test"}]},"position":{"x":0,"y":0}}],"edges":[]}`) + workflowRecorder := httptest.NewRecorder() + handler.ServeHTTP(workflowRecorder, httptest.NewRequest(http.MethodPost, "/api/projects/"+projectID+"/workflows", bytes.NewReader(workflowPayload))) + if workflowRecorder.Code != http.StatusCreated { + t.Fatalf("workflow create status: %d body=%s", workflowRecorder.Code, workflowRecorder.Body.String()) + } + var createdWorkflow map[string]any + if err := json.Unmarshal(workflowRecorder.Body.Bytes(), &createdWorkflow); err != nil { + t.Fatalf("decode workflow: %v", err) + } + + projectsRecorder := httptest.NewRecorder() + handler.ServeHTTP(projectsRecorder, httptest.NewRequest(http.MethodGet, "/api/projects", nil)) + if projectsRecorder.Code != http.StatusOK { + t.Fatalf("projects list status: %d body=%s", projectsRecorder.Code, projectsRecorder.Body.String()) + } + var projectsBody struct { + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(projectsRecorder.Body.Bytes(), &projectsBody); err != nil { + t.Fatalf("decode projects list: %v", err) + } + var countedProject map[string]any + for _, item := range projectsBody.Items { + if item["id"] == projectID { + countedProject = item + break + } + } + if countedProject == nil { + t.Fatalf("expected created project in projects list") + } + if countedProject["workflowCount"].(float64) != 1 { + t.Fatalf("expected workflowCount=1 in projects list, got %v", countedProject["workflowCount"]) + } + + projectGetRecorder := httptest.NewRecorder() + handler.ServeHTTP(projectGetRecorder, httptest.NewRequest(http.MethodGet, "/api/projects/"+projectID, nil)) + if projectGetRecorder.Code != http.StatusOK { + t.Fatalf("project get status: %d body=%s", projectGetRecorder.Code, projectGetRecorder.Body.String()) + } + var projectGetBody map[string]any + if err := json.Unmarshal(projectGetRecorder.Body.Bytes(), &projectGetBody); err != nil { + t.Fatalf("decode project get: %v", err) + } + if projectGetBody["workflowCount"].(float64) != 1 { + t.Fatalf("expected workflowCount=1 in project get, got %v", projectGetBody["workflowCount"]) + } + + runPayload := []byte(`{"trigger":"manual"}`) + runRecorder := httptest.NewRecorder() + handler.ServeHTTP(runRecorder, httptest.NewRequest(http.MethodPost, "/api/workflows/"+createdWorkflow["id"].(string)+"/run", bytes.NewReader(runPayload))) + if runRecorder.Code != http.StatusAccepted { + t.Fatalf("run create status: %d body=%s", runRecorder.Code, runRecorder.Body.String()) + } + var queuedRun map[string]any + if err := json.Unmarshal(runRecorder.Body.Bytes(), &queuedRun); err != nil { + t.Fatalf("decode queued run: %v", err) + } + if queuedRun["status"].(string) != "queued" { + t.Fatalf("expected queued status, got %v", queuedRun["status"]) + } + + userPayload := []byte(`{"name":"API User","email":"api-user@example.com","role":"admin"}`) + userRecorder := httptest.NewRecorder() + handler.ServeHTTP(userRecorder, httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(userPayload))) + if userRecorder.Code != http.StatusCreated { + t.Fatalf("user create status: %d body=%s", userRecorder.Code, userRecorder.Body.String()) + } + + runsRecorder := httptest.NewRecorder() + handler.ServeHTTP(runsRecorder, httptest.NewRequest(http.MethodGet, "/api/runs", nil)) + if runsRecorder.Code != http.StatusOK { + t.Fatalf("runs list status: %d", runsRecorder.Code) + } + var runsBody struct { + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(runsRecorder.Body.Bytes(), &runsBody); err != nil { + t.Fatalf("decode runs: %v", err) + } + foundRun := false + for _, item := range runsBody.Items { + if item["id"] == queuedRun["id"] { + foundRun = true + break + } + } + if !foundRun { + t.Fatalf("expected queued run in listing") + } + + bucketsRecorder := httptest.NewRecorder() + handler.ServeHTTP(bucketsRecorder, httptest.NewRequest(http.MethodGet, "/api/storage/buckets?sourcePath="+storageRoot, nil)) + if bucketsRecorder.Code != http.StatusOK { + t.Fatalf("buckets status: %d", bucketsRecorder.Code) + } + var bucketsBody struct { + Items []string `json:"items"` + } + if err := json.Unmarshal(bucketsRecorder.Body.Bytes(), &bucketsBody); err != nil { + t.Fatalf("decode buckets: %v", err) + } + if len(bucketsBody.Items) == 0 { + t.Fatalf("expected storage buckets") + } + + objectsRecorder := httptest.NewRecorder() + handler.ServeHTTP( + objectsRecorder, + httptest.NewRequest(http.MethodGet, "/api/storage/objects?sourcePath="+storageRoot+"&bucket=incoming&prefix=", nil), + ) + if objectsRecorder.Code != http.StatusOK { + t.Fatalf("objects status: %d", objectsRecorder.Code) + } + + missingSourcePathRecorder := httptest.NewRecorder() + handler.ServeHTTP(missingSourcePathRecorder, httptest.NewRequest(http.MethodGet, "/api/storage/buckets", nil)) + if missingSourcePathRecorder.Code != http.StatusBadRequest { + t.Fatalf("missing sourcePath status: %d", missingSourcePathRecorder.Code) + } + + aiRecorder := httptest.NewRecorder() + handler.ServeHTTP(aiRecorder, httptest.NewRequest(http.MethodGet, "/api/ai/status", nil)) + if aiRecorder.Code != http.StatusOK { + t.Fatalf("ai status: %d", aiRecorder.Code) + } +} + +func TestSettingsAIConfigurationDrivesAIStatus(t *testing.T) { + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + settingsPayload := []byte(`{ + "k8s": { + "endpoint": "https://120.48.161.230:6443", + "namespace": "droboticflow", + "kubeconfig": "apiVersion: v1\nkind: Config" + }, + "ai": { + "provider": "dashscope-openai-compatible", + "baseURL": "https://coding.dashscope.aliyuncs.com/v1", + "model": "glm-5", + "apiKey": "test-key" + }, + "timezone": "Asia/Tokyo" + }`) + + updateRecorder := httptest.NewRecorder() + updateRequest := httptest.NewRequest(http.MethodPut, "/api/settings", bytes.NewReader(settingsPayload)) + handler.ServeHTTP(updateRecorder, updateRequest) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("settings update status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + + statusRecorder := httptest.NewRecorder() + handler.ServeHTTP(statusRecorder, httptest.NewRequest(http.MethodGet, "/api/ai/status", nil)) + if statusRecorder.Code != http.StatusOK { + t.Fatalf("ai status code: %d body=%s", statusRecorder.Code, statusRecorder.Body.String()) + } + + var statusBody struct { + Enabled bool `json:"enabled"` + BaseURL string `json:"baseURL"` + Model string `json:"model"` + Provider string `json:"provider"` + } + if err := json.Unmarshal(statusRecorder.Body.Bytes(), &statusBody); err != nil { + t.Fatalf("decode ai status: %v", err) + } + if !statusBody.Enabled { + t.Fatalf("expected ai status enabled") + } + if statusBody.Model != "glm-5" { + t.Fatalf("unexpected model: %s", statusBody.Model) + } + if statusBody.BaseURL != "https://coding.dashscope.aliyuncs.com/v1" { + t.Fatalf("unexpected baseURL: %s", statusBody.BaseURL) + } + + settingsRecorder := httptest.NewRecorder() + handler.ServeHTTP(settingsRecorder, httptest.NewRequest(http.MethodGet, "/api/settings", nil)) + if settingsRecorder.Code != http.StatusOK { + t.Fatalf("settings get status: %d body=%s", settingsRecorder.Code, settingsRecorder.Body.String()) + } + var settingsBody struct { + K8s struct { + Kubeconfig string `json:"kubeconfig"` + KubeconfigPreview string `json:"kubeconfigPreview"` + HasKubeconfig bool `json:"hasKubeconfig"` + } `json:"k8s"` + AI struct { + APIKey string `json:"apiKey"` + APIKeyPreview string `json:"apiKeyPreview"` + HasAPIKey bool `json:"hasAPIKey"` + } `json:"ai"` + Timezone string `json:"timezone"` + Storage any `json:"storage"` + } + if err := json.Unmarshal(settingsRecorder.Body.Bytes(), &settingsBody); err != nil { + t.Fatalf("decode settings: %v", err) + } + if settingsBody.K8s.Kubeconfig != "" { + t.Fatalf("expected settings kubeconfig to be sanitized, got %q", settingsBody.K8s.Kubeconfig) + } + if !settingsBody.K8s.HasKubeconfig { + t.Fatalf("expected hasKubeconfig to be true") + } + if settingsBody.K8s.KubeconfigPreview == "" { + t.Fatalf("expected kubeconfig preview to be returned") + } + if settingsBody.AI.APIKey != "" { + t.Fatalf("expected settings apiKey to be sanitized") + } + if !settingsBody.AI.HasAPIKey { + t.Fatalf("expected hasAPIKey to be true") + } + if settingsBody.AI.APIKeyPreview != "te****ey" { + t.Fatalf("expected apiKey preview to be masked, got %q", settingsBody.AI.APIKeyPreview) + } + if settingsBody.Timezone != "Asia/Tokyo" { + t.Fatalf("expected timezone to roundtrip, got %q", settingsBody.Timezone) + } + if settingsBody.Storage != nil { + t.Fatalf("expected storage to be omitted from settings response") + } +} + +func TestTemplateExecutionConfigValidationAndPersistence(t *testing.T) { + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + createPayload := []byte(`{ + "name":"执行保障模板", + "description":"template execution config", + "type":"streaming", + "maxConcurrentRuns":2, + "nodeTimeoutSeconds":12, + "runTimeoutSeconds":34, + "maxRetryAttempts":1, + "retryBackoffSeconds":56, + "schedule":{ + "interval":90, + "sourcePath":"oss://incoming" + }, + "nodes":[], + "edges":[] + }`) + + createRecorder := httptest.NewRecorder() + handler.ServeHTTP( + createRecorder, + httptest.NewRequest(http.MethodPost, "/api/templates", bytes.NewReader(createPayload)), + ) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("template create status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var created map[string]any + if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created template: %v", err) + } + if created["type"].(string) != "streaming" { + t.Fatalf("expected streaming type, got %v", created["type"]) + } + if created["retryBackoffSeconds"].(float64) != 56 { + t.Fatalf("expected retryBackoffSeconds=56, got %v", created["retryBackoffSeconds"]) + } + + templateID := created["id"].(string) + getRecorder := httptest.NewRecorder() + handler.ServeHTTP(getRecorder, httptest.NewRequest(http.MethodGet, "/api/templates/"+templateID, nil)) + if getRecorder.Code != http.StatusOK { + t.Fatalf("template get status: %d body=%s", getRecorder.Code, getRecorder.Body.String()) + } + + var loaded map[string]any + if err := json.Unmarshal(getRecorder.Body.Bytes(), &loaded); err != nil { + t.Fatalf("decode loaded template: %v", err) + } + if loaded["nodeTimeoutSeconds"].(float64) != 12 { + t.Fatalf("expected nodeTimeoutSeconds=12, got %v", loaded["nodeTimeoutSeconds"]) + } + schedule := loaded["schedule"].(map[string]any) + if schedule["interval"].(float64) != 90 { + t.Fatalf("expected interval=90, got %v", schedule["interval"]) + } + + updatePayload := []byte(`{ + "type":"batch", + "maxConcurrentRuns":3, + "nodeTimeoutSeconds":6, + "runTimeoutSeconds":18, + "maxRetryAttempts":2, + "retryBackoffSeconds":9 + }`) + updateRecorder := httptest.NewRecorder() + handler.ServeHTTP( + updateRecorder, + httptest.NewRequest(http.MethodPut, "/api/templates/"+templateID, bytes.NewReader(updatePayload)), + ) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("template update status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + + var updated map[string]any + if err := json.Unmarshal(updateRecorder.Body.Bytes(), &updated); err != nil { + t.Fatalf("decode updated template: %v", err) + } + if updated["type"].(string) != "batch" { + t.Fatalf("expected batch type after update, got %v", updated["type"]) + } + if updated["maxConcurrentRuns"].(float64) != 3 { + t.Fatalf("expected maxConcurrentRuns=3 after update, got %v", updated["maxConcurrentRuns"]) + } + if updated["retryBackoffSeconds"].(float64) != 9 { + t.Fatalf("expected retryBackoffSeconds=9 after update, got %v", updated["retryBackoffSeconds"]) + } + + invalidPayload := []byte(`{"maxRetryAttempts":-1}`) + invalidRecorder := httptest.NewRecorder() + handler.ServeHTTP( + invalidRecorder, + httptest.NewRequest(http.MethodPut, "/api/templates/"+templateID, bytes.NewReader(invalidPayload)), + ) + if invalidRecorder.Code != http.StatusBadRequest { + t.Fatalf("invalid template update status: %d body=%s", invalidRecorder.Code, invalidRecorder.Body.String()) + } +} + +func TestWorkflowExecutionConfigValidationAndPersistence(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "执行配置项目", + Description: "workflow execution settings", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + createPayload := []byte(`{ + "name":"执行配置工作流", + "description":"workflow", + "type":"batch", + "maxConcurrentRuns":2, + "nodeTimeoutSeconds":30, + "runTimeoutSeconds":120, + "maxRetryAttempts":3, + "retryBackoffSeconds":15, + "nodes":[{"id":"node-source","type":"operator","data":{"label":"真实数据源","category":"IO","description":"source","inputs":[],"outputs":[{"id":"out-1","name":"file","type":"file"}],"hyperparams":[{"key":"SOURCE_TEXT","value":"api-test"}]},"position":{"x":0,"y":0}}], + "edges":[] + }`) + + createRecorder := httptest.NewRecorder() + createRequest := httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(createPayload)) + handler.ServeHTTP(createRecorder, createRequest) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("workflow create status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + var created domain.WorkflowRecord + if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { + t.Fatalf("decode created workflow: %v", err) + } + if created.RunTimeoutSeconds != 120 { + t.Fatalf("expected runTimeoutSeconds=120, got %d", created.RunTimeoutSeconds) + } + if created.NodeTimeoutSeconds != 30 { + t.Fatalf("expected nodeTimeoutSeconds=30, got %d", created.NodeTimeoutSeconds) + } + if created.MaxRetryAttempts != 3 { + t.Fatalf("expected maxRetryAttempts=3, got %d", created.MaxRetryAttempts) + } + if created.RetryBackoffSeconds != 15 { + t.Fatalf("expected retryBackoffSeconds=15, got %d", created.RetryBackoffSeconds) + } + + updatePayload := []byte(`{"nodeTimeoutSeconds":12,"runTimeoutSeconds":45,"maxRetryAttempts":1,"retryBackoffSeconds":9}`) + updateRecorder := httptest.NewRecorder() + updateRequest := httptest.NewRequest(http.MethodPut, "/api/workflows/"+created.ID, bytes.NewReader(updatePayload)) + handler.ServeHTTP(updateRecorder, updateRequest) + if updateRecorder.Code != http.StatusOK { + t.Fatalf("workflow update status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } + + var updated domain.WorkflowRecord + if err := json.Unmarshal(updateRecorder.Body.Bytes(), &updated); err != nil { + t.Fatalf("decode updated workflow: %v", err) + } + if updated.RunTimeoutSeconds != 45 { + t.Fatalf("expected updated runTimeoutSeconds=45, got %d", updated.RunTimeoutSeconds) + } + if updated.NodeTimeoutSeconds != 12 { + t.Fatalf("expected updated nodeTimeoutSeconds=12, got %d", updated.NodeTimeoutSeconds) + } + if updated.MaxRetryAttempts != 1 { + t.Fatalf("expected updated maxRetryAttempts=1, got %d", updated.MaxRetryAttempts) + } + if updated.RetryBackoffSeconds != 9 { + t.Fatalf("expected updated retryBackoffSeconds=9, got %d", updated.RetryBackoffSeconds) + } + + invalidCreatePayload := []byte(`{ + "name":"非法工作流", + "description":"workflow", + "type":"batch", + "nodeTimeoutSeconds":-1, + "runTimeoutSeconds":-1, + "maxRetryAttempts":-2, + "retryBackoffSeconds":-3, + "nodes":[], + "edges":[] + }`) + invalidCreateRecorder := httptest.NewRecorder() + invalidCreateRequest := httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflows", bytes.NewReader(invalidCreatePayload)) + handler.ServeHTTP(invalidCreateRecorder, invalidCreateRequest) + if invalidCreateRecorder.Code != http.StatusBadRequest { + t.Fatalf("invalid create status: %d body=%s", invalidCreateRecorder.Code, invalidCreateRecorder.Body.String()) + } + + invalidUpdatePayload := []byte(`{"nodeTimeoutSeconds":-1,"runTimeoutSeconds":-1,"maxRetryAttempts":-1,"retryBackoffSeconds":-1}`) + invalidUpdateRecorder := httptest.NewRecorder() + invalidUpdateRequest := httptest.NewRequest(http.MethodPut, "/api/workflows/"+created.ID, bytes.NewReader(invalidUpdatePayload)) + handler.ServeHTTP(invalidUpdateRecorder, invalidUpdateRequest) + if invalidUpdateRecorder.Code != http.StatusBadRequest { + t.Fatalf("invalid update status: %d body=%s", invalidUpdateRecorder.Code, invalidUpdateRecorder.Body.String()) + } +} + +func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "已完成项目", + Description: "禁止重复运行", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "已完成工作流", + Description: "completed", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "数据源", + Category: "IO", + Description: "source", + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + workflow.Status = domain.WorkflowStatusCompleted + if err := dataStore.UpsertWorkflow(ctx, *workflow); err != nil { + t.Fatalf("mark workflow completed: %v", err) + } + + handler := NewHandler(dataStore, nil) + runRecorder := httptest.NewRecorder() + runRequest := httptest.NewRequest( + http.MethodPost, + "/api/workflows/"+workflow.ID+"/run", + bytes.NewReader([]byte(`{"trigger":"manual"}`)), + ) + handler.ServeHTTP(runRecorder, runRequest) + if runRecorder.Code != http.StatusConflict { + t.Fatalf("expected 409 for completed workflow rerun, got %d body=%s", runRecorder.Code, runRecorder.Body.String()) + } + + var body map[string]string + if err := json.Unmarshal(runRecorder.Body.Bytes(), &body); err != nil { + t.Fatalf("decode rerun response: %v", err) + } + if body["error"] == "" { + t.Fatalf("expected rerun error message") + } +} + +func TestSettingsRejectsInvalidTimezone(t *testing.T) { + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + + updateRecorder := httptest.NewRecorder() + updateRequest := httptest.NewRequest( + http.MethodPut, + "/api/settings", + bytes.NewReader([]byte(`{"timezone":"Mars/Base"}`)), + ) + handler.ServeHTTP(updateRecorder, updateRequest) + if updateRecorder.Code != http.StatusBadRequest { + t.Fatalf("expected invalid timezone to fail, got %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 00000000..65ea2f67 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,68 @@ +package config + +import ( + "os" + "path/filepath" + "strconv" +) + +type AppConfig struct { + Port int + WorkerPort int + MongoURI string + MongoDB string + MongoConnectRetries int + MongoConnectRetryDelayMS int + PollIntervalMS int + K8sEnabled bool + K8sNamespace string + K8sImagePullSecretName string + OperatorRunnerImage string + LocalOperatorRunnerCmd string + AIAPIBaseURL string + AIAPIKey string + AIModel string + DistDir string +} + +func parseInt(value string, fallback int) int { + parsed, err := strconv.Atoi(value) + if err != nil { + return fallback + } + return parsed +} + +func boolValue(value string) bool { + return value == "true" || value == "1" || value == "yes" +} + +func Load() AppConfig { + workingDirectory, _ := os.Getwd() + return AppConfig{ + Port: parseInt(os.Getenv("PORT"), 3001), + WorkerPort: parseInt(os.Getenv("WORKER_PORT"), 3002), + MongoURI: getenv("MONGO_URI", "mongodb://127.0.0.1:27017"), + MongoDB: getenv("MONGO_DB", "droboticflow"), + MongoConnectRetries: parseInt(os.Getenv("MONGO_CONNECT_RETRIES"), 30), + MongoConnectRetryDelayMS: parseInt(os.Getenv("MONGO_CONNECT_RETRY_DELAY_MS"), 2000), + PollIntervalMS: parseInt(os.Getenv("WORKER_POLL_INTERVAL_MS"), 2000), + K8sEnabled: boolValue(os.Getenv("K8S_ENABLED")), + K8sNamespace: getenv("K8S_NAMESPACE", "droboticflow"), + K8sImagePullSecretName: getenv("K8S_IMAGE_PULL_SECRET_NAME", "droboticflow-registry"), + OperatorRunnerImage: getenv("OPERATOR_RUNNER_IMAGE", "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:latest"), + LocalOperatorRunnerCmd: getenv("LOCAL_OPERATOR_RUNNER_CMD", "python3 backend/operator-runner/main.py"), + AIAPIBaseURL: getenv("AI_API_BASE_URL", "https://coding.dashscope.aliyuncs.com/v1"), + AIAPIKey: os.Getenv("AI_API_KEY"), + AIModel: getenv("AI_MODEL", "glm-5"), + DistDir: getenv("DIST_DIR", filepath.Join(workingDirectory, "dist")), + } +} + +func getenv(key string, fallback string) string { + value := os.Getenv(key) + if value == "" { + return fallback + } + return value +} diff --git a/internal/domain/id.go b/internal/domain/id.go new file mode 100644 index 00000000..78c067f1 --- /dev/null +++ b/internal/domain/id.go @@ -0,0 +1,12 @@ +package domain + +import ( + "crypto/rand" + "encoding/hex" +) + +func NewID(prefix string) string { + random := make([]byte, 16) + _, _ = rand.Read(random) + return prefix + "-" + hex.EncodeToString(random) +} diff --git a/internal/domain/types.go b/internal/domain/types.go new file mode 100644 index 00000000..4d7a9c76 --- /dev/null +++ b/internal/domain/types.go @@ -0,0 +1,445 @@ +package domain + +type UserRole string + +const ( + UserRoleAdmin UserRole = "admin" + UserRoleAuditor UserRole = "auditor" + UserRoleOperator UserRole = "operator" +) + +type PortType string + +const ( + PortTypeJSON PortType = "json" + PortTypeImage PortType = "image" + PortTypeVideo PortType = "video" + PortTypeTensor PortType = "tensor" + PortTypeFile PortType = "file" +) + +type WorkflowStatus string + +const ( + WorkflowStatusDraft WorkflowStatus = "draft" + WorkflowStatusReady WorkflowStatus = "ready" + WorkflowStatusRunning WorkflowStatus = "running" + WorkflowStatusCompleted WorkflowStatus = "completed" + WorkflowStatusFailed WorkflowStatus = "failed" +) + +type WorkflowType string + +const ( + WorkflowTypeBatch WorkflowType = "batch" + WorkflowTypeStreaming WorkflowType = "streaming" +) + +type RunTrigger string + +const ( + RunTriggerManual RunTrigger = "manual" + RunTriggerScheduled RunTrigger = "scheduled" + RunTriggerRetry RunTrigger = "retry" +) + +type RunStatus string + +const ( + RunStatusQueued RunStatus = "queued" + RunStatusRunning RunStatus = "running" + RunStatusWaitingForAudit RunStatus = "waiting_for_audit" + RunStatusCompleted RunStatus = "completed" + RunStatusFailed RunStatus = "failed" + RunStatusCancelled RunStatus = "cancelled" +) + +type AuditStatus string + +const ( + AuditStatusPending AuditStatus = "pending" + AuditStatusApproved AuditStatus = "approved" + AuditStatusRejected AuditStatus = "rejected" +) + +type NodeExecutionStatus string + +const ( + NodeExecutionStatusQueued NodeExecutionStatus = "queued" + NodeExecutionStatusRunning NodeExecutionStatus = "running" + NodeExecutionStatusWaitingForAudit NodeExecutionStatus = "waiting_for_audit" + NodeExecutionStatusCompleted NodeExecutionStatus = "completed" + NodeExecutionStatusFailed NodeExecutionStatus = "failed" +) + +type Position struct { + X float64 `json:"x" bson:"x"` + Y float64 `json:"y" bson:"y"` +} + +type PortDefinition struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Type PortType `json:"type" bson:"type"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + Transmission string `json:"transmission,omitempty" bson:"transmission,omitempty"` +} + +type HyperParam struct { + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + Type string `json:"type,omitempty" bson:"type,omitempty"` +} + +type WorkflowNodeData struct { + Label string `json:"label" bson:"label"` + Category string `json:"category" bson:"category"` + Description string `json:"description" bson:"description"` + Inputs []PortDefinition `json:"inputs" bson:"inputs"` + Outputs []PortDefinition `json:"outputs" bson:"outputs"` + Image string `json:"image,omitempty" bson:"image,omitempty"` + SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` + Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"` + Pattern string `json:"pattern,omitempty" bson:"pattern,omitempty"` + CPU string `json:"cpu,omitempty" bson:"cpu,omitempty"` + Memory string `json:"memory,omitempty" bson:"memory,omitempty"` +} + +type WorkflowNode struct { + ID string `json:"id" bson:"id"` + Type string `json:"type" bson:"type"` + Data WorkflowNodeData `json:"data" bson:"data"` + Position Position `json:"position" bson:"position"` +} + +type WorkflowEdge struct { + ID string `json:"id" bson:"id"` + Source string `json:"source" bson:"source"` + SourceHandle string `json:"sourceHandle,omitempty" bson:"sourceHandle,omitempty"` + Target string `json:"target" bson:"target"` + TargetHandle string `json:"targetHandle,omitempty" bson:"targetHandle,omitempty"` + Animated bool `json:"animated,omitempty" bson:"animated,omitempty"` +} + +type ProjectRecord struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Status string `json:"status" bson:"status"` + WorkflowCount int `json:"workflowCount" bson:"-"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type OperatorResources struct { + CPURequest string `json:"cpuRequest,omitempty" bson:"cpuRequest,omitempty"` + CPULimit string `json:"cpuLimit,omitempty" bson:"cpuLimit,omitempty"` + MemoryRequest string `json:"memoryRequest,omitempty" bson:"memoryRequest,omitempty"` + MemoryLimit string `json:"memoryLimit,omitempty" bson:"memoryLimit,omitempty"` +} + +type OperatorRecord struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` + Category string `json:"category" bson:"category"` + Description string `json:"description" bson:"description"` + Inputs []PortDefinition `json:"inputs" bson:"inputs"` + Outputs []PortDefinition `json:"outputs" bson:"outputs"` + ConfigSchema map[string]any `json:"configSchema,omitempty" bson:"configSchema,omitempty"` + Image string `json:"image,omitempty" bson:"image,omitempty"` + SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` + Resources *OperatorResources `json:"resources,omitempty" bson:"resources,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type WorkflowSchedule struct { + Interval int `json:"interval" bson:"interval"` + Cron string `json:"cron,omitempty" bson:"cron,omitempty"` + SourcePath string `json:"sourcePath,omitempty" bson:"sourcePath,omitempty"` +} + +type WorkflowRecord struct { + ID string `json:"id" bson:"id"` + ProjectID string `json:"projectId" bson:"projectId"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type WorkflowType `json:"type" bson:"type"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` + Schedule *WorkflowSchedule `json:"schedule,omitempty" bson:"schedule,omitempty"` + Nodes []WorkflowNode `json:"nodes" bson:"nodes"` + Edges []WorkflowEdge `json:"edges" bson:"edges"` + Status WorkflowStatus `json:"status" bson:"status"` + LastRunID string `json:"lastRunId,omitempty" bson:"lastRunId,omitempty"` + LastRunAt string `json:"lastRunAt,omitempty" bson:"lastRunAt,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type TemplateRecord struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type WorkflowType `json:"type" bson:"type"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` + Schedule *WorkflowSchedule `json:"schedule,omitempty" bson:"schedule,omitempty"` + Nodes []WorkflowNode `json:"nodes" bson:"nodes"` + Edges []WorkflowEdge `json:"edges" bson:"edges"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type LogEntryRecord struct { + Timestamp string `json:"timestamp" bson:"timestamp"` + Level string `json:"level" bson:"level"` + Message string `json:"message" bson:"message"` + Operator string `json:"operator,omitempty" bson:"operator,omitempty"` +} + +type ResourceSample struct { + Timestamp string `json:"timestamp" bson:"timestamp"` + CPU int `json:"cpu" bson:"cpu"` + Memory int `json:"memory" bson:"memory"` + GPU int `json:"gpu,omitempty" bson:"gpu,omitempty"` +} + +type ArtifactMetadata struct { + Duration string `json:"duration,omitempty" bson:"duration,omitempty"` + Resolution string `json:"resolution,omitempty" bson:"resolution,omitempty"` + Codec string `json:"codec,omitempty" bson:"codec,omitempty"` + Thumbnail string `json:"thumbnail,omitempty" bson:"thumbnail,omitempty"` + SHA256 string `json:"sha256,omitempty" bson:"sha256,omitempty"` + MimeType string `json:"mimeType,omitempty" bson:"mimeType,omitempty"` +} + +type ExecutionArtifact struct { + ID string `json:"id" bson:"id"` + RunID string `json:"runId" bson:"runId"` + NodeID string `json:"nodeId" bson:"nodeId"` + PortID string `json:"portId,omitempty" bson:"portId,omitempty"` + PortName string `json:"portName,omitempty" bson:"portName,omitempty"` + Name string `json:"name" bson:"name"` + Type PortType `json:"type" bson:"type"` + Size string `json:"size" bson:"size"` + Path string `json:"path" bson:"path"` + Metadata *ArtifactMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"` + Data map[string]any `json:"data,omitempty" bson:"data,omitempty"` +} + +type NodeExecutionRecord struct { + NodeID string `json:"nodeId" bson:"nodeId"` + NodeLabel string `json:"nodeLabel" bson:"nodeLabel"` + Status NodeExecutionStatus `json:"status" bson:"status"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` + Logs []LogEntryRecord `json:"logs" bson:"logs"` + Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` + ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` + Error string `json:"error,omitempty" bson:"error,omitempty"` +} + +type K8sRef struct { + NodeID string `json:"nodeId" bson:"nodeId"` + JobName string `json:"jobName" bson:"jobName"` + Namespace string `json:"namespace" bson:"namespace"` +} + +type RunRecord struct { + ID string `json:"id" bson:"id"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + WorkflowName string `json:"workflowName" bson:"workflowName"` + ProjectID string `json:"projectId" bson:"projectId"` + ProjectName string `json:"projectName" bson:"projectName"` + Trigger RunTrigger `json:"trigger" bson:"trigger"` + Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` + ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` + AvailableAt string `json:"availableAt,omitempty" bson:"availableAt,omitempty"` + Status RunStatus `json:"status" bson:"status"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` + DurationMS int64 `json:"durationMs,omitempty" bson:"durationMs,omitempty"` + CurrentNodeID string `json:"currentNodeId,omitempty" bson:"currentNodeId,omitempty"` + NodeExecutions []NodeExecutionRecord `json:"nodeExecutions" bson:"nodeExecutions"` + Logs []LogEntryRecord `json:"logs" bson:"logs"` + ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` + Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` + K8sRefs []K8sRef `json:"k8sRefs" bson:"k8sRefs"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type AuditTaskPayload struct { + Image string `json:"image" bson:"image"` + Confidence float64 `json:"confidence" bson:"confidence"` + DetectedObjects []string `json:"detectedObjects" bson:"detectedObjects"` + Reason string `json:"reason" bson:"reason"` +} + +type AuditTaskRecord struct { + ID string `json:"id" bson:"id"` + RunID string `json:"runId" bson:"runId"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + ProjectID string `json:"projectId" bson:"projectId"` + NodeID string `json:"nodeId" bson:"nodeId"` + WorkflowName string `json:"workflowName" bson:"workflowName"` + ProjectName string `json:"projectName" bson:"projectName"` + OperatorName string `json:"operatorName" bson:"operatorName"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + Status AuditStatus `json:"status" bson:"status"` + Data AuditTaskPayload `json:"data" bson:"data"` +} + +type UserRecord struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Email string `json:"email" bson:"email"` + Role UserRole `json:"role" bson:"role"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type K8sSettings struct { + Enabled bool `json:"enabled" bson:"enabled"` + Endpoint string `json:"endpoint,omitempty" bson:"endpoint,omitempty"` + Namespace string `json:"namespace" bson:"namespace"` + Kubeconfig string `json:"kubeconfig,omitempty" bson:"kubeconfig,omitempty"` + KubeconfigPreview string `json:"kubeconfigPreview,omitempty" bson:"-"` + HasKubeconfig bool `json:"hasKubeconfig,omitempty" bson:"-"` +} + +type WorkerSettings struct { + PollIntervalMS int `json:"pollIntervalMs" bson:"pollIntervalMs"` + MaxConcurrentRuns int `json:"maxConcurrentRuns" bson:"maxConcurrentRuns"` +} + +type StorageSettings struct { + Endpoint string `json:"endpoint" bson:"endpoint"` + Buckets []string `json:"buckets" bson:"buckets"` +} + +type AISettings struct { + Provider string `json:"provider" bson:"provider"` + BaseURL string `json:"baseURL" bson:"baseURL"` + Model string `json:"model" bson:"model"` + APIKey string `json:"apiKey,omitempty" bson:"apiKey,omitempty"` + APIKeyPreview string `json:"apiKeyPreview,omitempty" bson:"-"` + HasAPIKey bool `json:"hasAPIKey,omitempty" bson:"-"` +} + +type DefaultResources struct { + CPURequest string `json:"cpuRequest" bson:"cpuRequest"` + CPULimit string `json:"cpuLimit" bson:"cpuLimit"` + MemoryRequest string `json:"memoryRequest" bson:"memoryRequest"` + MemoryLimit string `json:"memoryLimit" bson:"memoryLimit"` +} + +type SettingsRecord struct { + ID string `json:"id" bson:"id"` + K8s K8sSettings `json:"k8s" bson:"k8s"` + Worker WorkerSettings `json:"worker" bson:"worker"` + Storage StorageSettings `json:"storage" bson:"storage"` + AI AISettings `json:"ai" bson:"ai"` + Defaults DefaultResources `json:"defaults" bson:"defaults"` + Timezone string `json:"timezone" bson:"timezone"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type SettingsView struct { + ID string `json:"id"` + K8s K8sSettings `json:"k8s"` + Worker WorkerSettings `json:"worker"` + AI AISettings `json:"ai"` + Defaults DefaultResources `json:"defaults"` + Timezone string `json:"timezone"` + UpdatedAt string `json:"updatedAt"` +} + +type DashboardSummary struct { + Projects struct { + Active int `json:"active"` + Archived int `json:"archived"` + } `json:"projects"` + Operators struct { + Total int `json:"total"` + Custom int `json:"custom"` + } `json:"operators"` + Jobs struct { + Today int `json:"today"` + SuccessRate float64 `json:"successRate"` + Running int `json:"running"` + Failed int `json:"failed"` + AverageDurationMS int64 `json:"averageDurationMs"` + } `json:"jobs"` + Audits struct { + Pending int `json:"pending"` + } `json:"audits"` +} + +type CreateProjectInput struct { + Name string `json:"name"` + Description string `json:"description"` +} + +type CreateWorkflowInput struct { + ProjectID string `json:"projectId"` + Name string `json:"name"` + Description string `json:"description"` + Type WorkflowType `json:"type"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty"` + Nodes []WorkflowNode `json:"nodes"` + Edges []WorkflowEdge `json:"edges"` + Schedule *WorkflowSchedule `json:"schedule,omitempty"` +} + +type CreateTemplateInput struct { + Name string `json:"name"` + Description string `json:"description"` + Type WorkflowType `json:"type"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty"` + Schedule *WorkflowSchedule `json:"schedule,omitempty"` + Nodes []WorkflowNode `json:"nodes"` + Edges []WorkflowEdge `json:"edges"` +} + +type CreateOperatorInput struct { + Name string `json:"name"` + Type string `json:"type"` + Category string `json:"category"` + Description string `json:"description"` + Inputs []PortDefinition `json:"inputs"` + Outputs []PortDefinition `json:"outputs"` + ConfigSchema map[string]any `json:"configSchema,omitempty"` + Image string `json:"image,omitempty"` + SkillDefinition string `json:"skillDefinition,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty"` + Resources *OperatorResources `json:"resources,omitempty"` +} + +type CreateUserInput struct { + Name string `json:"name"` + Email string `json:"email"` + Role UserRole `json:"role"` +} diff --git a/internal/domain/workflow.go b/internal/domain/workflow.go new file mode 100644 index 00000000..b7cac647 --- /dev/null +++ b/internal/domain/workflow.go @@ -0,0 +1,83 @@ +package domain + +import "strings" + +func GetNodeByID(nodes []WorkflowNode, nodeID string) *WorkflowNode { + for index := range nodes { + if nodes[index].ID == nodeID { + return &nodes[index] + } + } + return nil +} + +func GetIncomingEdges(edges []WorkflowEdge, nodeID string) []WorkflowEdge { + items := make([]WorkflowEdge, 0) + for _, edge := range edges { + if edge.Target == nodeID { + items = append(items, edge) + } + } + return items +} + +func GetOutgoingEdges(edges []WorkflowEdge, nodeID string) []WorkflowEdge { + items := make([]WorkflowEdge, 0) + for _, edge := range edges { + if edge.Source == nodeID { + items = append(items, edge) + } + } + return items +} + +func GetStartNodeIDs(nodes []WorkflowNode, edges []WorkflowEdge) []string { + items := make([]string, 0) + for _, node := range nodes { + hasIncoming := false + for _, edge := range edges { + if edge.Target == node.ID { + hasIncoming = true + break + } + } + if !hasIncoming { + items = append(items, node.ID) + } + } + return items +} + +func auditDecisionMatchesHandle(handle string, decision AuditStatus) bool { + if handle == "" { + return decision == AuditStatusApproved + } + normalized := strings.ToLower(handle) + switch decision { + case AuditStatusApproved: + return strings.Contains(normalized, "approved") || strings.Contains(normalized, "通过") + case AuditStatusRejected: + return strings.Contains(normalized, "rejected") || strings.Contains(normalized, "拒绝") + default: + return false + } +} + +func GetDecisionOutgoingEdges(edges []WorkflowEdge, nodeID string, decision AuditStatus) []WorkflowEdge { + outgoing := GetOutgoingEdges(edges, nodeID) + filtered := make([]WorkflowEdge, 0) + for _, edge := range outgoing { + if auditDecisionMatchesHandle(edge.SourceHandle, decision) { + filtered = append(filtered, edge) + } + } + if len(filtered) > 0 { + return filtered + } + return outgoing +} + +func NodeRequiresAudit(node WorkflowNode) bool { + label := strings.ToLower(node.Data.Label) + return node.Data.Category == "HUMAN" || strings.Contains(label, "human review") || strings.Contains(node.Data.Label, "人工审核") +} diff --git a/internal/k8s/launcher.go b/internal/k8s/launcher.go new file mode 100644 index 00000000..1a95089f --- /dev/null +++ b/internal/k8s/launcher.go @@ -0,0 +1,230 @@ +package k8s + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +type LaunchInput struct { + Image string + RunID string + WorkflowID string + NodeID string + NodeLabel string + TimeoutSeconds int + Env map[string]string +} + +type JobReference struct { + JobName string + Namespace string +} + +type LaunchResult struct { + Reference *JobReference + Output string +} + +type Launcher struct { + enabled bool + namespace string + imagePullSecretName string + clientset kubernetes.Interface +} + +var invalidNameCharacters = regexp.MustCompile(`[^a-z0-9-]+`) + +func sanitizeName(value string) string { + normalized := strings.ToLower(value) + normalized = invalidNameCharacters.ReplaceAllString(normalized, "-") + normalized = strings.Trim(normalized, "-") + normalized = regexp.MustCompile(`-+`).ReplaceAllString(normalized, "-") + if len(normalized) > 50 { + return normalized[:50] + } + return normalized +} + +func CreateJobManifest(namespace string, imagePullSecretName string, input LaunchInput) *batchv1.Job { + jobPrefix := sanitizeName(input.RunID + "-" + input.NodeID) + if len(jobPrefix) > 52 { + jobPrefix = jobPrefix[:52] + } + if jobPrefix == "" { + jobPrefix = "droboticflow-node" + } + envVars := []corev1.EnvVar{ + {Name: "RUN_ID", Value: input.RunID}, + {Name: "WORKFLOW_ID", Value: input.WorkflowID}, + {Name: "NODE_ID", Value: input.NodeID}, + {Name: "NODE_LABEL", Value: input.NodeLabel}, + } + for key, value := range input.Env { + envVars = append(envVars, corev1.EnvVar{Name: key, Value: value}) + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: jobPrefix + "-", + Namespace: namespace, + Labels: map[string]string{ + "app": "droboticflow", + "component": "operator-runner", + "runId": input.RunID, + "workflowId": input.WorkflowID, + "nodeId": input.NodeID, + }, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: func() *int32 { value := int32(0); return &value }(), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "droboticflow", + "component": "operator-runner", + "runId": input.RunID, + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "operator-runner", + Image: input.Image, + ImagePullPolicy: corev1.PullAlways, + Env: envVars, + }, + }, + }, + }, + }, + } + if input.TimeoutSeconds > 0 { + job.Spec.ActiveDeadlineSeconds = func() *int64 { + value := int64(input.TimeoutSeconds) + return &value + }() + } + if imagePullSecretName != "" { + job.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: imagePullSecretName}} + } + return job +} + +func loadRestConfig() (*rest.Config, error) { + if config, err := rest.InClusterConfig(); err == nil { + return config, nil + } + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + kubeconfig = filepath.Join(home, ".kube", "config") + } + return clientcmd.BuildConfigFromFlags("", kubeconfig) +} + +func NewLauncher(enabled bool, namespace string, imagePullSecretName string) (*Launcher, error) { + launcher := &Launcher{ + enabled: enabled, + namespace: namespace, + imagePullSecretName: imagePullSecretName, + } + if !enabled { + return launcher, nil + } + config, err := loadRestConfig() + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + launcher.clientset = clientset + return launcher, nil +} + +func (launcher *Launcher) Enabled() bool { + return launcher.enabled +} + +func (launcher *Launcher) readJobOutput(ctx context.Context, jobName string) (string, error) { + var podName string + for attempt := 0; attempt < 30; attempt++ { + pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "job-name=" + jobName, + }) + if err != nil { + return "", err + } + if len(pods.Items) > 0 { + podName = pods.Items[0].Name + break + } + time.Sleep(time.Second) + } + if podName == "" { + return "", fmt.Errorf("no pod found for job %s", jobName) + } + stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, &corev1.PodLogOptions{}).Stream(ctx) + if err != nil { + return "", err + } + defer stream.Close() + output, err := io.ReadAll(stream) + if err != nil { + return "", err + } + return string(output), nil +} + +func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*LaunchResult, error) { + if !launcher.enabled { + return nil, nil + } + job := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, input) + created, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Create(ctx, job, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + jobName := created.Name + reference := &JobReference{JobName: jobName, Namespace: launcher.namespace} + for attempt := 0; attempt < 300; attempt++ { + status, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Get(ctx, jobName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + if status.Status.Succeeded > 0 { + output, err := launcher.readJobOutput(ctx, jobName) + if err != nil { + return nil, err + } + return &LaunchResult{Reference: reference, Output: output}, nil + } + if status.Status.Failed > 0 { + output, outputErr := launcher.readJobOutput(ctx, jobName) + if outputErr != nil { + return nil, fmt.Errorf("kubernetes job %s failed", jobName) + } + return nil, fmt.Errorf("kubernetes job %s failed: %s", jobName, strings.TrimSpace(output)) + } + time.Sleep(time.Second) + } + return nil, fmt.Errorf("timed out waiting for kubernetes job %s", jobName) +} diff --git a/internal/k8s/launcher_test.go b/internal/k8s/launcher_test.go new file mode 100644 index 00000000..863bf19c --- /dev/null +++ b/internal/k8s/launcher_test.go @@ -0,0 +1,36 @@ +package k8s + +import "testing" + +func TestCreateJobManifest(t *testing.T) { + manifest := CreateJobManifest("droboticflow", "droboticflow-registry", LaunchInput{ + Image: "ccr.example.com/dataflow/operator-runner:latest", + RunID: "run-1", + WorkflowID: "wf-1", + NodeID: "node-1", + NodeLabel: "模型推理", + Env: map[string]string{ + "CPU_LIMIT": "500m", + "MEMORY_LIMIT": "1Gi", + }, + }) + + if manifest.GenerateName == "" || manifest.GenerateName[:5] != "run-1" { + t.Fatalf("expected manifest generateName to contain run id, got %s", manifest.GenerateName) + } + if manifest.Spec.Template.Spec.Containers[0].Image != "ccr.example.com/dataflow/operator-runner:latest" { + t.Fatalf("unexpected image: %s", manifest.Spec.Template.Spec.Containers[0].Image) + } + if len(manifest.Spec.Template.Spec.ImagePullSecrets) == 0 || manifest.Spec.Template.Spec.ImagePullSecrets[0].Name != "droboticflow-registry" { + t.Fatalf("expected image pull secret") + } + foundNodeLabel := false + for _, item := range manifest.Spec.Template.Spec.Containers[0].Env { + if item.Name == "NODE_LABEL" && item.Value == "模型推理" { + foundNodeLabel = true + } + } + if !foundNodeLabel { + t.Fatalf("expected NODE_LABEL env var") + } +} diff --git a/internal/retry/retry.go b/internal/retry/retry.go new file mode 100644 index 00000000..d5855b26 --- /dev/null +++ b/internal/retry/retry.go @@ -0,0 +1,39 @@ +package retry + +import ( + "context" + "fmt" + "time" +) + +func Do[T any]( + ctx context.Context, + attempts int, + delay time.Duration, + label string, + run func(context.Context, int) (T, error), +) (T, error) { + var zero T + var lastErr error + + for attempt := 1; attempt <= attempts; attempt++ { + result, err := run(ctx, attempt) + if err == nil { + return result, nil + } + lastErr = err + if attempt == attempts { + break + } + + fmt.Printf("[startup] %s attempt %d/%d failed, retrying in %s\n", label, attempt, attempts, delay) + + select { + case <-ctx.Done(): + return zero, ctx.Err() + case <-time.After(delay): + } + } + + return zero, lastErr +} diff --git a/internal/retry/retry_test.go b/internal/retry/retry_test.go new file mode 100644 index 00000000..46c50604 --- /dev/null +++ b/internal/retry/retry_test.go @@ -0,0 +1,42 @@ +package retry + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestRetryDoRetriesUntilSuccess(t *testing.T) { + attempts := 0 + result, err := Do(context.Background(), 3, 0, "test operation", func(_ context.Context, _ int) (string, error) { + attempts++ + if attempts < 3 { + return "", errors.New("not yet") + } + return "ok", nil + }) + if err != nil { + t.Fatalf("retry returned error: %v", err) + } + if result != "ok" { + t.Fatalf("expected ok, got %s", result) + } + if attempts != 3 { + t.Fatalf("expected 3 attempts, got %d", attempts) + } +} + +func TestRetryDoReturnsLastError(t *testing.T) { + attempts := 0 + _, err := Do(context.Background(), 2, time.Millisecond, "test failure", func(_ context.Context, _ int) (string, error) { + attempts++ + return "", errors.New("boom") + }) + if err == nil { + t.Fatalf("expected error") + } + if attempts != 2 { + t.Fatalf("expected 2 attempts, got %d", attempts) + } +} diff --git a/internal/store/errors.go b/internal/store/errors.go new file mode 100644 index 00000000..252c2310 --- /dev/null +++ b/internal/store/errors.go @@ -0,0 +1,5 @@ +package store + +import "errors" + +var ErrWorkflowAlreadyCompleted = errors.New("workflow has completed successfully and cannot be run again manually") diff --git a/internal/store/helpers.go b/internal/store/helpers.go new file mode 100644 index 00000000..a2543f01 --- /dev/null +++ b/internal/store/helpers.go @@ -0,0 +1,265 @@ +package store + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "droboticflow/internal/domain" +) + +func nowISO() string { + return time.Now().UTC().Format(time.RFC3339Nano) +} + +func computeRunDuration(run *domain.RunRecord) { + if run.StartedAt == "" || run.EndedAt == "" { + return + } + startedAt, err := time.Parse(time.RFC3339Nano, run.StartedAt) + if err != nil { + return + } + endedAt, err := time.Parse(time.RFC3339Nano, run.EndedAt) + if err != nil { + return + } + run.DurationMS = endedAt.Sub(startedAt).Milliseconds() +} + +func parseISOTime(value string) (time.Time, bool) { + if value == "" { + return time.Time{}, false + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, false + } + return parsed, true +} + +func runAvailableAt(run *domain.RunRecord) time.Time { + if run == nil { + return time.Time{} + } + if parsed, ok := parseISOTime(run.AvailableAt); ok { + return parsed + } + if parsed, ok := parseISOTime(run.CreatedAt); ok { + return parsed + } + return time.Time{} +} + +func runIsAvailable(run *domain.RunRecord, now time.Time) bool { + availableAt := runAvailableAt(run) + if availableAt.IsZero() { + return true + } + return !availableAt.After(now) +} + +func clone[T any](value T) T { + data, _ := json.Marshal(value) + var out T + _ = json.Unmarshal(data, &out) + return out +} + +func defaultSettings() domain.SettingsRecord { + return domain.SettingsRecord{ + ID: "system", + K8s: domain.K8sSettings{ + Enabled: false, + Namespace: "droboticflow", + }, + Worker: domain.WorkerSettings{ + PollIntervalMS: 2000, + MaxConcurrentRuns: 1, + }, + Storage: domain.StorageSettings{ + Endpoint: "", + Buckets: []string{}, + }, + AI: domain.AISettings{ + Provider: "dashscope-openai-compatible", + BaseURL: "https://coding.dashscope.aliyuncs.com/v1", + Model: "glm-5", + }, + Timezone: "Asia/Shanghai", + Defaults: domain.DefaultResources{ + CPURequest: "500m", + CPULimit: "1000m", + MemoryRequest: "1Gi", + MemoryLimit: "2Gi", + }, + UpdatedAt: nowISO(), + } +} + +func ensureSettingsDefaults(item domain.SettingsRecord) domain.SettingsRecord { + defaults := defaultSettings() + if item.ID == "" { + item.ID = defaults.ID + } + if item.K8s.Namespace == "" { + item.K8s.Namespace = defaults.K8s.Namespace + } + if item.Worker.PollIntervalMS <= 0 { + item.Worker.PollIntervalMS = defaults.Worker.PollIntervalMS + } + if item.Worker.MaxConcurrentRuns <= 0 { + item.Worker.MaxConcurrentRuns = defaults.Worker.MaxConcurrentRuns + } + if item.AI.Provider == "" { + item.AI.Provider = defaults.AI.Provider + } + if item.AI.BaseURL == "" { + item.AI.BaseURL = defaults.AI.BaseURL + } + if item.AI.Model == "" { + item.AI.Model = defaults.AI.Model + } + if item.Timezone == "" { + item.Timezone = defaults.Timezone + } + if item.Defaults.CPURequest == "" { + item.Defaults.CPURequest = defaults.Defaults.CPURequest + } + if item.Defaults.CPULimit == "" { + item.Defaults.CPULimit = defaults.Defaults.CPULimit + } + if item.Defaults.MemoryRequest == "" { + item.Defaults.MemoryRequest = defaults.Defaults.MemoryRequest + } + if item.Defaults.MemoryLimit == "" { + item.Defaults.MemoryLimit = defaults.Defaults.MemoryLimit + } + if item.UpdatedAt == "" { + item.UpdatedAt = nowISO() + } + return item +} + +func sortByCreatedAtDescending[T interface{ GetCreatedAt() string }](items []T) { + sort.Slice(items, func(left int, right int) bool { + return items[left].GetCreatedAt() > items[right].GetCreatedAt() + }) +} + +func validateWorkflowProject(project *domain.ProjectRecord, workflowID string) error { + if project == nil { + return fmt.Errorf("project for workflow %s not found", workflowID) + } + return nil +} + +func storageRoot(sourcePath string) string { + trimmed := strings.TrimSpace(sourcePath) + if trimmed == "" { + return "" + } + if strings.HasPrefix(trimmed, "file://") { + return strings.TrimPrefix(trimmed, "file://") + } + if strings.Contains(trimmed, "://") { + return "" + } + return filepath.Clean(trimmed) +} + +func normalizeStorageBucket(bucket string) (string, bool) { + trimmed := strings.Trim(strings.TrimSpace(bucket), "/\\") + if trimmed == "" { + return "", false + } + if strings.ContainsAny(trimmed, "/\\") || trimmed == "." || trimmed == ".." { + return "", false + } + return trimmed, true +} + +func normalizeStoragePrefix(prefix string) (string, string, bool) { + trimmed := strings.Trim(strings.TrimSpace(prefix), "/\\") + if trimmed == "" { + return "", "", true + } + parts := strings.Split(strings.ReplaceAll(trimmed, "\\", "/"), "/") + cleanParts := make([]string, 0, len(parts)) + for _, part := range parts { + switch part { + case "", ".": + continue + case "..": + return "", "", false + default: + cleanParts = append(cleanParts, part) + } + } + if len(cleanParts) == 0 { + return "", "", true + } + return strings.Join(cleanParts, "/"), filepath.Join(cleanParts...), true +} + +func listBucketsFromSourcePath(sourcePath string) ([]string, error) { + root := storageRoot(sourcePath) + if root == "" { + return []string{}, nil + } + entries, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return []string{}, nil + } + return nil, err + } + items := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + items = append(items, entry.Name()) + } + } + sort.Strings(items) + return items, nil +} + +func listObjectsFromSourcePath(sourcePath string, bucket string, prefix string) ([]string, error) { + root := storageRoot(sourcePath) + bucketName, ok := normalizeStorageBucket(bucket) + if root == "" || !ok { + return []string{}, nil + } + normalizedPrefix, localPrefix, ok := normalizeStoragePrefix(prefix) + if !ok { + return []string{}, nil + } + basePath := filepath.Join(root, bucketName) + if localPrefix != "" { + basePath = filepath.Join(basePath, localPrefix) + } + entries, err := os.ReadDir(basePath) + if err != nil { + if os.IsNotExist(err) { + return []string{}, nil + } + return nil, err + } + if normalizedPrefix != "" && !strings.HasSuffix(normalizedPrefix, "/") { + normalizedPrefix += "/" + } + items := make([]string, 0, len(entries)) + for _, entry := range entries { + name := normalizedPrefix + entry.Name() + if entry.IsDir() { + name += "/" + } + items = append(items, name) + } + sort.Strings(items) + return items, nil +} diff --git a/internal/store/memory.go b/internal/store/memory.go new file mode 100644 index 00000000..41ad9cd5 --- /dev/null +++ b/internal/store/memory.go @@ -0,0 +1,766 @@ +package store + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "droboticflow/internal/domain" +) + +type MemoryStore struct { + mu sync.Mutex + projects map[string]domain.ProjectRecord + operators map[string]domain.OperatorRecord + workflows map[string]domain.WorkflowRecord + templates map[string]domain.TemplateRecord + runs map[string]domain.RunRecord + auditTasks map[string]domain.AuditTaskRecord + users map[string]domain.UserRecord + settings *domain.SettingsRecord +} + +func NewMemoryStore() *MemoryStore { + return &MemoryStore{ + projects: map[string]domain.ProjectRecord{}, + operators: map[string]domain.OperatorRecord{}, + workflows: map[string]domain.WorkflowRecord{}, + templates: map[string]domain.TemplateRecord{}, + runs: map[string]domain.RunRecord{}, + auditTasks: map[string]domain.AuditTaskRecord{}, + users: map[string]domain.UserRecord{}, + } +} + +func sortProjects(items []domain.ProjectRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + +func sortOperators(items []domain.OperatorRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + +func sortWorkflows(items []domain.WorkflowRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + +func sortTemplates(items []domain.TemplateRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + +func sortRuns(items []domain.RunRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + +func sortAuditTasks(items []domain.AuditTaskRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) +} + +func sortUsers(items []domain.UserRecord) { + sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt < items[right].CreatedAt }) +} + +func (store *MemoryStore) ListProjects(context.Context) ([]domain.ProjectRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.ProjectRecord, 0, len(store.projects)) + for _, project := range store.projects { + items = append(items, clone(project)) + } + sortProjects(items) + return items, nil +} + +func (store *MemoryStore) GetProject(_ context.Context, id string) (*domain.ProjectRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + project, ok := store.projects[id] + if !ok { + return nil, nil + } + cloned := clone(project) + return &cloned, nil +} + +func (store *MemoryStore) CreateProject(_ context.Context, input domain.CreateProjectInput) (*domain.ProjectRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + now := nowISO() + project := domain.ProjectRecord{ + ID: domain.NewID("project"), + Name: input.Name, + Description: input.Description, + Status: "active", + CreatedAt: now, + UpdatedAt: now, + } + store.projects[project.ID] = clone(project) + cloned := clone(project) + return &cloned, nil +} + +func (store *MemoryStore) DeleteProject(_ context.Context, id string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + if _, ok := store.projects[id]; !ok { + return false, nil + } + delete(store.projects, id) + + workflowIDs := make([]string, 0) + for workflowID, workflow := range store.workflows { + if workflow.ProjectID == id { + workflowIDs = append(workflowIDs, workflowID) + delete(store.workflows, workflowID) + } + } + for runID, run := range store.runs { + if run.ProjectID == id { + delete(store.runs, runID) + continue + } + for _, workflowID := range workflowIDs { + if run.WorkflowID == workflowID { + delete(store.runs, runID) + break + } + } + } + for taskID, task := range store.auditTasks { + if task.ProjectID == id { + delete(store.auditTasks, taskID) + continue + } + for _, workflowID := range workflowIDs { + if task.WorkflowID == workflowID { + delete(store.auditTasks, taskID) + break + } + } + } + return true, nil +} + +func (store *MemoryStore) UpsertProject(_ context.Context, project domain.ProjectRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.projects[project.ID] = clone(project) + return nil +} + +func (store *MemoryStore) ListOperators(context.Context) ([]domain.OperatorRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.OperatorRecord, 0, len(store.operators)) + for _, item := range store.operators { + items = append(items, clone(item)) + } + sortOperators(items) + return items, nil +} + +func (store *MemoryStore) GetOperator(_ context.Context, id string) (*domain.OperatorRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.operators[id] + if !ok { + return nil, nil + } + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) CreateOperator(_ context.Context, input domain.CreateOperatorInput) (*domain.OperatorRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + now := nowISO() + item := domain.OperatorRecord{ + ID: domain.NewID("operator"), + Name: input.Name, + Type: input.Type, + Category: input.Category, + Description: input.Description, + Inputs: clone(input.Inputs), + Outputs: clone(input.Outputs), + ConfigSchema: clone(input.ConfigSchema), + Image: input.Image, + SkillDefinition: input.SkillDefinition, + Hyperparams: clone(input.Hyperparams), + Resources: clone(input.Resources), + CreatedAt: now, + UpdatedAt: now, + } + store.operators[item.ID] = clone(item) + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) DeleteOperator(_ context.Context, id string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + if _, ok := store.operators[id]; !ok { + return false, nil + } + delete(store.operators, id) + return true, nil +} + +func (store *MemoryStore) UpsertOperator(_ context.Context, item domain.OperatorRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.operators[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) ListWorkflowsByProject(_ context.Context, projectID string) ([]domain.WorkflowRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.WorkflowRecord, 0) + for _, workflow := range store.workflows { + if workflow.ProjectID == projectID { + items = append(items, clone(workflow)) + } + } + sortWorkflows(items) + return items, nil +} + +func (store *MemoryStore) ListAllWorkflows(context.Context) ([]domain.WorkflowRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.WorkflowRecord, 0, len(store.workflows)) + for _, workflow := range store.workflows { + items = append(items, clone(workflow)) + } + sortWorkflows(items) + return items, nil +} + +func (store *MemoryStore) GetWorkflow(_ context.Context, id string) (*domain.WorkflowRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + workflow, ok := store.workflows[id] + if !ok { + return nil, nil + } + cloned := clone(workflow) + return &cloned, nil +} + +func (store *MemoryStore) CreateWorkflow(_ context.Context, input domain.CreateWorkflowInput) (*domain.WorkflowRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + now := nowISO() + item := domain.WorkflowRecord{ + ID: domain.NewID("workflow"), + ProjectID: input.ProjectID, + Name: input.Name, + Description: input.Description, + Type: input.Type, + MaxConcurrentRuns: input.MaxConcurrentRuns, + NodeTimeoutSeconds: input.NodeTimeoutSeconds, + RunTimeoutSeconds: input.RunTimeoutSeconds, + MaxRetryAttempts: input.MaxRetryAttempts, + RetryBackoffSeconds: input.RetryBackoffSeconds, + Schedule: clone(input.Schedule), + Nodes: clone(input.Nodes), + Edges: clone(input.Edges), + Status: domain.WorkflowStatusDraft, + CreatedAt: now, + UpdatedAt: now, + } + store.workflows[item.ID] = clone(item) + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) DeleteWorkflow(_ context.Context, id string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + if _, ok := store.workflows[id]; !ok { + return false, nil + } + delete(store.workflows, id) + for runID, run := range store.runs { + if run.WorkflowID == id { + delete(store.runs, runID) + } + } + for taskID, task := range store.auditTasks { + if task.WorkflowID == id { + delete(store.auditTasks, taskID) + } + } + return true, nil +} + +func (store *MemoryStore) UpsertWorkflow(_ context.Context, workflow domain.WorkflowRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.workflows[workflow.ID] = clone(workflow) + return nil +} + +func (store *MemoryStore) ListTemplates(context.Context) ([]domain.TemplateRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.TemplateRecord, 0, len(store.templates)) + for _, item := range store.templates { + items = append(items, clone(item)) + } + sortTemplates(items) + return items, nil +} + +func (store *MemoryStore) GetTemplate(_ context.Context, id string) (*domain.TemplateRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.templates[id] + if !ok { + return nil, nil + } + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) CreateTemplate(_ context.Context, input domain.CreateTemplateInput) (*domain.TemplateRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + now := nowISO() + templateType := input.Type + if templateType == "" { + templateType = domain.WorkflowTypeBatch + } + item := domain.TemplateRecord{ + ID: domain.NewID("template"), + Name: input.Name, + Description: input.Description, + Type: templateType, + MaxConcurrentRuns: input.MaxConcurrentRuns, + NodeTimeoutSeconds: input.NodeTimeoutSeconds, + RunTimeoutSeconds: input.RunTimeoutSeconds, + MaxRetryAttempts: input.MaxRetryAttempts, + RetryBackoffSeconds: input.RetryBackoffSeconds, + Schedule: clone(input.Schedule), + Nodes: clone(input.Nodes), + Edges: clone(input.Edges), + CreatedAt: now, + UpdatedAt: now, + } + store.templates[item.ID] = clone(item) + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) DeleteTemplate(_ context.Context, id string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + if _, ok := store.templates[id]; !ok { + return false, nil + } + delete(store.templates, id) + return true, nil +} + +func (store *MemoryStore) UpsertTemplate(_ context.Context, item domain.TemplateRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.templates[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) ListRuns(_ context.Context, status string) ([]domain.RunRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.RunRecord, 0) + for _, run := range store.runs { + if status == "" || string(run.Status) == status { + items = append(items, clone(run)) + } + } + sortRuns(items) + return items, nil +} + +func (store *MemoryStore) GetRun(_ context.Context, id string) (*domain.RunRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + run, ok := store.runs[id] + if !ok { + return nil, nil + } + cloned := clone(run) + return &cloned, nil +} + +func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigger domain.RunTrigger) (*domain.RunRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + workflow, ok := store.workflows[workflowID] + if !ok { + return nil, fmt.Errorf("workflow %s not found", workflowID) + } + if workflow.Status == domain.WorkflowStatusCompleted && trigger != domain.RunTriggerScheduled { + return nil, ErrWorkflowAlreadyCompleted + } + project, ok := store.projects[workflow.ProjectID] + if !ok { + return nil, fmt.Errorf("project %s not found", workflow.ProjectID) + } + + now := nowISO() + run := domain.RunRecord{ + ID: domain.NewID("run"), + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: trigger, + Attempt: 1, + AvailableAt: now, + Status: domain.RunStatusQueued, + NodeExecutions: []domain.NodeExecutionRecord{}, + Logs: []domain.LogEntryRecord{}, + ResourceSeries: []domain.ResourceSample{}, + Artifacts: []domain.ExecutionArtifact{}, + K8sRefs: []domain.K8sRef{}, + CreatedAt: now, + UpdatedAt: now, + } + store.runs[run.ID] = clone(run) + + workflow.LastRunID = run.ID + workflow.LastRunAt = now + workflow.Status = domain.WorkflowStatusRunning + workflow.UpdatedAt = now + store.workflows[workflow.ID] = workflow + + cloned := clone(run) + return &cloned, nil +} + +func (store *MemoryStore) ClaimNextQueuedRun(_ context.Context) (*domain.RunRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + var selected *domain.RunRecord + now := time.Now().UTC() + for _, run := range store.runs { + if run.Status != domain.RunStatusQueued { + continue + } + candidate := clone(run) + if !runIsAvailable(&candidate, now) { + continue + } + if selected == nil || candidate.CreatedAt < selected.CreatedAt { + selected = &candidate + } + } + if selected == nil { + return nil, nil + } + selected.Status = domain.RunStatusRunning + selected.StartedAt = nowISO() + selected.UpdatedAt = nowISO() + computeRunDuration(selected) + store.runs[selected.ID] = clone(*selected) + cloned := clone(*selected) + return &cloned, nil +} + +func (store *MemoryStore) ClaimQueuedRun(_ context.Context, runID string) (*domain.RunRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + run, ok := store.runs[runID] + if !ok || run.Status != domain.RunStatusQueued { + return nil, nil + } + if !runIsAvailable(&run, time.Now().UTC()) { + return nil, nil + } + run.Status = domain.RunStatusRunning + run.StartedAt = nowISO() + run.UpdatedAt = nowISO() + computeRunDuration(&run) + store.runs[run.ID] = clone(run) + cloned := clone(run) + return &cloned, nil +} + +func (store *MemoryStore) UpsertRun(_ context.Context, run domain.RunRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + computeRunDuration(&run) + store.runs[run.ID] = clone(run) + return nil +} + +func (store *MemoryStore) ListAuditTasks(_ context.Context, status string) ([]domain.AuditTaskRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.AuditTaskRecord, 0) + for _, task := range store.auditTasks { + if status == "" || status == "all" || string(task.Status) == status { + items = append(items, clone(task)) + } + } + sortAuditTasks(items) + return items, nil +} + +func (store *MemoryStore) GetAuditTask(_ context.Context, id string) (*domain.AuditTaskRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + task, ok := store.auditTasks[id] + if !ok { + return nil, nil + } + cloned := clone(task) + return &cloned, nil +} + +func (store *MemoryStore) CreateAuditTask(_ context.Context, task domain.AuditTaskRecord) (*domain.AuditTaskRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + store.auditTasks[task.ID] = clone(task) + cloned := clone(task) + return &cloned, nil +} + +func (store *MemoryStore) UpsertAuditTask(_ context.Context, task domain.AuditTaskRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.auditTasks[task.ID] = clone(task) + return nil +} + +func (store *MemoryStore) ListUsers(context.Context) ([]domain.UserRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.UserRecord, 0, len(store.users)) + for _, item := range store.users { + items = append(items, clone(item)) + } + sortUsers(items) + return items, nil +} + +func (store *MemoryStore) GetUser(_ context.Context, id string) (*domain.UserRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.users[id] + if !ok { + return nil, nil + } + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) CreateUser(_ context.Context, input domain.CreateUserInput) (*domain.UserRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + now := nowISO() + user := domain.UserRecord{ + ID: domain.NewID("user"), + Name: input.Name, + Email: input.Email, + Role: input.Role, + CreatedAt: now, + UpdatedAt: now, + } + store.users[user.ID] = clone(user) + cloned := clone(user) + return &cloned, nil +} + +func (store *MemoryStore) DeleteUser(_ context.Context, id string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + if _, ok := store.users[id]; !ok { + return false, nil + } + delete(store.users, id) + return true, nil +} + +func (store *MemoryStore) UpsertUser(_ context.Context, item domain.UserRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.users[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) GetSettings(_ context.Context) (*domain.SettingsRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + if store.settings == nil { + defaultValue := defaultSettings() + store.settings = &defaultValue + } + cloned := ensureSettingsDefaults(clone(*store.settings)) + return &cloned, nil +} + +func (store *MemoryStore) UpsertSettings(_ context.Context, item domain.SettingsRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + cloned := ensureSettingsDefaults(clone(item)) + store.settings = &cloned + return nil +} + +func (store *MemoryStore) GetDashboardSummary(ctx context.Context) (domain.DashboardSummary, error) { + projects, err := store.ListProjects(ctx) + if err != nil { + return domain.DashboardSummary{}, err + } + operators, err := store.ListOperators(ctx) + if err != nil { + return domain.DashboardSummary{}, err + } + runs, err := store.ListRuns(ctx, "") + if err != nil { + return domain.DashboardSummary{}, err + } + auditTasks, err := store.ListAuditTasks(ctx, "all") + if err != nil { + return domain.DashboardSummary{}, err + } + + return buildDashboardSummary(projects, operators, runs, auditTasks), nil +} + +func buildDashboardSummary( + projects []domain.ProjectRecord, + operators []domain.OperatorRecord, + runs []domain.RunRecord, + auditTasks []domain.AuditTaskRecord, +) domain.DashboardSummary { + completedRuns := make([]domain.RunRecord, 0) + failedRuns := make([]domain.RunRecord, 0) + runningRuns := make([]domain.RunRecord, 0) + for _, run := range runs { + switch run.Status { + case domain.RunStatusCompleted: + completedRuns = append(completedRuns, run) + case domain.RunStatusFailed: + failedRuns = append(failedRuns, run) + case domain.RunStatusRunning: + runningRuns = append(runningRuns, run) + } + } + + var durations []int64 + for _, run := range completedRuns { + if run.DurationMS > 0 { + durations = append(durations, run.DurationMS) + } + } + var average int64 + if len(durations) > 0 { + var total int64 + for _, duration := range durations { + total += duration + } + average = total / int64(len(durations)) + } + + var summary domain.DashboardSummary + for _, project := range projects { + if project.Status == "active" { + summary.Projects.Active++ + } + if project.Status == "archived" { + summary.Projects.Archived++ + } + } + summary.Operators.Total = len(operators) + for _, item := range operators { + if item.Type == "custom" { + summary.Operators.Custom++ + } + } + totalFinished := len(completedRuns) + len(failedRuns) + summary.Jobs.Today = len(runs) + if totalFinished == 0 { + summary.Jobs.SuccessRate = 100 + } else { + summary.Jobs.SuccessRate = float64(len(completedRuns)) * 100 / float64(totalFinished) + } + summary.Jobs.Running = len(runningRuns) + summary.Jobs.Failed = len(failedRuns) + summary.Jobs.AverageDurationMS = average + for _, task := range auditTasks { + if task.Status == domain.AuditStatusPending { + summary.Audits.Pending++ + } + } + return summary +} + +func (store *MemoryStore) ListStorageBuckets(ctx context.Context, sourcePath string) ([]string, error) { + return listBucketsFromSourcePath(sourcePath) +} + +func (store *MemoryStore) ListStorageObjects(ctx context.Context, sourcePath string, bucket string, prefix string) ([]string, error) { + return listObjectsFromSourcePath(sourcePath, bucket, prefix) +} + +func MergeSettings(current domain.SettingsRecord, patch domain.SettingsRecord) domain.SettingsRecord { + merged := current + if patch.K8s.Namespace != "" { + merged.K8s.Namespace = patch.K8s.Namespace + } + if patch.K8s.Endpoint != "" { + merged.K8s.Endpoint = patch.K8s.Endpoint + } + if patch.K8s.Kubeconfig != "" { + merged.K8s.Kubeconfig = patch.K8s.Kubeconfig + } + if patch.K8s.Enabled != current.K8s.Enabled { + merged.K8s.Enabled = patch.K8s.Enabled + } + if patch.Worker.PollIntervalMS > 0 { + merged.Worker.PollIntervalMS = patch.Worker.PollIntervalMS + } + if patch.Worker.MaxConcurrentRuns > 0 { + merged.Worker.MaxConcurrentRuns = patch.Worker.MaxConcurrentRuns + } + if patch.Storage.Endpoint != "" { + merged.Storage.Endpoint = patch.Storage.Endpoint + } + if len(patch.Storage.Buckets) > 0 { + merged.Storage.Buckets = clone(patch.Storage.Buckets) + } + if patch.Defaults.CPURequest != "" { + merged.Defaults.CPURequest = patch.Defaults.CPURequest + } + if patch.Defaults.CPULimit != "" { + merged.Defaults.CPULimit = patch.Defaults.CPULimit + } + if patch.Defaults.MemoryRequest != "" { + merged.Defaults.MemoryRequest = patch.Defaults.MemoryRequest + } + if patch.Defaults.MemoryLimit != "" { + merged.Defaults.MemoryLimit = patch.Defaults.MemoryLimit + } + if patch.Timezone != "" { + merged.Timezone = patch.Timezone + } + merged.UpdatedAt = nowISO() + return merged +} + +func CopyRunWithUpdate(run domain.RunRecord) domain.RunRecord { + copy := clone(run) + computeRunDuration(©) + copy.UpdatedAt = nowISO() + return copy +} + +func NormalizeStatusFilter(status string) string { + return strings.TrimSpace(status) +} diff --git a/internal/store/memory_test.go b/internal/store/memory_test.go new file mode 100644 index 00000000..41dccda2 --- /dev/null +++ b/internal/store/memory_test.go @@ -0,0 +1,277 @@ +package store_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func TestMemoryStoreCRUD(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + storageRoot := t.TempDir() + for _, bucket := range []string{"incoming", "processed"} { + if err := os.Mkdir(filepath.Join(storageRoot, bucket), 0o755); err != nil { + t.Fatalf("create bucket %s: %v", bucket, err) + } + } + + settings, err := dataStore.GetSettings(ctx) + if err != nil { + t.Fatalf("get settings: %v", err) + } + if err := dataStore.UpsertSettings(ctx, *settings); err != nil { + t.Fatalf("update settings: %v", err) + } + if err := os.MkdirAll(filepath.Join(storageRoot, "incoming", "clips"), 0o755); err != nil { + t.Fatalf("create nested prefix: %v", err) + } + if err := os.WriteFile(filepath.Join(storageRoot, "incoming", "clips", "sample.mp4"), []byte("data"), 0o644); err != nil { + t.Fatalf("write sample object: %v", err) + } + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "新项目", + Description: "用于仓储测试", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + if project.ID == "" { + t.Fatalf("expected created project id") + } + + operator, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ + Name: "真实数据源", + Type: "builtin", + Category: "IO", + Description: "Emit a manifest artifact from source text", + Inputs: []domain.PortDefinition{}, + Outputs: []domain.PortDefinition{ + {ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}, + }, + }) + if err != nil { + t.Fatalf("create operator: %v", err) + } + if operator.ID == "" { + t.Fatalf("expected created operator id") + } + + template, err := dataStore.CreateTemplate(ctx, domain.CreateTemplateInput{ + Name: "基础模板", + Description: "用于模板列表测试", + Nodes: []domain.WorkflowNode{ + { + ID: "node-1", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "真实数据源", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{ + {ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create template: %v", err) + } + if template.ID == "" { + t.Fatalf("expected created template id") + } + + user, err := dataStore.CreateUser(ctx, domain.CreateUserInput{ + Name: "测试用户", + Email: "tester@example.com", + Role: domain.UserRoleAdmin, + }) + if err != nil { + t.Fatalf("create user: %v", err) + } + if user.ID == "" { + t.Fatalf("expected created user id") + } + + projects, err := dataStore.ListProjects(ctx) + if err != nil { + t.Fatalf("list projects: %v", err) + } + if len(projects) != 1 { + t.Fatalf("expected 1 project, got %d", len(projects)) + } + + operators, err := dataStore.ListOperators(ctx) + if err != nil { + t.Fatalf("list operators: %v", err) + } + if len(operators) != 1 { + t.Fatalf("expected 1 operator, got %d", len(operators)) + } + + templates, err := dataStore.ListTemplates(ctx) + if err != nil { + t.Fatalf("list templates: %v", err) + } + if len(templates) != 1 { + t.Fatalf("expected 1 template, got %d", len(templates)) + } + + users, err := dataStore.ListUsers(ctx) + if err != nil { + t.Fatalf("list users: %v", err) + } + if len(users) != 1 { + t.Fatalf("expected 1 user, got %d", len(users)) + } + + buckets, err := dataStore.ListStorageBuckets(ctx, storageRoot) + if err != nil { + t.Fatalf("list storage buckets: %v", err) + } + if len(buckets) != 2 { + t.Fatalf("expected 2 buckets, got %d", len(buckets)) + } + + objects, err := dataStore.ListStorageObjects(ctx, storageRoot, "incoming", "clips") + if err != nil { + t.Fatalf("list storage objects: %v", err) + } + if len(objects) != 1 || objects[0] != "clips/sample.mp4" { + t.Fatalf("unexpected storage objects: %#v", objects) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "测试工作流", + Description: "测试 DAG", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "n1", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "真实数据源", + Category: "IO", + Description: "source", + Inputs: []domain.PortDefinition{}, + Outputs: []domain.PortDefinition{ + {ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + if workflow.ProjectID != project.ID { + t.Fatalf("expected workflow project %s, got %s", project.ID, workflow.ProjectID) + } + + projectWorkflows, err := dataStore.ListWorkflowsByProject(ctx, project.ID) + if err != nil { + t.Fatalf("list project workflows: %v", err) + } + if len(projectWorkflows) != 1 { + t.Fatalf("expected workflow in project listing") + } + + project.Description = "已更新描述" + project.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + if err := dataStore.UpsertProject(ctx, *project); err != nil { + t.Fatalf("update project: %v", err) + } + + updatedProject, err := dataStore.GetProject(ctx, project.ID) + if err != nil { + t.Fatalf("get updated project: %v", err) + } + if updatedProject == nil || updatedProject.Description != "已更新描述" { + t.Fatalf("expected updated description") + } + + deleted, err := dataStore.DeleteProject(ctx, project.ID) + if err != nil { + t.Fatalf("delete project: %v", err) + } + if !deleted { + t.Fatalf("expected project deleted") + } + + deletedProject, err := dataStore.GetProject(ctx, project.ID) + if err != nil { + t.Fatalf("get deleted project: %v", err) + } + if deletedProject != nil { + t.Fatalf("expected deleted project to be nil") + } +} + +func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ + Name: "只读项目", + Description: "completed workflow guard", + }) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "已完成工作流", + Description: "completed", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "node-1", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "真实数据源", + Category: "IO", + Description: "source", + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + workflow.Status = domain.WorkflowStatusCompleted + if err := dataStore.UpsertWorkflow(ctx, *workflow); err != nil { + t.Fatalf("mark workflow completed: %v", err) + } + + if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err == nil { + t.Fatalf("expected manual rerun to be rejected") + } else if !errors.Is(err, store.ErrWorkflowAlreadyCompleted) { + t.Fatalf("expected ErrWorkflowAlreadyCompleted, got %v", err) + } + + scheduledRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled) + if err != nil { + t.Fatalf("expected scheduled run to remain allowed, got %v", err) + } + if scheduledRun == nil || scheduledRun.Status != domain.RunStatusQueued { + t.Fatalf("expected scheduled run queued, got %#v", scheduledRun) + } +} diff --git a/internal/store/mongo.go b/internal/store/mongo.go new file mode 100644 index 00000000..a00a8f5f --- /dev/null +++ b/internal/store/mongo.go @@ -0,0 +1,606 @@ +package store + +import ( + "context" + "fmt" + "sort" + "strings" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type MongoStore struct { + db *mongo.Database + projects *mongo.Collection + operators *mongo.Collection + workflows *mongo.Collection + templates *mongo.Collection + runs *mongo.Collection + auditTasks *mongo.Collection + users *mongo.Collection + settings *mongo.Collection +} + +func NewMongoStore(db *mongo.Database) *MongoStore { + return &MongoStore{ + db: db, + projects: db.Collection("projects"), + operators: db.Collection("operators"), + workflows: db.Collection("workflows"), + templates: db.Collection("templates"), + runs: db.Collection("runs"), + auditTasks: db.Collection("auditTasks"), + users: db.Collection("users"), + settings: db.Collection("settings"), + } +} + +func decodeMany[T any](ctx context.Context, cursor *mongo.Cursor) ([]T, error) { + defer cursor.Close(ctx) + items := make([]T, 0) + for cursor.Next(ctx) { + var item T + if err := cursor.Decode(&item); err != nil { + return nil, err + } + items = append(items, item) + } + return items, cursor.Err() +} + +func (store *MongoStore) ListProjects(ctx context.Context) ([]domain.ProjectRecord, error) { + cursor, err := store.projects.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.ProjectRecord](ctx, cursor) +} + +func (store *MongoStore) GetProject(ctx context.Context, id string) (*domain.ProjectRecord, error) { + var item domain.ProjectRecord + err := store.projects.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateProject(ctx context.Context, input domain.CreateProjectInput) (*domain.ProjectRecord, error) { + now := nowISO() + item := domain.ProjectRecord{ + ID: domain.NewID("project"), + Name: input.Name, + Description: input.Description, + Status: "active", + CreatedAt: now, + UpdatedAt: now, + } + if err := store.UpsertProject(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, error) { + workflowCursor, err := store.workflows.Find(ctx, bson.M{"projectId": id}, options.Find().SetProjection(bson.M{"id": 1})) + if err != nil { + return false, err + } + workflowDocs, err := decodeMany[struct { + ID string `bson:"id"` + }](ctx, workflowCursor) + if err != nil { + return false, err + } + workflowIDs := make([]string, 0, len(workflowDocs)) + for _, item := range workflowDocs { + workflowIDs = append(workflowIDs, item.ID) + } + + if _, err := store.workflows.DeleteMany(ctx, bson.M{"projectId": id}); err != nil { + return false, err + } + if _, err := store.runs.DeleteMany(ctx, bson.M{"projectId": id}); err != nil { + return false, err + } + if _, err := store.auditTasks.DeleteMany(ctx, bson.M{"projectId": id}); err != nil { + return false, err + } + if len(workflowIDs) > 0 { + if _, err := store.runs.DeleteMany(ctx, bson.M{"workflowId": bson.M{"$in": workflowIDs}}); err != nil { + return false, err + } + } + result, err := store.projects.DeleteOne(ctx, bson.M{"id": id}) + if err != nil { + return false, err + } + return result.DeletedCount == 1, nil +} + +func (store *MongoStore) UpsertProject(ctx context.Context, item domain.ProjectRecord) error { + _, err := store.projects.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) ListOperators(ctx context.Context) ([]domain.OperatorRecord, error) { + cursor, err := store.operators.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.OperatorRecord](ctx, cursor) +} + +func (store *MongoStore) GetOperator(ctx context.Context, id string) (*domain.OperatorRecord, error) { + var item domain.OperatorRecord + err := store.operators.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateOperator(ctx context.Context, input domain.CreateOperatorInput) (*domain.OperatorRecord, error) { + now := nowISO() + item := domain.OperatorRecord{ + ID: domain.NewID("operator"), + Name: input.Name, + Type: input.Type, + Category: input.Category, + Description: input.Description, + Inputs: input.Inputs, + Outputs: input.Outputs, + ConfigSchema: input.ConfigSchema, + Image: input.Image, + SkillDefinition: input.SkillDefinition, + Hyperparams: input.Hyperparams, + Resources: input.Resources, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.UpsertOperator(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) DeleteOperator(ctx context.Context, id string) (bool, error) { + result, err := store.operators.DeleteOne(ctx, bson.M{"id": id}) + if err != nil { + return false, err + } + return result.DeletedCount == 1, nil +} + +func (store *MongoStore) UpsertOperator(ctx context.Context, item domain.OperatorRecord) error { + _, err := store.operators.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) ListWorkflowsByProject(ctx context.Context, projectID string) ([]domain.WorkflowRecord, error) { + cursor, err := store.workflows.Find(ctx, bson.M{"projectId": projectID}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.WorkflowRecord](ctx, cursor) +} + +func (store *MongoStore) ListAllWorkflows(ctx context.Context) ([]domain.WorkflowRecord, error) { + cursor, err := store.workflows.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.WorkflowRecord](ctx, cursor) +} + +func (store *MongoStore) GetWorkflow(ctx context.Context, id string) (*domain.WorkflowRecord, error) { + var item domain.WorkflowRecord + err := store.workflows.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateWorkflow(ctx context.Context, input domain.CreateWorkflowInput) (*domain.WorkflowRecord, error) { + now := nowISO() + item := domain.WorkflowRecord{ + ID: domain.NewID("workflow"), + ProjectID: input.ProjectID, + Name: input.Name, + Description: input.Description, + Type: input.Type, + MaxConcurrentRuns: input.MaxConcurrentRuns, + NodeTimeoutSeconds: input.NodeTimeoutSeconds, + RunTimeoutSeconds: input.RunTimeoutSeconds, + MaxRetryAttempts: input.MaxRetryAttempts, + RetryBackoffSeconds: input.RetryBackoffSeconds, + Schedule: input.Schedule, + Nodes: input.Nodes, + Edges: input.Edges, + Status: domain.WorkflowStatusDraft, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.UpsertWorkflow(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, error) { + if _, err := store.runs.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil { + return false, err + } + if _, err := store.auditTasks.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil { + return false, err + } + result, err := store.workflows.DeleteOne(ctx, bson.M{"id": id}) + if err != nil { + return false, err + } + return result.DeletedCount == 1, nil +} + +func (store *MongoStore) UpsertWorkflow(ctx context.Context, item domain.WorkflowRecord) error { + _, err := store.workflows.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) ListTemplates(ctx context.Context) ([]domain.TemplateRecord, error) { + cursor, err := store.templates.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.TemplateRecord](ctx, cursor) +} + +func (store *MongoStore) GetTemplate(ctx context.Context, id string) (*domain.TemplateRecord, error) { + var item domain.TemplateRecord + err := store.templates.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateTemplate(ctx context.Context, input domain.CreateTemplateInput) (*domain.TemplateRecord, error) { + now := nowISO() + templateType := input.Type + if templateType == "" { + templateType = domain.WorkflowTypeBatch + } + item := domain.TemplateRecord{ + ID: domain.NewID("template"), + Name: input.Name, + Description: input.Description, + Type: templateType, + MaxConcurrentRuns: input.MaxConcurrentRuns, + NodeTimeoutSeconds: input.NodeTimeoutSeconds, + RunTimeoutSeconds: input.RunTimeoutSeconds, + MaxRetryAttempts: input.MaxRetryAttempts, + RetryBackoffSeconds: input.RetryBackoffSeconds, + Schedule: input.Schedule, + Nodes: input.Nodes, + Edges: input.Edges, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.UpsertTemplate(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) DeleteTemplate(ctx context.Context, id string) (bool, error) { + result, err := store.templates.DeleteOne(ctx, bson.M{"id": id}) + if err != nil { + return false, err + } + return result.DeletedCount == 1, nil +} + +func (store *MongoStore) UpsertTemplate(ctx context.Context, item domain.TemplateRecord) error { + _, err := store.templates.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) ListRuns(ctx context.Context, status string) ([]domain.RunRecord, error) { + filter := bson.M{} + if status != "" { + filter["status"] = status + } + cursor, err := store.runs.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.RunRecord](ctx, cursor) +} + +func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunRecord, error) { + var item domain.RunRecord + err := store.runs.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger) (*domain.RunRecord, error) { + workflow, err := store.GetWorkflow(ctx, workflowID) + if err != nil { + return nil, err + } + if workflow == nil { + return nil, fmt.Errorf("workflow %s not found", workflowID) + } + if workflow.Status == domain.WorkflowStatusCompleted && trigger != domain.RunTriggerScheduled { + return nil, ErrWorkflowAlreadyCompleted + } + project, err := store.GetProject(ctx, workflow.ProjectID) + if err != nil { + return nil, err + } + if project == nil { + return nil, fmt.Errorf("project %s not found", workflow.ProjectID) + } + now := nowISO() + run := domain.RunRecord{ + ID: domain.NewID("run"), + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: project.ID, + ProjectName: project.Name, + Trigger: trigger, + Attempt: 1, + AvailableAt: now, + Status: domain.RunStatusQueued, + NodeExecutions: []domain.NodeExecutionRecord{}, + Logs: []domain.LogEntryRecord{}, + ResourceSeries: []domain.ResourceSample{}, + Artifacts: []domain.ExecutionArtifact{}, + K8sRefs: []domain.K8sRef{}, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.UpsertRun(ctx, run); err != nil { + return nil, err + } + workflow.LastRunID = run.ID + workflow.LastRunAt = now + workflow.Status = domain.WorkflowStatusRunning + workflow.UpdatedAt = now + if err := store.UpsertWorkflow(ctx, *workflow); err != nil { + return nil, err + } + return &run, nil +} + +func (store *MongoStore) ClaimNextQueuedRun(ctx context.Context) (*domain.RunRecord, error) { + now := nowISO() + result := store.runs.FindOneAndUpdate( + ctx, + bson.M{ + "status": string(domain.RunStatusQueued), + "$or": []bson.M{ + {"availableAt": bson.M{"$exists": false}}, + {"availableAt": ""}, + {"availableAt": bson.M{"$lte": now}}, + }, + }, + bson.M{"$set": bson.M{ + "status": string(domain.RunStatusRunning), + "startedAt": now, + "updatedAt": now, + }}, + options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After), + ) + var item domain.RunRecord + err := result.Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) ClaimQueuedRun(ctx context.Context, runID string) (*domain.RunRecord, error) { + now := nowISO() + result := store.runs.FindOneAndUpdate( + ctx, + bson.M{ + "id": runID, + "status": string(domain.RunStatusQueued), + "$or": []bson.M{ + {"availableAt": bson.M{"$exists": false}}, + {"availableAt": ""}, + {"availableAt": bson.M{"$lte": now}}, + }, + }, + bson.M{"$set": bson.M{ + "status": string(domain.RunStatusRunning), + "startedAt": now, + "updatedAt": now, + }}, + options.FindOneAndUpdate().SetReturnDocument(options.After), + ) + var item domain.RunRecord + err := result.Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) UpsertRun(ctx context.Context, item domain.RunRecord) error { + computeRunDuration(&item) + _, err := store.runs.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) ListAuditTasks(ctx context.Context, status string) ([]domain.AuditTaskRecord, error) { + filter := bson.M{} + if status != "" && status != "all" { + filter["status"] = status + } + cursor, err := store.auditTasks.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.AuditTaskRecord](ctx, cursor) +} + +func (store *MongoStore) GetAuditTask(ctx context.Context, id string) (*domain.AuditTaskRecord, error) { + var item domain.AuditTaskRecord + err := store.auditTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateAuditTask(ctx context.Context, item domain.AuditTaskRecord) (*domain.AuditTaskRecord, error) { + if err := store.UpsertAuditTask(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) UpsertAuditTask(ctx context.Context, item domain.AuditTaskRecord) error { + _, err := store.auditTasks.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) ListUsers(ctx context.Context) ([]domain.UserRecord, error) { + cursor, err := store.users.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.UserRecord](ctx, cursor) +} + +func (store *MongoStore) GetUser(ctx context.Context, id string) (*domain.UserRecord, error) { + var item domain.UserRecord + err := store.users.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) CreateUser(ctx context.Context, input domain.CreateUserInput) (*domain.UserRecord, error) { + now := nowISO() + item := domain.UserRecord{ + ID: domain.NewID("user"), + Name: input.Name, + Email: input.Email, + Role: input.Role, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.UpsertUser(ctx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) DeleteUser(ctx context.Context, id string) (bool, error) { + result, err := store.users.DeleteOne(ctx, bson.M{"id": id}) + if err != nil { + return false, err + } + return result.DeletedCount > 0, nil +} + +func (store *MongoStore) UpsertUser(ctx context.Context, item domain.UserRecord) error { + _, err := store.users.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) GetSettings(ctx context.Context) (*domain.SettingsRecord, error) { + var item domain.SettingsRecord + err := store.settings.FindOne(ctx, bson.M{"id": "system"}).Decode(&item) + if err == mongo.ErrNoDocuments { + fallback := defaultSettings() + if err := store.UpsertSettings(ctx, fallback); err != nil { + return nil, err + } + return &fallback, nil + } + if err != nil { + return nil, err + } + normalized := ensureSettingsDefaults(item) + return &normalized, nil +} + +func (store *MongoStore) UpsertSettings(ctx context.Context, item domain.SettingsRecord) error { + item = ensureSettingsDefaults(item) + _, err := store.settings.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) GetDashboardSummary(ctx context.Context) (domain.DashboardSummary, error) { + projects, err := store.ListProjects(ctx) + if err != nil { + return domain.DashboardSummary{}, err + } + operators, err := store.ListOperators(ctx) + if err != nil { + return domain.DashboardSummary{}, err + } + runs, err := store.ListRuns(ctx, "") + if err != nil { + return domain.DashboardSummary{}, err + } + auditTasks, err := store.ListAuditTasks(ctx, "all") + if err != nil { + return domain.DashboardSummary{}, err + } + summary := buildDashboardSummary(projects, operators, runs, auditTasks) + summary.Jobs.SuccessRate = float64(int(summary.Jobs.SuccessRate*10)) / 10 + return summary, nil +} + +func (store *MongoStore) ListStorageBuckets(ctx context.Context, sourcePath string) ([]string, error) { + return listBucketsFromSourcePath(sourcePath) +} + +func (store *MongoStore) ListStorageObjects(ctx context.Context, sourcePath string, bucket string, prefix string) ([]string, error) { + items, err := listObjectsFromSourcePath(sourcePath, strings.TrimSpace(bucket), strings.TrimSpace(prefix)) + if err != nil { + return nil, err + } + sort.Strings(items) + return items, nil +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 00000000..55a11497 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,59 @@ +package store + +import ( + "context" + + "droboticflow/internal/domain" +) + +type Store interface { + ListProjects(context.Context) ([]domain.ProjectRecord, error) + GetProject(context.Context, string) (*domain.ProjectRecord, error) + CreateProject(context.Context, domain.CreateProjectInput) (*domain.ProjectRecord, error) + DeleteProject(context.Context, string) (bool, error) + UpsertProject(context.Context, domain.ProjectRecord) error + + ListOperators(context.Context) ([]domain.OperatorRecord, error) + GetOperator(context.Context, string) (*domain.OperatorRecord, error) + CreateOperator(context.Context, domain.CreateOperatorInput) (*domain.OperatorRecord, error) + DeleteOperator(context.Context, string) (bool, error) + UpsertOperator(context.Context, domain.OperatorRecord) error + + ListWorkflowsByProject(context.Context, string) ([]domain.WorkflowRecord, error) + ListAllWorkflows(context.Context) ([]domain.WorkflowRecord, error) + GetWorkflow(context.Context, string) (*domain.WorkflowRecord, error) + CreateWorkflow(context.Context, domain.CreateWorkflowInput) (*domain.WorkflowRecord, error) + DeleteWorkflow(context.Context, string) (bool, error) + UpsertWorkflow(context.Context, domain.WorkflowRecord) error + + ListTemplates(context.Context) ([]domain.TemplateRecord, error) + GetTemplate(context.Context, string) (*domain.TemplateRecord, error) + CreateTemplate(context.Context, domain.CreateTemplateInput) (*domain.TemplateRecord, error) + DeleteTemplate(context.Context, string) (bool, error) + UpsertTemplate(context.Context, domain.TemplateRecord) error + + ListRuns(context.Context, string) ([]domain.RunRecord, error) + GetRun(context.Context, string) (*domain.RunRecord, error) + CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error) + ClaimNextQueuedRun(context.Context) (*domain.RunRecord, error) + ClaimQueuedRun(context.Context, string) (*domain.RunRecord, error) + UpsertRun(context.Context, domain.RunRecord) error + + ListAuditTasks(context.Context, string) ([]domain.AuditTaskRecord, error) + GetAuditTask(context.Context, string) (*domain.AuditTaskRecord, error) + CreateAuditTask(context.Context, domain.AuditTaskRecord) (*domain.AuditTaskRecord, error) + UpsertAuditTask(context.Context, domain.AuditTaskRecord) error + + ListUsers(context.Context) ([]domain.UserRecord, error) + GetUser(context.Context, string) (*domain.UserRecord, error) + CreateUser(context.Context, domain.CreateUserInput) (*domain.UserRecord, error) + DeleteUser(context.Context, string) (bool, error) + UpsertUser(context.Context, domain.UserRecord) error + + GetSettings(context.Context) (*domain.SettingsRecord, error) + UpsertSettings(context.Context, domain.SettingsRecord) error + + GetDashboardSummary(context.Context) (domain.DashboardSummary, error) + ListStorageBuckets(context.Context, string) ([]string, error) + ListStorageObjects(context.Context, string, string, string) ([]string, error) +} diff --git a/internal/worker/executor.go b/internal/worker/executor.go new file mode 100644 index 00000000..f44bcbe9 --- /dev/null +++ b/internal/worker/executor.go @@ -0,0 +1,329 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + + "droboticflow/internal/domain" + "droboticflow/internal/k8s" +) + +const resultPrefix = "DROBOTICFLOW_RESULT=" + +type executionResult struct { + Logs []domain.LogEntryRecord `json:"logs"` + Artifacts []domain.ExecutionArtifact `json:"artifacts"` + ResourceSeries []domain.ResourceSample `json:"resourceSeries"` +} + +var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`) + +func sanitizeEnvKey(value string) string { + normalized := strings.ToUpper(strings.TrimSpace(value)) + if normalized == "" { + return "" + } + normalized = invalidEnvCharacters.ReplaceAllString(normalized, "_") + normalized = strings.Trim(normalized, "_") + if normalized == "" { + return "" + } + if normalized[0] >= '0' && normalized[0] <= '9' { + normalized = "HP_" + normalized + } + return normalized +} + +func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact { + cloned := make([]domain.ExecutionArtifact, 0, len(items)) + for _, item := range items { + cloned = append(cloned, item) + } + return cloned +} + +func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact { + items := make([]domain.ExecutionArtifact, 0) + seen := map[string]struct{}{} + for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) { + execution, ok := executionMap[edge.Source] + if !ok { + continue + } + requiresHandle := strings.TrimSpace(edge.SourceHandle) != "" + hasPortAwareArtifacts := false + for _, artifact := range execution.Artifacts { + if strings.TrimSpace(artifact.PortID) != "" { + hasPortAwareArtifacts = true + break + } + } + for _, artifact := range execution.Artifacts { + if requiresHandle && hasPortAwareArtifacts && artifact.PortID != edge.SourceHandle { + continue + } + key := artifact.ID + if key == "" { + key = artifact.NodeID + ":" + artifact.Name + } + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + items = append(items, artifact) + } + } + return items +} + +func extractConfidence(artifact domain.ExecutionArtifact) (float64, bool) { + if artifact.Data == nil { + return 0, false + } + for _, key := range []string{"confidence", "score"} { + value, ok := artifact.Data[key] + if !ok { + continue + } + switch typed := value.(type) { + case float64: + return typed, true + case int: + return float64(typed), true + } + } + return 0, false +} + +func extractObjectHints(artifact domain.ExecutionArtifact) []string { + if artifact.Data == nil { + return nil + } + items := make([]string, 0) + for _, key := range []string{"detectedObjects", "labels", "entries"} { + value, ok := artifact.Data[key] + if !ok { + continue + } + switch typed := value.(type) { + case []any: + for _, candidate := range typed { + text := strings.TrimSpace(fmt.Sprint(candidate)) + if text != "" { + items = append(items, text) + } + } + case []string: + for _, candidate := range typed { + if trimmed := strings.TrimSpace(candidate); trimmed != "" { + items = append(items, trimmed) + } + } + } + } + return items +} + +func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) domain.AuditTaskPayload { + payload := domain.AuditTaskPayload{ + Reason: fmt.Sprintf("%s requires explicit approval before downstream execution", node.Data.Label), + DetectedObjects: []string{}, + } + for _, artifact := range inputArtifacts { + if payload.Image == "" { + switch { + case artifact.Metadata != nil && artifact.Metadata.Thumbnail != "": + payload.Image = artifact.Metadata.Thumbnail + case artifact.Type == domain.PortTypeImage || artifact.Type == domain.PortTypeVideo: + payload.Image = artifact.Path + } + } + if payload.Confidence == 0 { + if confidence, ok := extractConfidence(artifact); ok { + payload.Confidence = confidence + } + } + payload.DetectedObjects = append(payload.DetectedObjects, extractObjectHints(artifact)...) + } + if len(payload.DetectedObjects) == 0 { + payload.DetectedObjects = []string{fmt.Sprintf("%d upstream artifact(s)", len(inputArtifacts))} + } + return payload +} + +func prepareOutputDir(runID string, nodeID string) (string, error) { + outputDir := filepath.Join(os.TempDir(), "droboticflow", runID, nodeID) + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return "", err + } + return outputDir, nil +} + +func envMapToSlice(values map[string]string) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + slices.Sort(keys) + items := make([]string, 0, len(keys)) + for _, key := range keys { + items = append(items, key+"="+values[key]) + } + return items +} + +func buildNodeEnv(run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, outputDir string) (map[string]string, error) { + nodeDataJSON, err := json.Marshal(node.Data) + if err != nil { + return nil, err + } + inputArtifactsJSON, err := json.Marshal(inputArtifacts) + if err != nil { + return nil, err + } + env := map[string]string{ + "RUN_ID": run.ID, + "WORKFLOW_ID": run.WorkflowID, + "NODE_ID": node.ID, + "NODE_LABEL": node.Data.Label, + "NODE_TYPE": node.Type, + "NODE_CATEGORY": node.Data.Category, + "NODE_DESCRIPTION": node.Data.Description, + "NODE_DATA_JSON": string(nodeDataJSON), + "INPUT_ARTIFACTS_JSON": string(inputArtifactsJSON), + "OUTPUT_DIR": outputDir, + } + if node.Data.Bucket != "" { + env["STORAGE_BUCKET"] = node.Data.Bucket + } + if node.Data.Prefix != "" { + env["STORAGE_PREFIX"] = node.Data.Prefix + } + if node.Data.Pattern != "" { + env["STORAGE_PATTERN"] = node.Data.Pattern + } + if node.Data.CPU != "" { + env["CPU_LIMIT"] = node.Data.CPU + } + if node.Data.Memory != "" { + env["MEMORY_LIMIT"] = node.Data.Memory + } + if node.Data.Image != "" { + env["NODE_IMAGE"] = node.Data.Image + } + for _, item := range node.Data.Hyperparams { + key := sanitizeEnvKey(item.Key) + if key == "" { + continue + } + env[key] = item.Value + } + return env, nil +} + +func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, error) { + result := executionResult{ + Logs: []domain.LogEntryRecord{}, + Artifacts: []domain.ExecutionArtifact{}, + ResourceSeries: []domain.ResourceSample{}, + } + lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n") + structuredPayload := "" + rawLogs := make([]domain.LogEntryRecord, 0) + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + if strings.HasPrefix(trimmed, resultPrefix) { + structuredPayload = strings.TrimPrefix(trimmed, resultPrefix) + continue + } + rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel)) + } + if structuredPayload == "" { + result.Logs = rawLogs + return result, fmt.Errorf("operator execution did not emit %s payload", resultPrefix) + } + var parsed executionResult + if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil { + return result, fmt.Errorf("decode operator result: %w", err) + } + if len(parsed.Logs) > 0 { + result.Logs = append(result.Logs, parsed.Logs...) + } else { + result.Logs = append(result.Logs, rawLogs...) + } + result.Artifacts = parsed.Artifacts + result.ResourceSeries = parsed.ResourceSeries + return result, nil +} + +func (runtime *Runtime) executeLocalProcess(ctx context.Context, command string, env map[string]string) (string, error) { + cmd := exec.CommandContext(ctx, "sh", "-lc", command) + cmd.Env = append(os.Environ(), envMapToSlice(env)...) + output, err := cmd.CombinedOutput() + return string(output), err +} + +func (runtime *Runtime) executeLocalDocker(ctx context.Context, image string, env map[string]string, outputDir string) (string, error) { + args := []string{"run", "--rm", "-v", outputDir + ":" + outputDir} + for key, value := range env { + args = append(args, "-e", key+"="+value) + } + args = append(args, image) + cmd := exec.CommandContext(ctx, "docker", args...) + output, err := cmd.CombinedOutput() + return string(output), err +} + +func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) { + outputDir, err := prepareOutputDir(run.ID, node.ID) + if err != nil { + return executionResult{}, nil, err + } + env, err := buildNodeEnv(run, node, inputArtifacts, outputDir) + if err != nil { + return executionResult{}, nil, err + } + if runtime.launcher.Enabled() { + nodeTimeoutSeconds := 0 + if workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID); err == nil && workflow != nil { + nodeTimeoutSeconds = workflow.NodeTimeoutSeconds + } + launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{ + Image: chooseImage(node, runtime.operatorRunnerImage), + RunID: run.ID, + WorkflowID: run.WorkflowID, + NodeID: node.ID, + NodeLabel: node.Data.Label, + TimeoutSeconds: nodeTimeoutSeconds, + Env: env, + }) + if err != nil { + return executionResult{}, nil, err + } + parsed, err := parseExecutionOutput(launched.Output, node.Data.Label) + return parsed, launched.Reference, err + } + + image := chooseImage(node, runtime.operatorRunnerImage) + var stdout string + if node.Data.Image != "" && node.Data.Image != runtime.operatorRunnerImage { + stdout, err = runtime.executeLocalDocker(ctx, image, env, outputDir) + } else { + stdout, err = runtime.executeLocalProcess(ctx, runtime.localRunnerCommand, env) + } + if err != nil { + return executionResult{}, nil, fmt.Errorf("%w: %s", err, stdout) + } + parsed, err := parseExecutionOutput(stdout, node.Data.Label) + return parsed, nil, err +} diff --git a/internal/worker/http.go b/internal/worker/http.go new file mode 100644 index 00000000..7815d96b --- /dev/null +++ b/internal/worker/http.go @@ -0,0 +1,97 @@ +package worker + +import ( + "encoding/json" + "net/http" + + "droboticflow/internal/domain" + "droboticflow/internal/store" + + "github.com/go-chi/chi/v5" +) + +type HTTPServer struct { + store store.Store + runtime *Runtime +} + +func NewHTTPHandler(dataStore store.Store, runtime *Runtime) http.Handler { + server := &HTTPServer{store: dataStore, runtime: runtime} + router := chi.NewRouter() + + router.Get("/worker-api/health", server.handleHealth) + router.Get("/worker-api/stats", server.handleStats) + router.Post("/worker-api/process-once", server.handleProcessOnce) + router.Post("/worker-api/runs/{id}/retry", server.handleRetry) + + return router +} + +func (server *HTTPServer) handleHealth(writer http.ResponseWriter, _ *http.Request) { + renderWorkerJSON(writer, http.StatusOK, map[string]string{"status": "ok", "service": "droboticflow-worker"}) +} + +func (server *HTTPServer) handleStats(writer http.ResponseWriter, request *http.Request) { + runs, err := server.store.ListRuns(request.Context(), "") + if err != nil { + renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + response := map[string]int{"queued": 0, "running": 0, "waitingForAudit": 0} + for _, run := range runs { + switch run.Status { + case domain.RunStatusQueued: + response["queued"]++ + case domain.RunStatusRunning: + response["running"]++ + case domain.RunStatusWaitingForAudit: + response["waitingForAudit"]++ + } + } + renderWorkerJSON(writer, http.StatusOK, response) +} + +func (server *HTTPServer) handleProcessOnce(writer http.ResponseWriter, request *http.Request) { + run, err := server.runtime.ProcessOnce(request.Context()) + if err != nil { + renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + renderWorkerJSON(writer, http.StatusOK, map[string]any{"run": run}) +} + +func (server *HTTPServer) handleRetry(writer http.ResponseWriter, request *http.Request) { + run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if run == nil { + renderWorkerJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) + return + } + run.Status = domain.RunStatusQueued + run.StartedAt = "" + run.EndedAt = "" + run.DurationMS = 0 + run.CurrentNodeID = "" + run.NodeExecutions = []domain.NodeExecutionRecord{} + run.Logs = []domain.LogEntryRecord{} + run.ResourceSeries = []domain.ResourceSample{} + run.Artifacts = []domain.ExecutionArtifact{} + run.K8sRefs = []domain.K8sRef{} + run.Error = "" + run.AvailableAt = nowISO() + run.UpdatedAt = nowISO() + if err := server.store.UpsertRun(request.Context(), *run); err != nil { + renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + renderWorkerJSON(writer, http.StatusAccepted, run) +} + +func renderWorkerJSON(writer http.ResponseWriter, status int, payload any) { + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(payload) +} diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go new file mode 100644 index 00000000..2b0a3d66 --- /dev/null +++ b/internal/worker/runtime.go @@ -0,0 +1,724 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/k8s" + "droboticflow/internal/store" +) + +type Runtime struct { + store store.Store + mode string + namespace string + imagePullSecretName string + operatorRunnerImage string + localRunnerCommand string + launcher *k8s.Launcher +} + +var activeDispatchStatuses = map[domain.RunStatus]struct{}{ + domain.RunStatusRunning: {}, + domain.RunStatusWaitingForAudit: {}, +} + +var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{ + domain.RunStatusQueued: {}, + domain.RunStatusRunning: {}, + domain.RunStatusWaitingForAudit: {}, +} + +func NewRuntime(dataStore store.Store, mode string, namespace string, imagePullSecretName string, operatorRunnerImage string, localRunnerCommand string) (*Runtime, error) { + launcher, err := k8s.NewLauncher(mode == "k8s", namespace, imagePullSecretName) + if err != nil { + return nil, err + } + return &Runtime{ + store: dataStore, + mode: mode, + namespace: namespace, + imagePullSecretName: imagePullSecretName, + operatorRunnerImage: operatorRunnerImage, + localRunnerCommand: localRunnerCommand, + launcher: launcher, + }, nil +} + +func nowISO() string { + return time.Now().UTC().Format(time.RFC3339Nano) +} + +func logEntry(level string, message string, operator string) domain.LogEntryRecord { + return domain.LogEntryRecord{ + Timestamp: nowISO(), + Level: level, + Message: message, + Operator: operator, + } +} + +func findNodeExecution(run *domain.RunRecord, nodeID string) (int, *domain.NodeExecutionRecord) { + for index := range run.NodeExecutions { + if run.NodeExecutions[index].NodeID == nodeID { + return index, &run.NodeExecutions[index] + } + } + return -1, nil +} + +func appendLog(run *domain.RunRecord, entry domain.LogEntryRecord) { + run.Logs = append(run.Logs, entry) +} + +func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) { + index, _ := findNodeExecution(run, execution.NodeID) + if index >= 0 { + run.NodeExecutions[index] = execution + } else { + run.NodeExecutions = append(run.NodeExecutions, execution) + } + run.Logs = []domain.LogEntryRecord{} + run.ResourceSeries = []domain.ResourceSample{} + run.Artifacts = []domain.ExecutionArtifact{} + for _, item := range run.NodeExecutions { + for _, entry := range item.Logs { + appendLog(run, entry) + } + if len(item.ResourceSeries) > 0 { + run.ResourceSeries = append(run.ResourceSeries, item.ResourceSeries...) + } + if len(item.Artifacts) > 0 { + run.Artifacts = append(run.Artifacts, item.Artifacts...) + } + } +} + +func globalConcurrencyLimit(settings *domain.SettingsRecord) int { + if settings != nil && settings.Worker.MaxConcurrentRuns > 0 { + return settings.Worker.MaxConcurrentRuns + } + return 1 +} + +func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings *domain.SettingsRecord) int { + if workflow != nil && workflow.MaxConcurrentRuns > 0 { + return workflow.MaxConcurrentRuns + } + return globalConcurrencyLimit(settings) +} + +func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string { + globalLimit := globalConcurrencyLimit(settings) + workflowByID := make(map[string]domain.WorkflowRecord, len(workflows)) + for _, workflow := range workflows { + workflowByID[workflow.ID] = workflow + } + now := time.Now().UTC() + + activeByWorkflow := map[string]int{} + activeTotal := 0 + queued := make([]domain.RunRecord, 0) + for _, run := range runs { + if _, ok := activeDispatchStatuses[run.Status]; ok { + activeTotal++ + activeByWorkflow[run.WorkflowID]++ + continue + } + if run.Status == domain.RunStatusQueued { + if !runDispatchReady(run, now) { + continue + } + queued = append(queued, run) + } + } + + available := globalLimit - activeTotal + if available <= 0 { + return nil + } + + sort.Slice(queued, func(left int, right int) bool { + leftAvailableAt := runAvailableAt(queued[left]) + rightAvailableAt := runAvailableAt(queued[right]) + if leftAvailableAt.Equal(rightAvailableAt) { + return queued[left].CreatedAt < queued[right].CreatedAt + } + return leftAvailableAt.Before(rightAvailableAt) + }) + + selected := make([]string, 0, available) + for _, run := range queued { + if available == 0 { + break + } + workflow := workflowByID[run.WorkflowID] + workflowLimit := effectiveWorkflowConcurrencyLimit(&workflow, settings) + if activeByWorkflow[run.WorkflowID] >= workflowLimit { + continue + } + selected = append(selected, run.ID) + activeByWorkflow[run.WorkflowID]++ + available-- + } + return selected +} + +func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error { + workflows, err := runtime.store.ListAllWorkflows(ctx) + if err != nil { + return err + } + runs, err := runtime.store.ListRuns(ctx, "") + if err != nil { + return err + } + settings, err := runtime.store.GetSettings(ctx) + if err != nil { + return err + } + now := time.Now().UTC() + for _, workflow := range workflows { + if workflow.Type != domain.WorkflowTypeStreaming || workflow.Schedule == nil || workflow.Schedule.Interval <= 0 { + continue + } + inflightCount := 0 + for _, run := range runs { + if run.WorkflowID == workflow.ID { + if _, ok := queuedOrActiveStatuses[run.Status]; ok { + inflightCount++ + } + } + } + if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) { + continue + } + lastRunAt := time.Time{} + if workflow.LastRunAt != "" { + parsed, err := time.Parse(time.RFC3339Nano, workflow.LastRunAt) + if err == nil { + lastRunAt = parsed + } + } + if workflow.LastRunAt == "" || now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second { + if _, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled); err != nil { + return err + } + } + } + return nil +} + +func (runtime *Runtime) processClaimedRun(claimed *domain.RunRecord) { + if _, err := runtime.processRun(context.Background(), claimed); err != nil { + fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err) + } +} + +func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { + if err := runtime.enqueueStreamingWorkflows(ctx); err != nil { + return err + } + settings, err := runtime.store.GetSettings(ctx) + if err != nil { + return err + } + workflows, err := runtime.store.ListAllWorkflows(ctx) + if err != nil { + return err + } + runs, err := runtime.store.ListRuns(ctx, "") + if err != nil { + return err + } + for _, runID := range selectQueuedRunsForDispatch(runs, workflows, settings) { + claimed, err := runtime.store.ClaimQueuedRun(ctx, runID) + if err != nil { + return err + } + if claimed == nil { + continue + } + go runtime.processClaimedRun(claimed) + } + return nil +} + +func chooseImage(node domain.WorkflowNode, fallback string) string { + if node.Data.Image != "" { + return node.Data.Image + } + return fallback +} + +func chooseValue(value string, fallback string) string { + if value == "" { + return fallback + } + return value +} + +func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration { + if workflow == nil || workflow.RunTimeoutSeconds <= 0 { + return 0 + } + return time.Duration(workflow.RunTimeoutSeconds) * time.Second +} + +func workflowNodeTimeout(workflow *domain.WorkflowRecord) time.Duration { + if workflow == nil || workflow.NodeTimeoutSeconds <= 0 { + return 0 + } + return time.Duration(workflow.NodeTimeoutSeconds) * time.Second +} + +func normalizeRunError(workflow *domain.WorkflowRecord, err error, executionCtx context.Context, nodeTimedOut bool) string { + if err == nil { + return "" + } + if workflow != nil && workflow.RunTimeoutSeconds > 0 && (errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded)) { + return fmt.Sprintf("工作流执行超时(%d 秒)", workflow.RunTimeoutSeconds) + } + if workflow != nil && workflow.NodeTimeoutSeconds > 0 && nodeTimedOut { + return fmt.Sprintf("节点执行超时(%d 秒)", workflow.NodeTimeoutSeconds) + } + return err.Error() +} + +func shouldAutoRetry(workflow *domain.WorkflowRecord, run *domain.RunRecord) bool { + if workflow == nil || run == nil || workflow.MaxRetryAttempts <= 0 { + return false + } + attempt := run.Attempt + if attempt <= 0 { + attempt = 1 + } + return attempt <= workflow.MaxRetryAttempts +} + +func nextRunAttempt(run *domain.RunRecord) int { + if run == nil || run.Attempt <= 0 { + return 1 + } + return run.Attempt + 1 +} + +func nextRunAvailableAt(workflow *domain.WorkflowRecord) string { + if workflow == nil || workflow.RetryBackoffSeconds <= 0 { + return nowISO() + } + return time.Now().UTC().Add(time.Duration(workflow.RetryBackoffSeconds) * time.Second).Format(time.RFC3339Nano) +} + +func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.WorkflowRecord, failedRun *domain.RunRecord) (*domain.RunRecord, error) { + if workflow == nil || failedRun == nil { + return nil, nil + } + retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry) + if err != nil { + return nil, err + } + retryRun.Attempt = nextRunAttempt(failedRun) + retryRun.ParentRunID = failedRun.ID + retryRun.AvailableAt = nextRunAvailableAt(workflow) + retryRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil { + return nil, err + } + return retryRun, nil +} + +type nodeExecutionOutcome struct { + nodeID string + result executionResult + reference *k8s.JobReference + err error + nodeTimedOut bool +} + +func runAvailableAt(run domain.RunRecord) time.Time { + if run.AvailableAt != "" { + if parsed, err := time.Parse(time.RFC3339Nano, run.AvailableAt); err == nil { + return parsed + } + } + if parsed, err := time.Parse(time.RFC3339Nano, run.CreatedAt); err == nil { + return parsed + } + return time.Time{} +} + +func runDispatchReady(run domain.RunRecord, now time.Time) bool { + availableAt := runAvailableAt(run) + if availableAt.IsZero() { + return true + } + return !availableAt.After(now) +} + +func syncCurrentNodeID(run *domain.RunRecord) { + run.CurrentNodeID = "" + for _, status := range []domain.NodeExecutionStatus{ + domain.NodeExecutionStatusWaitingForAudit, + domain.NodeExecutionStatusRunning, + } { + for _, execution := range run.NodeExecutions { + if execution.Status == status { + run.CurrentNodeID = execution.NodeID + return + } + } + } +} + +func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool { + for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) { + dependency, ok := executionMap[candidate.Source] + if !ok || dependency.Status != domain.NodeExecutionStatusCompleted { + return false + } + } + return true +} + +func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, nodeID string) { + if _, queued := readySet[nodeID]; queued { + return + } + if execution, ok := executionMap[nodeID]; ok { + if execution.Status == domain.NodeExecutionStatusCompleted || + execution.Status == domain.NodeExecutionStatusRunning || + execution.Status == domain.NodeExecutionStatusWaitingForAudit { + return + } + } + if !dependenciesComplete(workflow, executionMap, nodeID) { + return + } + *readyQueue = append(*readyQueue, nodeID) + readySet[nodeID] = struct{}{} +} + +func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, nodeID string) { + for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) { + queueNodeIfReady(workflow, executionMap, readyQueue, readySet, edge.Target) + } +} + +func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) { + workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID) + if err != nil { + return nil, err + } + if workflow == nil { + claimed.Status = domain.RunStatusFailed + claimed.EndedAt = nowISO() + claimed.Error = fmt.Sprintf("workflow %s not found", claimed.WorkflowID) + claimed.UpdatedAt = nowISO() + return claimed, runtime.store.UpsertRun(ctx, *claimed) + } + + currentRun, err := runtime.store.GetRun(ctx, claimed.ID) + if err != nil { + return nil, err + } + if currentRun == nil { + currentRun = claimed + } + settings, err := runtime.store.GetSettings(ctx) + if err != nil { + return nil, err + } + nodeConcurrencyLimit := effectiveWorkflowConcurrencyLimit(workflow, settings) + if nodeConcurrencyLimit <= 0 { + nodeConcurrencyLimit = 1 + } + + executionMap := map[string]domain.NodeExecutionRecord{} + for _, execution := range currentRun.NodeExecutions { + executionMap[execution.NodeID] = execution + } + readyQueue := make([]string, 0) + readySet := make(map[string]struct{}) + auditQueue := make([]string, 0) + + if currentRun.CurrentNodeID != "" { + tasks, err := runtime.store.ListAuditTasks(ctx, "all") + if err != nil { + return nil, err + } + for _, task := range tasks { + if task.RunID == currentRun.ID && task.NodeID == currentRun.CurrentNodeID && task.Status != domain.AuditStatusPending { + node := domain.GetNodeByID(workflow.Nodes, currentRun.CurrentNodeID) + if node != nil { + execution := executionMap[node.ID] + execution.NodeID = node.ID + execution.NodeLabel = node.Data.Label + execution.Status = domain.NodeExecutionStatusCompleted + execution.EndedAt = nowISO() + execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label)) + executionMap[node.ID] = execution + applyExecution(currentRun, execution) + for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) { + queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, target) + } + currentRun.Status = domain.RunStatusRunning + syncCurrentNodeID(currentRun) + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + } + break + } + } + } + + if len(readyQueue) == 0 && len(executionMap) == 0 { + for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) { + queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, nodeID) + } + } + + executionCtx := ctx + cancelExecution := func() {} + if timeout := workflowRunTimeout(workflow); timeout > 0 { + executionCtx, cancelExecution = context.WithTimeout(ctx, timeout) + } else { + executionCtx, cancelExecution = context.WithCancel(ctx) + } + defer cancelExecution() + results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit) + runningNodes := make(map[string]struct{}) + for { + launched := false + for len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 { + nodeID := readyQueue[0] + readyQueue = readyQueue[1:] + delete(readySet, nodeID) + + node := domain.GetNodeByID(workflow.Nodes, nodeID) + if node == nil { + continue + } + if existing, ok := executionMap[node.ID]; ok { + if existing.Status == domain.NodeExecutionStatusCompleted || + existing.Status == domain.NodeExecutionStatusRunning || + existing.Status == domain.NodeExecutionStatusWaitingForAudit { + continue + } + } + if !dependenciesComplete(*workflow, executionMap, node.ID) { + continue + } + if domain.NodeRequiresAudit(*node) { + auditQueue = append(auditQueue, node.ID) + continue + } + + inputArtifacts := gatherInputArtifacts(*workflow, executionMap, node.ID) + execution := domain.NodeExecutionRecord{ + NodeID: node.ID, + NodeLabel: node.Data.Label, + Status: domain.NodeExecutionStatusRunning, + StartedAt: nowISO(), + Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)}, + Artifacts: []domain.ExecutionArtifact{}, + ResourceSeries: []domain.ResourceSample{}, + } + executionMap[node.ID] = execution + applyExecution(currentRun, execution) + runningNodes[node.ID] = struct{}{} + launched = true + + runSnapshot := *currentRun + nodeSnapshot := *node + inputSnapshot := cloneArtifacts(inputArtifacts) + nodeExecutionCtx := executionCtx + cancelNodeExecution := func() {} + if timeout := workflowNodeTimeout(workflow); timeout > 0 { + nodeExecutionCtx, cancelNodeExecution = context.WithTimeout(executionCtx, timeout) + } + go func() { + defer cancelNodeExecution() + result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot) + results <- nodeExecutionOutcome{ + nodeID: nodeSnapshot.ID, + result: result, + reference: reference, + err: err, + nodeTimedOut: errors.Is(nodeExecutionCtx.Err(), context.DeadlineExceeded), + } + }() + } + + if launched { + currentRun.Status = domain.RunStatusRunning + syncCurrentNodeID(currentRun) + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + } + + if len(runningNodes) == 0 { + if len(readyQueue) > 0 { + continue + } + if len(auditQueue) > 0 { + nodeID := auditQueue[0] + auditQueue = auditQueue[1:] + node := domain.GetNodeByID(workflow.Nodes, nodeID) + if node == nil { + continue + } + inputArtifacts := gatherInputArtifacts(*workflow, executionMap, node.ID) + execution := executionMap[node.ID] + if execution.NodeID == "" { + execution = domain.NodeExecutionRecord{ + NodeID: node.ID, + NodeLabel: node.Data.Label, + Status: domain.NodeExecutionStatusRunning, + StartedAt: nowISO(), + Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)}, + Artifacts: []domain.ExecutionArtifact{}, + ResourceSeries: []domain.ResourceSample{}, + } + } + task := domain.AuditTaskRecord{ + ID: domain.NewID("audit"), + RunID: currentRun.ID, + WorkflowID: workflow.ID, + ProjectID: workflow.ProjectID, + NodeID: node.ID, + WorkflowName: workflow.Name, + ProjectName: currentRun.ProjectName, + OperatorName: node.Data.Label, + CreatedAt: nowISO(), + UpdatedAt: nowISO(), + Status: domain.AuditStatusPending, + Data: createAuditPayload(*node, inputArtifacts), + } + if _, err := runtime.store.CreateAuditTask(ctx, task); err != nil { + return nil, err + } + execution.Status = domain.NodeExecutionStatusWaitingForAudit + execution.EndedAt = nowISO() + execution.Artifacts = cloneArtifacts(inputArtifacts) + execution.Logs = append(execution.Logs, logEntry("WARN", "Node is waiting for human review", node.Data.Label)) + executionMap[node.ID] = execution + applyExecution(currentRun, execution) + currentRun.Status = domain.RunStatusWaitingForAudit + syncCurrentNodeID(currentRun) + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + workflow.Status = domain.WorkflowStatusRunning + workflow.UpdatedAt = nowISO() + if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil { + return nil, err + } + return currentRun, nil + } + break + } + + outcome := <-results + delete(runningNodes, outcome.nodeID) + execution := executionMap[outcome.nodeID] + if outcome.err != nil { + cancelExecution() + execution.Status = domain.NodeExecutionStatusFailed + execution.EndedAt = nowISO() + execution.Error = normalizeRunError(workflow, outcome.err, executionCtx, outcome.nodeTimedOut) + execution.Logs = append(execution.Logs, logEntry("ERROR", execution.Error, execution.NodeLabel)) + executionMap[outcome.nodeID] = execution + applyExecution(currentRun, execution) + currentRun.Status = domain.RunStatusFailed + currentRun.CurrentNodeID = outcome.nodeID + currentRun.EndedAt = nowISO() + currentRun.Error = execution.Error + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + workflow.Status = domain.WorkflowStatusFailed + workflow.UpdatedAt = nowISO() + if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil { + return nil, err + } + if shouldAutoRetry(workflow, currentRun) { + retryRun, err := runtime.enqueueRetryRun(ctx, workflow, currentRun) + if err != nil { + return nil, err + } + if retryRun != nil { + appendLog(currentRun, logEntry("WARN", fmt.Sprintf("已自动创建重试运行,第 %d 次尝试已进入队列", retryRun.Attempt), execution.NodeLabel)) + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + } + } + return currentRun, nil + } + + execution.Status = domain.NodeExecutionStatusCompleted + execution.EndedAt = nowISO() + execution.Logs = append(execution.Logs, outcome.result.Logs...) + execution.Artifacts = outcome.result.Artifacts + execution.ResourceSeries = outcome.result.ResourceSeries + executionMap[outcome.nodeID] = execution + applyExecution(currentRun, execution) + if outcome.reference != nil { + currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace}) + } + enqueueDependents(*workflow, executionMap, &readyQueue, readySet, outcome.nodeID) + currentRun.Status = domain.RunStatusRunning + syncCurrentNodeID(currentRun) + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + } + + currentRun.Status = domain.RunStatusCompleted + currentRun.EndedAt = nowISO() + syncCurrentNodeID(currentRun) + currentRun.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { + return nil, err + } + workflow.Status = domain.WorkflowStatusCompleted + workflow.LastRunID = currentRun.ID + workflow.LastRunAt = nowISO() + workflow.UpdatedAt = nowISO() + if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil { + return nil, err + } + return currentRun, nil +} + +func edgeTargets(edges []domain.WorkflowEdge) []string { + items := make([]string, 0, len(edges)) + for _, edge := range edges { + items = append(items, edge.Target) + } + return items +} + +func (runtime *Runtime) ProcessOnce(ctx context.Context) (*domain.RunRecord, error) { + if err := runtime.enqueueStreamingWorkflows(ctx); err != nil { + return nil, err + } + claimed, err := runtime.store.ClaimNextQueuedRun(ctx) + if err != nil || claimed == nil { + return claimed, err + } + return runtime.processRun(ctx, claimed) +} diff --git a/internal/worker/runtime_test.go b/internal/worker/runtime_test.go new file mode 100644 index 00000000..ed0e8b77 --- /dev/null +++ b/internal/worker/runtime_test.go @@ -0,0 +1,1338 @@ +package worker + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func localRunnerCommand(t *testing.T) string { + t.Helper() + path, err := filepath.Abs("../../backend/operator-runner/main.py") + if err != nil { + t.Fatalf("resolve operator runner: %v", err) + } + return "python3 " + path +} + +func mustNodeExecution(t *testing.T, run *domain.RunRecord, nodeID string) domain.NodeExecutionRecord { + t.Helper() + for _, item := range run.NodeExecutions { + if item.NodeID == nodeID { + return item + } + } + t.Fatalf("missing node execution for %s", nodeID) + return domain.NodeExecutionRecord{} +} + +func mustArtifact(t *testing.T, execution domain.NodeExecutionRecord, name string) domain.ExecutionArtifact { + t.Helper() + for _, item := range execution.Artifacts { + if item.Name == name { + return item + } + } + t.Fatalf("missing artifact %s on node %s", name, execution.NodeID) + return domain.ExecutionArtifact{} +} + +func writeFile(t *testing.T, path string, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func createRosbagDB3(t *testing.T, path string) { + t.Helper() + command := exec.Command( + "python3", + "-c", + strings.Join([]string{ + "import sqlite3, sys", + "conn = sqlite3.connect(sys.argv[1])", + "cur = conn.cursor()", + "cur.execute('create table topics (id integer primary key, name text, type text, serialization_format text)')", + "cur.execute('create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)')", + "cur.execute(\"insert into topics (id, name, type, serialization_format) values (1, '/camera', 'sensor_msgs/msg/Image', 'cdr')\")", + "cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 100, X'01')\")", + "cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 200, X'02')\")", + "conn.commit()", + "conn.close()", + }, "; "), + path, + ) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("create rosbag db3 %s: %v: %s", path, err, string(output)) + } +} + +func waitForRunStatus(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected domain.RunStatus, timeout time.Duration) *domain.RunRecord { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + run, err := dataStore.GetRun(ctx, runID) + if err != nil { + t.Fatalf("get run %s: %v", runID, err) + } + if run != nil && run.Status == expected { + return run + } + time.Sleep(100 * time.Millisecond) + } + run, err := dataStore.GetRun(ctx, runID) + if err != nil { + t.Fatalf("get run %s: %v", runID, err) + } + t.Fatalf("timed out waiting for run %s to become %s, got %#v", runID, expected, run) + return nil +} + +func waitForRunningNodeCount(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected int, timeout time.Duration) *domain.RunRecord { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + run, err := dataStore.GetRun(ctx, runID) + if err != nil { + t.Fatalf("get run %s: %v", runID, err) + } + if run == nil { + time.Sleep(100 * time.Millisecond) + continue + } + running := 0 + for _, execution := range run.NodeExecutions { + if execution.Status == domain.NodeExecutionStatusRunning { + running++ + } + } + if running == expected { + return run + } + time.Sleep(100 * time.Millisecond) + } + run, err := dataStore.GetRun(ctx, runID) + if err != nil { + t.Fatalf("get run %s: %v", runID, err) + } + t.Fatalf("timed out waiting for run %s to have %d running nodes, got %#v", runID, expected, run) + return nil +} + +func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Worker 项目", Description: "worker"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "批处理工作流", + Description: "complete path", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "OSS 数据源", + Category: "IO", + Description: "source", + Inputs: []domain.PortDefinition{}, + Outputs: []domain.PortDefinition{ + {ID: "out-1", Name: "file", Type: domain.PortTypeFile}, + }, + Hyperparams: []domain.HyperParam{ + {Key: "SOURCE_TEXT", Value: "worker-runtime-test"}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "sink", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "结果上传", + Category: "IO", + Description: "sink", + Inputs: []domain.PortDefinition{ + {ID: "in-1", Name: "json", Type: domain.PortTypeJSON}, + }, + Outputs: []domain.PortDefinition{}, + }, + Position: domain.Position{X: 100, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "e1", Source: "source", SourceHandle: "out-1", Target: "sink", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + + completed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if completed == nil || completed.Status != domain.RunStatusCompleted { + t.Fatalf("expected completed run, got %#v", completed) + } + for _, item := range completed.NodeExecutions { + if item.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected completed node execution, got %s", item.Status) + } + } + if len(completed.Artifacts) == 0 { + t.Fatalf("expected artifacts") + } + if completed.Artifacts[0].Path == "" { + t.Fatalf("expected real artifact path") + } + if len(completed.ResourceSeries) == 0 { + t.Fatalf("expected resource samples from operator runner") + } +} + +func TestRuntimePausesForAuditAndResumes(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Audit 项目", Description: "audit"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "审核工作流", + Description: "audit path", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "review", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "人工审核 (Human Review)", + Category: "HUMAN", + Description: "review", + Inputs: []domain.PortDefinition{ + {ID: "in-1", Name: "json", Type: domain.PortTypeJSON}, + }, + Outputs: []domain.PortDefinition{ + {ID: "approved", Name: "审核通过", Type: domain.PortTypeJSON}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "upload", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "结果上传", + Category: "IO", + Description: "sink", + Inputs: []domain.PortDefinition{ + {ID: "in-1", Name: "json", Type: domain.PortTypeJSON}, + }, + Outputs: []domain.PortDefinition{}, + }, + Position: domain.Position{X: 100, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "e1", Source: "review", SourceHandle: "approved", Target: "upload", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + + paused, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get paused run: %v", err) + } + if paused == nil || paused.Status != domain.RunStatusWaitingForAudit { + t.Fatalf("expected waiting_for_audit run, got %#v", paused) + } + + tasks, err := dataStore.ListAuditTasks(ctx, "all") + if err != nil { + t.Fatalf("list audit tasks: %v", err) + } + var task *domain.AuditTaskRecord + for _, item := range tasks { + if item.RunID == run.ID && item.Status == domain.AuditStatusPending { + copied := item + task = &copied + break + } + } + if task == nil { + t.Fatalf("expected pending audit task") + } + + task.Status = domain.AuditStatusApproved + task.UpdatedAt = nowISO() + if err := dataStore.UpsertAuditTask(ctx, *task); err != nil { + t.Fatalf("approve audit task: %v", err) + } + paused.Status = domain.RunStatusQueued + paused.UpdatedAt = nowISO() + if err := dataStore.UpsertRun(ctx, *paused); err != nil { + t.Fatalf("queue paused run: %v", err) + } + + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("resume process once: %v", err) + } + + resumed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get resumed run: %v", err) + } + if resumed == nil || resumed.Status != domain.RunStatusCompleted { + t.Fatalf("expected completed resumed run, got %#v", resumed) + } +} + +func TestRuntimeRoutesArtifactsByOutputHandle(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Route 项目", Description: "route"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "条件分流工作流", + Description: "route by handle", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "source", + Outputs: []domain.PortDefinition{ + {ID: "out-manifest", Name: "manifest", Type: domain.PortTypeJSON}, + }, + Hyperparams: []domain.HyperParam{ + {Key: "COMMAND", Value: `printf '{"items":[{"kind":"keep","value":1},{"kind":"drop","value":2}]}' > routed.json`}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "route", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "条件分流", + Category: "LOGIC", + Description: "route", + Inputs: []domain.PortDefinition{ + {ID: "in-1", Name: "input", Type: domain.PortTypeJSON}, + }, + Outputs: []domain.PortDefinition{ + {ID: "matched", Name: "matched", Type: domain.PortTypeJSON}, + {ID: "unmatched", Name: "unmatched", Type: domain.PortTypeJSON}, + }, + Hyperparams: []domain.HyperParam{ + {Key: "FIELD_PATH", Value: "kind"}, + {Key: "OPERATOR", Value: "eq"}, + {Key: "EXPECTED_JSON", Value: `"keep"`}, + }, + }, + Position: domain.Position{X: 200, Y: 0}, + }, + { + ID: "keep-upload", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "结果上传", + Category: "IO", + Description: "keep sink", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 420, Y: -80}, + }, + { + ID: "drop-upload", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "结果上传", + Category: "IO", + Description: "drop sink", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 420, Y: 80}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "source-route", Source: "source", SourceHandle: "out-manifest", Target: "route", TargetHandle: "in-1"}, + {ID: "route-keep", Source: "route", SourceHandle: "matched", Target: "keep-upload", TargetHandle: "in-1"}, + {ID: "route-drop", Source: "route", SourceHandle: "unmatched", Target: "drop-upload", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + + completed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if completed == nil || completed.Status != domain.RunStatusCompleted { + t.Fatalf("expected completed run, got %#v", completed) + } + + routeExecution := mustNodeExecution(t, completed, "route") + matched := mustArtifact(t, routeExecution, "matched.json") + unmatched := mustArtifact(t, routeExecution, "unmatched.json") + if matched.PortID != "matched" || unmatched.PortID != "unmatched" { + t.Fatalf("expected output port IDs on split artifacts, got matched=%q unmatched=%q", matched.PortID, unmatched.PortID) + } + + keepUpload := mustNodeExecution(t, completed, "keep-upload") + dropUpload := mustNodeExecution(t, completed, "drop-upload") + keepSummary := mustArtifact(t, keepUpload, "uploaded-results.json") + dropSummary := mustArtifact(t, dropUpload, "uploaded-results.json") + keepItems, _ := keepSummary.Data["items"].([]any) + dropItems, _ := dropSummary.Data["items"].([]any) + if len(keepItems) != 1 || len(dropItems) != 1 { + t.Fatalf("expected one routed payload per sink, got keep=%d drop=%d", len(keepItems), len(dropItems)) + } + + keepPayload, _ := keepItems[0].(map[string]any) + dropPayload, _ := dropItems[0].(map[string]any) + keepRows, _ := keepPayload["items"].([]any) + dropRows, _ := dropPayload["items"].([]any) + if len(keepRows) != 1 || len(dropRows) != 1 { + t.Fatalf("expected one row in each routed payload, got keep=%d drop=%d", len(keepRows), len(dropRows)) + } + if keepRow, _ := keepRows[0].(map[string]any); keepRow != nil { + if keepRow["kind"] != "keep" { + t.Fatalf("unexpected keep row: %#v", keepRow) + } + } + if dropRow, _ := dropRows[0].(map[string]any); dropRow != nil { + if dropRow["kind"] != "drop" { + t.Fatalf("unexpected drop row: %#v", dropRow) + } + } +} + +func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { + t.Run("command previews text and csv outputs", func(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Preview 项目", Description: "preview"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "文件预览工作流", + Description: "preview files", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "command", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "command", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{ + {Key: "COMMAND", Value: "printf 'name,score\\nalpha,1\\nbeta,2\\n' > report.csv && printf 'hello\\nworld\\n' > notes.txt"}, + }, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + completed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + commandExecution := mustNodeExecution(t, completed, "command") + report := mustArtifact(t, commandExecution, "report.csv") + notes := mustArtifact(t, commandExecution, "notes.txt") + reportPreview, _ := report.Data["preview"].(map[string]any) + notesPreview, _ := notes.Data["preview"].(map[string]any) + if reportPreview["kind"] != "table" { + t.Fatalf("expected table preview for csv artifact, got %#v", reportPreview) + } + if notesPreview["kind"] != "text" { + t.Fatalf("expected text preview for text artifact, got %#v", notesPreview) + } + }) + + t.Run("lerobot dataset summary", func(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + datasetRoot := t.TempDir() + writeFile(t, filepath.Join(datasetRoot, "meta", "info.json"), `{"robot_type":"aloha","fps":30}`) + writeFile(t, filepath.Join(datasetRoot, "meta", "stats.json"), `{"episodes":2}`) + writeFile(t, filepath.Join(datasetRoot, "meta", "tasks.jsonl"), "{\"task\":\"pick\"}\n{\"task\":\"place\"}\n") + writeFile(t, filepath.Join(datasetRoot, "data", "episode_000.parquet"), "PAR1") + writeFile(t, filepath.Join(datasetRoot, "videos", "episode_000.mp4"), "video") + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "LeRobot 项目", Description: "lerobot"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "LeRobot 概览", + Description: "lerobot summary", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "真实数据源", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: datasetRoot}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "summary", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "LeRobot 数据集概览", + Category: "ROBOTICS", + Description: "lerobot", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}}, + }, + Position: domain.Position{X: 240, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + completed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "lerobot-summary.json") + preview, _ := summary.Data["preview"].(map[string]any) + if preview["format"] != "lerobot" { + t.Fatalf("expected lerobot preview, got %#v", preview) + } + if summary.Data["datasetType"] != "lerobot" { + t.Fatalf("expected datasetType lerobot, got %#v", summary.Data) + } + }) + + t.Run("rosbag sqlite summary", func(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + datasetRoot := t.TempDir() + dbPath := filepath.Join(datasetRoot, "bag_0.db3") + createRosbagDB3(t, dbPath) + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "ROS 项目", Description: "rosbag"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "ROSBag 概览", + Description: "rosbag summary", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "真实数据源", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "summary", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "ROSBag 数据概览", + Category: "ROBOTICS", + Description: "rosbag", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}}, + }, + Position: domain.Position{X: 240, Y: 0}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + completed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "rosbag-summary.json") + preview, _ := summary.Data["preview"].(map[string]any) + if preview["format"] != "rosbag2" { + t.Fatalf("expected rosbag2 preview, got %#v", preview) + } + topics, _ := preview["topics"].([]any) + if len(topics) != 1 { + t.Fatalf("expected one topic summary, got %#v", topics) + } + }) +} + +func TestRuntimeProcessAvailableDispatchesUpToGlobalLimit(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + settings, err := dataStore.GetSettings(ctx) + if err != nil { + t.Fatalf("get settings: %v", err) + } + settings.Worker.MaxConcurrentRuns = 2 + if err := dataStore.UpsertSettings(ctx, *settings); err != nil { + t.Fatalf("upsert settings: %v", err) + } + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Dispatch 项目", Description: "dispatch"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + createWorkflow := func(name string) *domain.WorkflowRecord { + t.Helper() + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: name, + Description: "dispatch test", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{ + { + ID: "command", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "command", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + }) + if err != nil { + t.Fatalf("create workflow %s: %v", name, err) + } + return workflow + } + + workflowA := createWorkflow("并发工作流 A") + workflowB := createWorkflow("并发工作流 B") + runA, err := dataStore.CreateRun(ctx, workflowA.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run A: %v", err) + } + runB, err := dataStore.CreateRun(ctx, workflowB.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run B: %v", err) + } + + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available: %v", err) + } + + runs, err := dataStore.ListRuns(ctx, "") + if err != nil { + t.Fatalf("list runs: %v", err) + } + runningCount := 0 + for _, run := range runs { + if run.Status == domain.RunStatusRunning { + runningCount++ + } + } + if runningCount != 2 { + t.Fatalf("expected 2 running runs after dispatch, got %d", runningCount) + } + + waitForRunStatus(t, ctx, dataStore, runA.ID, domain.RunStatusCompleted, 8*time.Second) + waitForRunStatus(t, ctx, dataStore, runB.ID, domain.RunStatusCompleted, 8*time.Second) +} + +func TestRuntimeProcessAvailableRespectsWorkflowConcurrencyOverride(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + settings, err := dataStore.GetSettings(ctx) + if err != nil { + t.Fatalf("get settings: %v", err) + } + settings.Worker.MaxConcurrentRuns = 3 + if err := dataStore.UpsertSettings(ctx, *settings); err != nil { + t.Fatalf("upsert settings: %v", err) + } + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Override 项目", Description: "override"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + limitedWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "受限工作流", + Description: "limited", + Type: domain.WorkflowTypeBatch, + MaxConcurrentRuns: 1, + Nodes: []domain.WorkflowNode{{ + ID: "command", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "command", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }}, + }) + if err != nil { + t.Fatalf("create limited workflow: %v", err) + } + freeWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "自由工作流", + Description: "free", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{{ + ID: "command", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "command", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }}, + }) + if err != nil { + t.Fatalf("create free workflow: %v", err) + } + + runLimitedA, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create limited run A: %v", err) + } + runLimitedB, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create limited run B: %v", err) + } + runFree, err := dataStore.CreateRun(ctx, freeWorkflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create free run: %v", err) + } + + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available: %v", err) + } + + limitedRunAState, err := dataStore.GetRun(ctx, runLimitedA.ID) + if err != nil { + t.Fatalf("get limited run A: %v", err) + } + limitedRunBState, err := dataStore.GetRun(ctx, runLimitedB.ID) + if err != nil { + t.Fatalf("get limited run B: %v", err) + } + freeRunState, err := dataStore.GetRun(ctx, runFree.ID) + if err != nil { + t.Fatalf("get free run: %v", err) + } + if limitedRunAState.Status != domain.RunStatusRunning { + t.Fatalf("expected first limited run to be running, got %s", limitedRunAState.Status) + } + if limitedRunBState.Status != domain.RunStatusQueued { + t.Fatalf("expected second limited run to remain queued, got %s", limitedRunBState.Status) + } + if freeRunState.Status != domain.RunStatusRunning { + t.Fatalf("expected free workflow run to be running, got %s", freeRunState.Status) + } + + waitForRunStatus(t, ctx, dataStore, runLimitedA.ID, domain.RunStatusCompleted, 8*time.Second) + waitForRunStatus(t, ctx, dataStore, runFree.ID, domain.RunStatusCompleted, 8*time.Second) + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available second pass: %v", err) + } + waitForRunStatus(t, ctx, dataStore, runLimitedB.ID, domain.RunStatusCompleted, 8*time.Second) +} + +func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + settings, err := dataStore.GetSettings(ctx) + if err != nil { + t.Fatalf("get settings: %v", err) + } + settings.Worker.MaxConcurrentRuns = 3 + if err := dataStore.UpsertSettings(ctx, *settings); err != nil { + t.Fatalf("upsert settings: %v", err) + } + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Parallel DAG 项目", Description: "parallel dag"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "并行分支工作流", + Description: "parallel dag", + Type: domain.WorkflowTypeBatch, + MaxConcurrentRuns: 3, + Nodes: []domain.WorkflowNode{ + { + ID: "source", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "OSS 数据源", + Category: "IO", + Description: "source", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, + Hyperparams: []domain.HyperParam{{Key: "SOURCE_TEXT", Value: "parallel"}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + { + ID: "branch-a", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "branch a", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-a", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"a\"}' > result-a.json"}}, + }, + Position: domain.Position{X: 200, Y: -120}, + }, + { + ID: "branch-b", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "branch b", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-b", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"b\"}' > result-b.json"}}, + }, + Position: domain.Position{X: 200, Y: 0}, + }, + { + ID: "branch-c", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "branch c", + Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-c", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"c\"}' > result-c.json"}}, + }, + Position: domain.Position{X: 200, Y: 120}, + }, + }, + Edges: []domain.WorkflowEdge{ + {ID: "source-a", Source: "source", SourceHandle: "out-1", Target: "branch-a", TargetHandle: "in-1"}, + {ID: "source-b", Source: "source", SourceHandle: "out-1", Target: "branch-b", TargetHandle: "in-1"}, + {ID: "source-c", Source: "source", SourceHandle: "out-1", Target: "branch-c", TargetHandle: "in-1"}, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available: %v", err) + } + + runningRun := waitForRunningNodeCount(t, ctx, dataStore, run.ID, 3, 5*time.Second) + if runningRun.Status != domain.RunStatusRunning { + t.Fatalf("expected run to be running while branches execute, got %s", runningRun.Status) + } + + completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second) + for _, nodeID := range []string{"branch-a", "branch-b", "branch-c"} { + if execution := mustNodeExecution(t, completed, nodeID); execution.Status != domain.NodeExecutionStatusCompleted { + t.Fatalf("expected node %s to complete, got %s", nodeID, execution.Status) + } + } +} + +func TestRuntimeProcessRunFailsWhenWorkflowTimeoutReached(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Timeout 项目", Description: "timeout"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "超时工作流", + Description: "timeout", + Type: domain.WorkflowTypeBatch, + RunTimeoutSeconds: 1, + Nodes: []domain.WorkflowNode{ + { + ID: "slow-node", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "slow", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + + failed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get failed run: %v", err) + } + if failed == nil || failed.Status != domain.RunStatusFailed { + t.Fatalf("expected failed run, got %#v", failed) + } + if !strings.Contains(failed.Error, "超时") { + t.Fatalf("expected timeout error, got %q", failed.Error) + } + execution := mustNodeExecution(t, failed, "slow-node") + if execution.Status != domain.NodeExecutionStatusFailed { + t.Fatalf("expected failed node execution, got %s", execution.Status) + } + + storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID) + if err != nil { + t.Fatalf("get workflow: %v", err) + } + if storedWorkflow == nil || storedWorkflow.Status != domain.WorkflowStatusFailed { + t.Fatalf("expected failed workflow, got %#v", storedWorkflow) + } +} + +func TestRuntimeProcessRunAutoRetriesFailedWorkflows(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + markerPath := filepath.Join(t.TempDir(), "retry-marker") + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry 项目", Description: "retry"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "自动重试工作流", + Description: "retry", + Type: domain.WorkflowTypeBatch, + MaxRetryAttempts: 1, + Nodes: []domain.WorkflowNode{ + { + ID: "flaky-node", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "flaky", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{ + Key: "COMMAND", + Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json", + }}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create initial run: %v", err) + } + + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("first process once: %v", err) + } + + firstAttempt, err := dataStore.GetRun(ctx, initialRun.ID) + if err != nil { + t.Fatalf("get first attempt: %v", err) + } + if firstAttempt == nil || firstAttempt.Status != domain.RunStatusFailed { + t.Fatalf("expected first attempt to fail, got %#v", firstAttempt) + } + if firstAttempt.Attempt != 1 { + t.Fatalf("expected first attempt number to be 1, got %d", firstAttempt.Attempt) + } + + runs, err := dataStore.ListRuns(ctx, "") + if err != nil { + t.Fatalf("list runs: %v", err) + } + if len(runs) != 2 { + t.Fatalf("expected original + retry run, got %d", len(runs)) + } + + var retryRun domain.RunRecord + foundRetry := false + for _, candidate := range runs { + if candidate.ID == initialRun.ID { + continue + } + retryRun = candidate + foundRetry = true + break + } + if !foundRetry { + t.Fatalf("expected retry run to be created") + } + if retryRun.Trigger != domain.RunTriggerRetry { + t.Fatalf("expected retry trigger, got %s", retryRun.Trigger) + } + if retryRun.Attempt != 2 { + t.Fatalf("expected retry attempt number to be 2, got %d", retryRun.Attempt) + } + if retryRun.ParentRunID != initialRun.ID { + t.Fatalf("expected retry parent run %s, got %s", initialRun.ID, retryRun.ParentRunID) + } + if retryRun.Status != domain.RunStatusQueued { + t.Fatalf("expected retry run to be queued, got %s", retryRun.Status) + } + + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("second process once: %v", err) + } + + completedRetry, err := dataStore.GetRun(ctx, retryRun.ID) + if err != nil { + t.Fatalf("get retry run: %v", err) + } + if completedRetry == nil || completedRetry.Status != domain.RunStatusCompleted { + t.Fatalf("expected retry run to complete, got %#v", completedRetry) + } + + finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID) + if err != nil { + t.Fatalf("get final workflow: %v", err) + } + if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted { + t.Fatalf("expected workflow to recover and complete, got %#v", finalWorkflow) + } +} + +func TestRuntimeProcessRunFailsWhenNodeTimeoutReached(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Node Timeout 项目", Description: "node timeout"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "节点超时工作流", + Description: "node timeout", + Type: domain.WorkflowTypeBatch, + NodeTimeoutSeconds: 1, + Nodes: []domain.WorkflowNode{ + { + ID: "slow-node", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "slow", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("process once: %v", err) + } + + failed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get failed run: %v", err) + } + if failed == nil || failed.Status != domain.RunStatusFailed { + t.Fatalf("expected failed run, got %#v", failed) + } + if !strings.Contains(failed.Error, "节点执行超时") { + t.Fatalf("expected node timeout error, got %q", failed.Error) + } +} + +func TestRuntimeProcessAvailableHonorsRetryBackoff(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + markerPath := filepath.Join(t.TempDir(), "retry-backoff-marker") + + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry Backoff 项目", Description: "retry backoff"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "重试延迟工作流", + Description: "retry backoff", + Type: domain.WorkflowTypeBatch, + MaxRetryAttempts: 1, + RetryBackoffSeconds: 2, + Nodes: []domain.WorkflowNode{ + { + ID: "flaky-node", + Type: "operator", + Data: domain.WorkflowNodeData{ + Label: "Shell 命令执行", + Category: "UTIL", + Description: "flaky", + Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, + Hyperparams: []domain.HyperParam{{ + Key: "COMMAND", + Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json", + }}, + }, + Position: domain.Position{X: 0, Y: 0}, + }, + }, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + + initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create initial run: %v", err) + } + + runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t)) + if err != nil { + t.Fatalf("create runtime: %v", err) + } + if _, err := runtime.ProcessOnce(ctx); err != nil { + t.Fatalf("first process once: %v", err) + } + + runs, err := dataStore.ListRuns(ctx, "") + if err != nil { + t.Fatalf("list runs: %v", err) + } + if len(runs) != 2 { + t.Fatalf("expected original + retry run, got %d", len(runs)) + } + var retryRun domain.RunRecord + for _, candidate := range runs { + if candidate.ID != initialRun.ID { + retryRun = candidate + break + } + } + if retryRun.AvailableAt == "" { + t.Fatalf("expected retry run to carry availableAt") + } + + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available before backoff expires: %v", err) + } + stillQueued, err := dataStore.GetRun(ctx, retryRun.ID) + if err != nil { + t.Fatalf("get retry run before backoff: %v", err) + } + if stillQueued == nil || stillQueued.Status != domain.RunStatusQueued { + t.Fatalf("expected retry run to remain queued before backoff expires, got %#v", stillQueued) + } + + time.Sleep(2200 * time.Millisecond) + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available after backoff expires: %v", err) + } + completedRetry := waitForRunStatus(t, ctx, dataStore, retryRun.ID, domain.RunStatusCompleted, 8*time.Second) + if completedRetry.Attempt != 2 { + t.Fatalf("expected retry attempt number to stay 2, got %d", completedRetry.Attempt) + } +} diff --git a/k8s/droboticflow.template.yaml b/k8s/droboticflow.template.yaml new file mode 100644 index 00000000..4c7af64f --- /dev/null +++ b/k8s/droboticflow.template.yaml @@ -0,0 +1,419 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: __NAMESPACE__ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: droboticflow-worker + namespace: __NAMESPACE__ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: droboticflow-worker + namespace: __NAMESPACE__ +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: ["batch"] + resources: ["jobs/status"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: droboticflow-worker + namespace: __NAMESPACE__ +subjects: + - kind: ServiceAccount + name: droboticflow-worker + namespace: __NAMESPACE__ +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: droboticflow-worker +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: droboticflow-config + namespace: __NAMESPACE__ +data: + PORT: "3001" + WORKER_PORT: "3002" + MONGO_DB: "__MONGO_DB__" + MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__" + MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__" + WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__" + K8S_NAMESPACE: "__NAMESPACE__" + K8S_IMAGE_PULL_SECRET_NAME: "__K8S_IMAGE_PULL_SECRET_NAME__" + OPERATOR_RUNNER_IMAGE: "__OPERATOR_RUNNER_IMAGE__" + AI_API_BASE_URL: "__AI_API_BASE_URL__" + AI_MODEL: "__AI_MODEL__" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-init + namespace: __NAMESPACE__ +data: + init-app-user.sh: | + #!/bin/sh + set -eu + mongosh --quiet <=6.9.0" } }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", - "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", - "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", - "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", - "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", - "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", - "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", - "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", - "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", - "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", - "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", - "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", - "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", - "cpu": [ - "loong64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", - "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", - "cpu": [ - "mips64el" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", - "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", - "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", - "cpu": [ - "riscv64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", - "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", - "cpu": [ - "s390x" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", - "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", - "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", - "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", - "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", - "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", - "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", - "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", - "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", - "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", - "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@google/genai": { - "version": "1.48.0", - "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.48.0.tgz", - "integrity": "sha512-plonYK4ML2PrxsRD9SeqmFt76eREWkQdPCglOA6aYDzL1AAbE+7PUnT54SvpWGfws13L0AZEqGSpL7+1IPnTxQ==", - "license": "Apache-2.0", - "dependencies": { - "google-auth-library": "^10.3.0", - "p-retry": "^4.6.2", - "protobufjs": "^7.5.4", - "ws": "^8.18.0" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@modelcontextprotocol/sdk": "^1.25.2" - }, - "peerDependenciesMeta": { - "@modelcontextprotocol/sdk": { - "optional": true - } - } - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -783,70 +342,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", - "license": "BSD-3-Clause" - }, "node_modules/@reactflow/background": { "version": "11.3.14", "resolved": "https://registry.npmjs.org/@reactflow/background/-/background-11.3.14.tgz", @@ -1626,27 +1121,6 @@ "@babel/types": "^7.28.2" } }, - "node_modules/@types/body-parser": { - "version": "1.19.6", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", - "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/d3": { "version": "7.4.3", "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", @@ -1912,114 +1386,22 @@ "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", "license": "MIT" }, - "node_modules/@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.19.8", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", - "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*", - "@types/send": "*" - } - }, "node_modules/@types/geojson": { "version": "7946.0.16", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", "license": "MIT" }, - "node_modules/@types/http-errors": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", - "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/node": { "version": "22.19.17", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", + "devOptional": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } }, - "node_modules/@types/qs": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.0.tgz", - "integrity": "sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/range-parser": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", - "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", - "license": "MIT" - }, - "node_modules/@types/send": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", - "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/serve-static": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", - "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "<1" - } - }, - "node_modules/@types/serve-static/node_modules/@types/send": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", - "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/mime": "^1", - "@types/node": "*" - } - }, "node_modules/@types/use-sync-external-store": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz", @@ -2046,34 +1428,22 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "license": "MIT", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "color-convert": "^2.0.1" }, "engines": { - "node": ">= 0.6" + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, "node_modules/autoprefixer": { "version": "10.4.27", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.27.tgz", @@ -2111,26 +1481,6 @@ "postcss": "^8.1.0" } }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/baseline-browser-mapping": { "version": "2.10.16", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.16.tgz", @@ -2143,54 +1493,6 @@ "node": ">=6.0.0" } }, - "node_modules/bignumber.js": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", - "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/body-parser": { - "version": "1.20.4", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", - "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.14.0", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/body-parser/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/browserslist": { "version": "4.28.2", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", @@ -2224,50 +1526,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", - "license": "BSD-3-Clause" - }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/caniuse-lite": { "version": "1.0.30001786", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001786.tgz", @@ -2288,12 +1546,120 @@ ], "license": "CC-BY-4.0" }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/classcat": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.5.tgz", "integrity": "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==", "license": "MIT" }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -2303,25 +1669,49 @@ "node": ">=6" } }, - "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "license": "MIT", "dependencies": { - "safe-buffer": "5.2.1" + "color-name": "~1.1.4" }, "engines": { - "node": ">= 0.6" + "node": ">=7.0.0" } }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concurrently": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-9.2.1.tgz", + "integrity": "sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng==", + "dev": true, "license": "MIT", + "dependencies": { + "chalk": "4.1.2", + "rxjs": "7.8.2", + "shell-quote": "1.8.3", + "supports-color": "8.1.1", + "tree-kill": "1.2.2", + "yargs": "17.7.2" + }, + "bin": { + "conc": "dist/bin/concurrently.js", + "concurrently": "dist/bin/concurrently.js" + }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" } }, "node_modules/convert-source-map": { @@ -2330,21 +1720,6 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "license": "MIT" }, - "node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", - "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", - "license": "MIT" - }, "node_modules/d3-array": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", @@ -2542,15 +1917,6 @@ "lodash": "^4.17.15" } }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -2574,25 +1940,6 @@ "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==", "license": "MIT" }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -2602,62 +1949,12 @@ "node": ">=8" } }, - "node_modules/dotenv": { - "version": "17.4.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.1.tgz", - "integrity": "sha512-k8DaKGP6r1G30Lx8V4+pCsLzKr8vLmV2paqEj1Y55GdAgJuIqpRp5FfajGF8KtwMxCz9qJc6wUIJnm053d/WCw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "license": "MIT" - }, "node_modules/electron-to-chromium": { "version": "1.5.331", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.331.tgz", "integrity": "sha512-IbxXrsTlD3hRodkLnbxAPP4OuJYdWCeM3IOdT+CpcMoIwIoDfCmRpEtSPfwBXxVkg9xmBeY7Lz2Eo2TDn/HC3Q==", "license": "ISC" }, - "node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/enhanced-resolve": { "version": "5.20.1", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.20.1.tgz", @@ -2671,36 +1968,6 @@ "node": ">=10.13.0" } }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/es-toolkit": { "version": "1.45.1", "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.45.1.tgz", @@ -2711,48 +1978,6 @@ "benchmarks" ] }, - "node_modules/esbuild": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", - "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", - "devOptional": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.7", - "@esbuild/android-arm": "0.27.7", - "@esbuild/android-arm64": "0.27.7", - "@esbuild/android-x64": "0.27.7", - "@esbuild/darwin-arm64": "0.27.7", - "@esbuild/darwin-x64": "0.27.7", - "@esbuild/freebsd-arm64": "0.27.7", - "@esbuild/freebsd-x64": "0.27.7", - "@esbuild/linux-arm": "0.27.7", - "@esbuild/linux-arm64": "0.27.7", - "@esbuild/linux-ia32": "0.27.7", - "@esbuild/linux-loong64": "0.27.7", - "@esbuild/linux-mips64el": "0.27.7", - "@esbuild/linux-ppc64": "0.27.7", - "@esbuild/linux-riscv64": "0.27.7", - "@esbuild/linux-s390x": "0.27.7", - "@esbuild/linux-x64": "0.27.7", - "@esbuild/netbsd-arm64": "0.27.7", - "@esbuild/netbsd-x64": "0.27.7", - "@esbuild/openbsd-arm64": "0.27.7", - "@esbuild/openbsd-x64": "0.27.7", - "@esbuild/openharmony-arm64": "0.27.7", - "@esbuild/sunos-x64": "0.27.7", - "@esbuild/win32-arm64": "0.27.7", - "@esbuild/win32-ia32": "0.27.7", - "@esbuild/win32-x64": "0.27.7" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -2762,94 +1987,12 @@ "node": ">=6" } }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "license": "MIT" - }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/eventemitter3": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", "license": "MIT" }, - "node_modules/express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.3", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.14.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/express/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/express/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -2867,83 +2010,6 @@ } } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/finalhandler": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", - "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "statuses": "~2.0.2", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/finalhandler/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/finalhandler/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/fraction.js": { "version": "5.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", @@ -2985,15 +2051,6 @@ } } }, - "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -3008,43 +2065,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gaxios": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", - "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "node-fetch": "^3.3.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/gcp-metadata": { - "version": "8.1.2", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", - "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^7.0.0", - "google-logging-utils": "^1.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -3054,92 +2074,14 @@ "node": ">=6.9.0" } }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/get-tsconfig": { - "version": "4.13.7", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.7.tgz", - "integrity": "sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, - "node_modules/google-auth-library": { - "version": "10.6.2", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", - "integrity": "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^7.1.4", - "gcp-metadata": "8.1.2", - "google-logging-utils": "1.1.3", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/google-logging-utils": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", - "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "node": "6.* || 8.* || >= 10.*" } }, "node_modules/graceful-fs": { @@ -3157,73 +2099,14 @@ "lodash": "^4.17.15" } }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "license": "MIT", "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" + "node": ">=8" } }, "node_modules/immer": { @@ -3236,12 +2119,6 @@ "url": "https://opencollective.com/immer" } }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, "node_modules/internmap": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", @@ -3251,13 +2128,14 @@ "node": ">=12" } }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, "license": "MIT", "engines": { - "node": ">= 0.10" + "node": ">=8" } }, "node_modules/jiti": { @@ -3287,15 +2165,6 @@ "node": ">=6" } }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -3308,27 +2177,6 @@ "node": ">=6" } }, - "node_modules/jwa": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", - "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "^1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", - "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", - "license": "MIT", - "dependencies": { - "jwa": "^2.0.1", - "safe-buffer": "^5.0.1" - } - }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", @@ -3584,12 +2432,6 @@ "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, - "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -3617,75 +2459,6 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/motion": { "version": "12.38.0", "resolved": "https://registry.npmjs.org/motion/-/motion-12.38.0.tgz", @@ -3751,111 +2524,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/node-releases": { "version": "2.0.37", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.37.tgz", "integrity": "sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg==", "license": "MIT" }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/p-retry": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", - "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", - "license": "MIT", - "dependencies": { - "@types/retry": "0.12.0", - "retry": "^0.13.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-to-regexp": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", - "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", - "license": "MIT" - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -3874,6 +2548,53 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.8", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", @@ -3909,82 +2630,6 @@ "dev": true, "license": "MIT" }, - "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/qs": { - "version": "6.14.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", - "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/react": { "version": "19.2.4", "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", @@ -4159,31 +2804,22 @@ "redux": "^5.0.0" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/reselect": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.1.1.tgz", "integrity": "sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w==", "license": "MIT" }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "devOptional": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/rollup": { "version": "4.60.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.1.tgz", @@ -4228,31 +2864,15 @@ "fsevents": "~2.3.2" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + } }, "node_modules/scheduler": { "version": "0.27.0", @@ -4269,137 +2889,18 @@ "semver": "bin/semver.js" } }, - "node_modules/send": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", - "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.1", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "~2.4.1", - "range-parser": "~1.2.1", - "statuses": "~2.0.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/send/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/serve-static": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", - "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", - "license": "MIT", - "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "~0.19.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/set-cookie-parser": { "version": "2.7.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", "license": "MIT" }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "license": "ISC" - }, - "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "node_modules/shell-quote": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz", + "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", + "dev": true, "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, "engines": { "node": ">= 0.4" }, @@ -4416,13 +2917,20 @@ "node": ">=0.10.0" } }, - "node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, "engines": { - "node": ">= 0.8" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" } }, "node_modules/tailwind-merge": { @@ -4476,13 +2984,14 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "dev": true, "license": "MIT", - "engines": { - "node": ">=0.6" + "bin": { + "tree-kill": "cli.js" } }, "node_modules/tslib": { @@ -4491,39 +3000,6 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, - "node_modules/tsx": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", - "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.27.0", - "get-tsconfig": "^4.7.5" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "license": "MIT", - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/typescript": { "version": "5.8.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", @@ -4542,17 +3018,9 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "devOptional": true, "license": "MIT" }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/update-browserslist-db": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", @@ -4592,24 +3060,6 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/victory-vendor": { "version": "37.3.6", "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz", @@ -5163,34 +3613,14 @@ "@esbuild/win32-x64": "0.25.12" } }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", "engines": { - "node": ">= 8" - } - }, - "node_modules/ws": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", - "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } + "node": ">=10" } }, "node_modules/yallist": { @@ -5199,6 +3629,89 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "license": "ISC" }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/zod": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/zustand": { "version": "4.5.7", "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz", diff --git a/package.json b/package.json index ec27d799..a4d663bb 100644 --- a/package.json +++ b/package.json @@ -1,24 +1,42 @@ { - "name": "react-example", + "name": "droboticflow", "private": true, "version": "0.0.0", "type": "module", "scripts": { - "dev": "tsx server.ts", - "build": "vite build", + "dev": "concurrently -k -n api,web \"npm:dev:api\" \"npm:dev:web\"", + "dev:full": "concurrently -k -n api,worker,web \"npm:dev:api\" \"npm:dev:worker\" \"npm:dev:web\"", + "dev:api": "go run ./cmd/api", + "dev:worker": "go run ./cmd/worker", + "dev:web": "vite", + "build": "npm run build:web && npm run build:go", + "build:web": "vite build", + "build:go": "mkdir -p build && go build -o build/droboticflow-api ./cmd/api && go build -o build/droboticflow-worker ./cmd/worker", + "start": "./build/droboticflow-api", + "start:api": "./build/droboticflow-api", + "start:worker": "./build/droboticflow-worker", + "start:operator-runner": "python3 backend/operator-runner/main.py", "preview": "vite preview", - "clean": "rm -rf dist", - "lint": "tsc --noEmit" + "clean": "rm -rf dist build", + "lint": "npm run lint:web && npm run lint:go", + "lint:web": "tsc --noEmit", + "lint:go": "bash -lc 'test -z \"$(gofmt -l cmd internal)\"'", + "test": "go test ./...", + "verify": "npm run lint && npm test && npm run build", + "docker:build:api": "docker build -f Dockerfile.api -t droboticflow-api:local .", + "docker:build:worker": "docker build -f Dockerfile.worker -t droboticflow-worker:local .", + "docker:build:operator-runner": "docker build -f Dockerfile.operator-runner -t droboticflow-operator-runner:local .", + "k8s:render": "./scripts/render-manifests.sh", + "k8s:deploy": "./scripts/deploy-k8s.sh", + "smoke": "./scripts/smoke-test.sh", + "reconcile:data": "./scripts/reconcile-system-data.sh" }, "dependencies": { - "@google/genai": "^1.48.0", "@tailwindcss/vite": "^4.1.14", "@types/dagre": "^0.7.54", "@vitejs/plugin-react": "^5.0.4", "clsx": "^2.1.1", "dagre": "^0.8.5", - "dotenv": "^17.2.3", - "express": "^4.21.2", "lucide-react": "^0.546.0", "motion": "^12.23.24", "react": "^19.0.0", @@ -27,14 +45,15 @@ "reactflow": "^11.11.4", "recharts": "^3.8.1", "tailwind-merge": "^3.5.0", - "vite": "^6.2.0" + "vite": "^6.2.0", + "zod": "^4.1.11" }, "devDependencies": { - "@types/express": "^4.17.21", "@types/node": "^22.14.0", "autoprefixer": "^10.4.21", + "concurrently": "^9.1.2", + "playwright": "^1.59.1", "tailwindcss": "^4.1.14", - "tsx": "^4.21.0", "typescript": "~5.8.2", "vite": "^6.2.0" } diff --git a/scripts/build-and-push-images.sh b/scripts/build-and-push-images.sh new file mode 100755 index 00000000..89d6a49d --- /dev/null +++ b/scripts/build-and-push-images.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +REGISTRY_HOST="${REGISTRY_HOST:-ccr-29eug8s3-pub.cnc.bj.baidubce.com}" +REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-dataflow}" +IMAGE_TAG="${IMAGE_TAG:-$(git -C "$ROOT_DIR" rev-parse --short HEAD 2>/dev/null || date +%Y%m%d%H%M%S)}" +PLATFORM="${PLATFORM:-linux/amd64}" +PUSH="${PUSH:-true}" + +API_IMAGE="${API_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-api:$IMAGE_TAG}" +WORKER_IMAGE="${WORKER_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-worker:$IMAGE_TAG}" +OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-operator-runner:$IMAGE_TAG}" + +docker buildx inspect >/dev/null 2>&1 || docker buildx create --use >/dev/null + +build_image() { + local dockerfile="$1" + local image="$2" + local action_flag="--load" + + if [[ "$PUSH" == "true" ]]; then + action_flag="--push" + fi + + docker buildx build \ + --platform "$PLATFORM" \ + "$action_flag" \ + -f "$ROOT_DIR/$dockerfile" \ + -t "$image" \ + "$ROOT_DIR" +} + +build_image "Dockerfile.api" "$API_IMAGE" +build_image "Dockerfile.worker" "$WORKER_IMAGE" +build_image "Dockerfile.operator-runner" "$OPERATOR_RUNNER_IMAGE" + +printf 'API_IMAGE=%s\n' "$API_IMAGE" +printf 'WORKER_IMAGE=%s\n' "$WORKER_IMAGE" +printf 'OPERATOR_RUNNER_IMAGE=%s\n' "$OPERATOR_RUNNER_IMAGE" diff --git a/scripts/deploy-k8s.sh b/scripts/deploy-k8s.sh new file mode 100755 index 00000000..48a3e2dd --- /dev/null +++ b/scripts/deploy-k8s.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" + +REGISTRY_SERVER="${REGISTRY_SERVER:-ccr-29eug8s3-pub.cnc.bj.baidubce.com}" +REGISTRY_USERNAME="${REGISTRY_USERNAME:-}" +REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}" +CREATE_REGISTRY_SECRET="${CREATE_REGISTRY_SECRET:-true}" + +MONGO_DB="${MONGO_DB:-droboticflow}" +MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}" +MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}" +WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}" +K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}" +API_NODE_PORT="${API_NODE_PORT:-30080}" +API_REPLICAS="${API_REPLICAS:-1}" +WORKER_REPLICAS="${WORKER_REPLICAS:-1}" +API_IMAGE="${API_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-api:latest}" +WORKER_IMAGE="${WORKER_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-worker:latest}" +OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:latest}" +MONGO_IMAGE="${MONGO_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-mongo:7-amd64}" +MONGO_ROOT_USERNAME="${MONGO_ROOT_USERNAME:-}" +MONGO_ROOT_PASSWORD="${MONGO_ROOT_PASSWORD:-}" +MONGO_APP_USERNAME="${MONGO_APP_USERNAME:-}" +MONGO_APP_PASSWORD="${MONGO_APP_PASSWORD:-}" +AI_API_BASE_URL="${AI_API_BASE_URL:-https://coding.dashscope.aliyuncs.com/v1}" +AI_MODEL="${AI_MODEL:-glm-5}" +AI_API_KEY="${AI_API_KEY:-}" + +read_secret_value() { + local secret_name="$1" + local key="$2" + kubectl -n "$K8S_NAMESPACE" get secret "$secret_name" \ + -o "jsonpath={.data.$key}" 2>/dev/null | base64 --decode 2>/dev/null || true +} + +kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$K8S_NAMESPACE" + +if kubectl -n "$K8S_NAMESPACE" get statefulset mongodb >/dev/null 2>&1; then + kubectl -n "$K8S_NAMESPACE" delete statefulset mongodb --ignore-not-found --wait=true + kubectl -n "$K8S_NAMESPACE" delete pvc mongodb-data-mongodb-0 --ignore-not-found +fi + +if [[ "$CREATE_REGISTRY_SECRET" == "true" ]]; then + : "${REGISTRY_USERNAME:?REGISTRY_USERNAME is required when CREATE_REGISTRY_SECRET=true}" + : "${REGISTRY_PASSWORD:?REGISTRY_PASSWORD is required when CREATE_REGISTRY_SECRET=true}" + + kubectl -n "$K8S_NAMESPACE" create secret docker-registry droboticflow-registry \ + --docker-server="$REGISTRY_SERVER" \ + --docker-username="$REGISTRY_USERNAME" \ + --docker-password="$REGISTRY_PASSWORD" \ + --dry-run=client \ + -o yaml | kubectl apply -f - +fi + +if kubectl -n "$K8S_NAMESPACE" get secret droboticflow-mongo-secret >/dev/null 2>&1; then + MONGO_ROOT_USERNAME="${MONGO_ROOT_USERNAME:-$(read_secret_value droboticflow-mongo-secret MONGO_ROOT_USERNAME)}" + MONGO_ROOT_PASSWORD="${MONGO_ROOT_PASSWORD:-$(read_secret_value droboticflow-mongo-secret MONGO_ROOT_PASSWORD)}" + MONGO_APP_USERNAME="${MONGO_APP_USERNAME:-$(read_secret_value droboticflow-mongo-secret MONGO_APP_USERNAME)}" + MONGO_APP_PASSWORD="${MONGO_APP_PASSWORD:-$(read_secret_value droboticflow-mongo-secret MONGO_APP_PASSWORD)}" +fi + +if kubectl -n "$K8S_NAMESPACE" get secret droboticflow-ai-secret >/dev/null 2>&1; then + AI_API_KEY="${AI_API_KEY:-$(read_secret_value droboticflow-ai-secret AI_API_KEY)}" +fi + +MONGO_ROOT_USERNAME="${MONGO_ROOT_USERNAME:-root}" +MONGO_ROOT_PASSWORD="${MONGO_ROOT_PASSWORD:-$(openssl rand -hex 16)}" +MONGO_APP_USERNAME="${MONGO_APP_USERNAME:-droboticflow}" +MONGO_APP_PASSWORD="${MONGO_APP_PASSWORD:-$(openssl rand -hex 16)}" + +kubectl -n "$K8S_NAMESPACE" create secret generic droboticflow-mongo-secret \ + --from-literal=MONGO_ROOT_USERNAME="$MONGO_ROOT_USERNAME" \ + --from-literal=MONGO_ROOT_PASSWORD="$MONGO_ROOT_PASSWORD" \ + --from-literal=MONGO_APP_USERNAME="$MONGO_APP_USERNAME" \ + --from-literal=MONGO_APP_PASSWORD="$MONGO_APP_PASSWORD" \ + --dry-run=client \ + -o yaml | kubectl apply -f - + +kubectl -n "$K8S_NAMESPACE" create secret generic droboticflow-ai-secret \ + --from-literal=AI_API_KEY="$AI_API_KEY" \ + --dry-run=client \ + -o yaml | kubectl apply -f - + +env \ + K8S_NAMESPACE="$K8S_NAMESPACE" \ + MONGO_DB="$MONGO_DB" \ + MONGO_CONNECT_RETRIES="$MONGO_CONNECT_RETRIES" \ + MONGO_CONNECT_RETRY_DELAY_MS="$MONGO_CONNECT_RETRY_DELAY_MS" \ + WORKER_POLL_INTERVAL_MS="$WORKER_POLL_INTERVAL_MS" \ + K8S_IMAGE_PULL_SECRET_NAME="$K8S_IMAGE_PULL_SECRET_NAME" \ + API_NODE_PORT="$API_NODE_PORT" \ + API_REPLICAS="$API_REPLICAS" \ + WORKER_REPLICAS="$WORKER_REPLICAS" \ + AI_API_BASE_URL="$AI_API_BASE_URL" \ + AI_MODEL="$AI_MODEL" \ + AI_API_KEY="$AI_API_KEY" \ + API_IMAGE="$API_IMAGE" \ + WORKER_IMAGE="$WORKER_IMAGE" \ + OPERATOR_RUNNER_IMAGE="$OPERATOR_RUNNER_IMAGE" \ + MONGO_IMAGE="$MONGO_IMAGE" \ + "$ROOT_DIR/scripts/render-manifests.sh" | kubectl apply -f - + +kubectl -n "$K8S_NAMESPACE" rollout status deployment/mongodb --timeout=300s +kubectl -n "$K8S_NAMESPACE" rollout status deployment/droboticflow-api --timeout=300s +kubectl -n "$K8S_NAMESPACE" rollout status deployment/droboticflow-worker --timeout=300s + +kubectl -n "$K8S_NAMESPACE" get pods +kubectl -n "$K8S_NAMESPACE" get svc droboticflow-api diff --git a/scripts/playwright-functional-check.mjs b/scripts/playwright-functional-check.mjs new file mode 100644 index 00000000..1251f24d --- /dev/null +++ b/scripts/playwright-functional-check.mjs @@ -0,0 +1,507 @@ +import { chromium } from "playwright"; + +const baseURL = (process.env.BASE_URL || process.argv[2] || "http://114.111.24.223:30080").replace(/\/+$/, ""); +const apiBaseURL = `${baseURL}`; +const suffix = Date.now().toString(36); + +const names = { + admin: `PW Admin ${suffix}`, + adminEmail: `pw-admin-${suffix}@playwright.local`, + extraUser: `PW Operator ${suffix}`, + extraUserEmail: `pw-operator-${suffix}@playwright.local`, + project: `PW Project ${suffix}`, + workflow: `PW Workflow ${suffix}`, + template: `PW Template ${suffix}`, + projectTemplate: `PW Project Template ${suffix}`, + auditProject: `PW Audit ${suffix}`, +}; + +function log(step) { + process.stdout.write(`[playwright] ${step}\n`); +} + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +async function api(path, init = {}) { + let lastError; + for (let attempt = 1; attempt <= 6; attempt += 1) { + try { + const response = await fetch(`${apiBaseURL}${path}`, { + ...init, + headers: { + "Content-Type": "application/json", + ...(init.headers || {}), + }, + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`${init.method || "GET"} ${path} failed: ${response.status} ${text}`); + } + if (!text) { + return null; + } + return JSON.parse(text); + } catch (error) { + lastError = error; + if (attempt === 6) { + break; + } + await new Promise((resolve) => setTimeout(resolve, Math.min(attempt, 3) * 1000)); + } + } + throw lastError; +} + +async function poll(fn, predicate, label, timeoutMs = 120000, intervalMs = 2000) { + const deadline = Date.now() + timeoutMs; + let lastValue; + while (Date.now() < deadline) { + lastValue = await fn(); + if (predicate(lastValue)) { + return lastValue; + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error(`Timed out waiting for ${label}. Last value: ${JSON.stringify(lastValue)}`); +} + +async function gotoWithRetry(page, url) { + let lastError; + for (let attempt = 1; attempt <= 6; attempt += 1) { + try { + await page.goto(url, { waitUntil: "networkidle" }); + return; + } catch (error) { + lastError = error; + if (attempt === 6) { + break; + } + await page.waitForTimeout(Math.min(attempt, 3) * 1000); + } + } + throw lastError; +} + +async function clickWithRetry(page, locator) { + let lastError; + for (let attempt = 1; attempt <= 6; attempt += 1) { + try { + await locator.waitFor({ state: "visible" }); + await locator.scrollIntoViewIfNeeded(); + const box = await locator.boundingBox(); + if (!box) { + throw new Error("Unable to resolve click target bounding box"); + } + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); + return; + } catch (error) { + lastError = error; + if (attempt === 6) { + break; + } + await new Promise((resolve) => setTimeout(resolve, Math.min(attempt, 3) * 500)); + } + } + throw lastError; +} + +async function cleanupTestData() { + log("cleanup test data"); + const users = (await api("/api/users")).items; + for (const user of users) { + if (String(user.email || "").endsWith("@playwright.local")) { + await api(`/api/users/${encodeURIComponent(user.id)}`, { method: "DELETE" }); + } + } + + const projects = (await api("/api/projects")).items; + for (const project of projects) { + if (String(project.name || "").startsWith("PW ")) { + await api(`/api/projects/${encodeURIComponent(project.id)}`, { method: "DELETE" }); + } + } + + const templates = (await api("/api/templates")).items; + for (const template of templates) { + if (String(template.name || "").startsWith("PW ")) { + await api(`/api/templates/${encodeURIComponent(template.id)}`, { method: "DELETE" }); + } + } + + const operators = (await api("/api/operators")).items; + for (const operator of operators) { + if (String(operator.name || "").startsWith("PW ")) { + await api(`/api/operators/${encodeURIComponent(operator.id)}`, { method: "DELETE" }); + } + } +} + +async function createAuditWorkflow(projectName, workflowName) { + const project = await api("/api/projects", { + method: "POST", + body: JSON.stringify({ + name: projectName, + description: "Created by Playwright audit setup", + }), + }); + + const workflow = await api(`/api/projects/${project.id}/workflows`, { + method: "POST", + body: JSON.stringify({ + name: workflowName, + description: "Audit workflow created by Playwright", + type: "batch", + nodes: [ + { + id: "review", + type: "operator", + data: { + label: "人工审核 (Human Review)", + category: "HUMAN", + description: "review", + inputs: [{ id: "in-1", name: "json", type: "json" }], + outputs: [{ id: "approved", name: "审核通过", type: "json" }], + }, + position: { x: 0, y: 0 }, + }, + { + id: "upload", + type: "operator", + data: { + label: "结果上传", + category: "IO", + description: "sink", + inputs: [{ id: "in-1", name: "json", type: "json" }], + outputs: [], + }, + position: { x: 200, y: 0 }, + }, + ], + edges: [{ id: "review-upload", source: "review", sourceHandle: "approved", target: "upload", targetHandle: "in-1" }], + }), + }); + + const run = await api(`/api/workflows/${workflow.id}/run`, { + method: "POST", + body: JSON.stringify({ trigger: "manual" }), + }); + + await poll( + async () => api(`/api/runs/${run.id}`), + (item) => item.status === "waiting_for_audit", + `audit run ${run.id} to enter waiting_for_audit`, + 120000, + 3000, + ); + + return { project, workflow, run }; +} + +async function fillLabeledInput(page, labelText, value) { + const group = page.locator(`label:has-text("${labelText}")`).locator(".."); + const input = group.locator("input, textarea, select").first(); + await input.click(); + await input.fill(value); +} + +async function main() { + await cleanupTestData(); + + const browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ viewport: { width: 1440, height: 1100 } }); + const page = await context.newPage(); + + try { + log("open bootstrap"); + await gotoWithRetry(page, baseURL); + const bootstrapVisible = await page + .getByText("新增真实平台用户") + .isVisible({ timeout: 5000 }) + .catch(() => false); + if (bootstrapVisible) { + await page.getByPlaceholder("输入真实姓名或账号名").fill(names.admin); + await page.getByPlaceholder("name@example.com").fill(names.adminEmail); + await page.locator("select").first().selectOption("admin"); + await page.getByRole("button", { name: "创建并进入系统" }).click(); + } + await page.getByRole("heading", { name: "DroboticFlow 仪表盘" }).waitFor(); + + log("settings save round-trip"); + await page.getByRole("link", { name: "系统配置" }).click(); + await page.getByRole("heading", { name: "系统配置" }).waitFor(); + const pollGroup = page.locator('label:has-text("轮询间隔 (ms)")').locator(".."); + const pollInput = pollGroup.locator("input"); + const originalPoll = Number(await pollInput.inputValue()); + await pollInput.fill(String(originalPoll + 1)); + await page.getByRole("button", { name: /保存配置/ }).click(); + await page.getByText("配置已保存到后端。").waitFor(); + await pollInput.fill(String(originalPoll)); + await page.getByRole("button", { name: /保存配置/ }).click(); + await page.getByText("配置已保存到后端。").waitFor(); + + log("operators create edit delete"); + await page.getByRole("link", { name: "算子管理" }).click(); + await page.getByRole("heading", { name: "算子管理" }).waitFor(); + await page.getByRole("button", { name: "注册算子" }).click(); + await page.getByPlaceholder("例如: 图像锐化").fill(`PW Operator ${suffix}`); + await page.getByPlaceholder("描述该算子的功能与用途...").fill("Playwright created operator"); + await page.getByRole("button", { name: "确认注册" }).click(); + await page.getByPlaceholder("搜索算子...").fill(`PW Operator ${suffix}`); + await page.getByText(`PW Operator ${suffix}`).waitFor(); + const operatorCard = page.locator("div").filter({ hasText: `PW Operator ${suffix}` }).first(); + await operatorCard.getByRole("button", { name: /配置详情|查看详情/ }).click(); + await page.getByPlaceholder("描述该算子的功能与用途...").fill("Playwright updated operator"); + await page.getByRole("button", { name: "保存修改" }).click(); + await page.getByPlaceholder("搜索算子...").fill(`PW Operator ${suffix}`); + await page.locator("button:has(svg.lucide-trash-2)").first().click(); + await page.getByPlaceholder("搜索算子...").fill(`PW Operator ${suffix}`); + await page.getByText("没有匹配的算子,请调整搜索条件。").waitFor(); + await page.getByPlaceholder("搜索算子...").fill(""); + + log("projects create and open"); + await page.getByRole("link", { name: "项目管理" }).click(); + await page.getByRole("heading", { name: "项目管理" }).waitFor(); + await page.getByRole("button", { name: "创建项目" }).click(); + await page.getByPlaceholder("请输入项目名称").fill(names.project); + await page.getByPlaceholder("请输入项目描述,例如:处理机械臂抓取过程中的视觉数据").fill("Playwright project"); + await page.getByRole("button", { name: "立即创建" }).click(); + await page.getByPlaceholder("搜索项目...").fill(names.project); + await page.getByText(names.project).waitFor(); + + const projectList = (await api("/api/projects")).items; + const project = projectList.find((item) => item.name === names.project); + assert(project, "created project not found via api"); + await gotoWithRetry(page, `${baseURL}/project/${project.id}`); + + log("project detail create workflow"); + await page.getByRole("heading", { name: names.project }).waitFor(); + await page.getByRole("button", { name: "新建工作流" }).click(); + await page.getByPlaceholder("例如: 实时障碍物检测").fill(names.workflow); + await page.getByRole("button", { name: "确认创建" }).click(); + await page.getByRole("button", { name: "运行工作流" }).waitFor(); + + const workflows = (await api(`/api/projects/${project.id}/workflows`)).items; + const workflow = workflows.find((item) => item.name === names.workflow); + assert(workflow, "created workflow not found via api"); + + log("seed workflow graph"); + await api(`/api/workflows/${workflow.id}`, { + method: "PUT", + body: JSON.stringify({ + nodes: [ + { + id: "source-1", + type: "operator", + data: { + label: "真实数据源", + category: "IO", + description: "Playwright source node", + inputs: [], + outputs: [{ id: "out-1", name: "manifest", type: "json" }], + hyperparams: [{ key: "SOURCE_TEXT", value: `playwright-${suffix}`, description: "直接注入文本内容" }], + }, + position: { x: 100, y: 100 }, + }, + { + id: "review-1", + type: "operator", + data: { + label: "人工审核 (Human Review)", + category: "HUMAN", + description: "Playwright review node", + inputs: [{ id: "in-1", name: "json", type: "json" }], + outputs: [ + { id: "approved", name: "审核通过", type: "json" }, + { id: "rejected", name: "审核拒绝", type: "json" }, + ], + }, + position: { x: 360, y: 100 }, + }, + { + id: "upload-1", + type: "operator", + data: { + label: "结果上传", + category: "IO", + description: "Playwright upload node", + inputs: [{ id: "in-1", name: "json", type: "json" }], + outputs: [], + }, + position: { x: 620, y: 100 }, + }, + ], + edges: [ + { + id: "source-review", + source: "source-1", + sourceHandle: "out-1", + target: "review-1", + targetHandle: "in-1", + }, + { + id: "review-upload", + source: "review-1", + sourceHandle: "approved", + target: "upload-1", + targetHandle: "in-1", + }, + ], + }), + }); + + await page.reload({ waitUntil: "networkidle" }); + await page.getByText("真实数据源").dblclick(); + const sourceGroup = page.locator('label:has-text("SOURCE_TEXT")').locator("..").locator(".."); + await sourceGroup.locator("input").fill(`playwright-ui-${suffix}`); + await page.getByRole("button", { name: "保存" }).click(); + + log("run workflow from editor without redirect"); + await page.getByRole("button", { name: "运行工作流" }).click(); + await page.getByRole("button", { name: "查看执行监控" }).waitFor(); + assert(/\/workflow\//.test(page.url()), `expected to stay on workflow editor, got ${page.url()}`); + await page.getByText(/当前节点:/).waitFor(); + await poll( + async () => page.locator(".react-flow__edge.animated").count(), + (count) => count > 0, + "animated workflow edge on editor", + 30000, + 1000, + ); + await poll( + async () => { + const runList = (await api("/api/runs")).items; + return runList.find((item) => item.workflowName === names.workflow) || null; + }, + (item) => Boolean(item?.id && item.status === "waiting_for_audit"), + `editor run for ${names.workflow} waiting_for_audit`, + 120000, + 2000, + ); + const editorRun = (await api("/api/runs")).items.find((item) => item.workflowName === names.workflow); + assert(editorRun, "editor run not found via api"); + await page.getByRole("button", { name: "查看执行监控" }).click(); + await page.waitForURL(new RegExp(`/monitoring\\?runId=${editorRun.id}`)); + await page.getByRole("heading", { name: "执行监控" }).waitFor(); + await page.getByPlaceholder("搜索运行 ID、工作流或项目...").fill(names.workflow); + await page.getByText(names.workflow).waitFor(); + const runRow = page.locator("tr").filter({ hasText: names.workflow }).first(); + await runRow.hover(); + await runRow.getByTitle("查看日志").click(); + await page.getByRole("heading", { name: "执行日志" }).waitFor(); + await page.getByRole("button", { name: "关闭" }).click(); + await runRow.hover(); + await runRow.getByTitle("查看资源").click(); + await page.getByRole("heading", { name: "资源监控" }).waitFor(); + await page.getByRole("button", { name: "关闭" }).click(); + await runRow.hover(); + await runRow.getByTitle("查看数据").click(); + await page.getByRole("heading", { name: "产出数据" }).waitFor(); + await page.getByRole("button", { name: "关闭" }).click(); + + const pendingAudit = await poll( + async () => { + const tasks = (await api("/api/audit-tasks?status=all")).items; + return tasks.find((task) => task.runId === editorRun.id && task.status === "pending") || null; + }, + (item) => Boolean(item?.id), + `audit task for editor run ${editorRun.id}`, + 120000, + 2000, + ); + await api(`/api/audit-tasks/${pendingAudit.id}/approve`, { method: "POST" }); + await poll( + async () => api(`/api/runs/${editorRun.id}`), + (item) => item.status === "completed", + `editor run ${editorRun.id} completion after approval`, + 120000, + 2000, + ); + + log("export template from workflow editor"); + await runRow.dblclick(); + await page.getByRole("button", { name: "导出模板" }).click(); + await page.locator('input').filter({ has: page.locator('.. >> text=模板名称') }).first().fill(names.template); + await page.getByRole("button", { name: "创建模板" }).click(); + await page.waitForURL(/type=template/); + + log("templates search and use"); + await page.getByRole("link", { name: "工作流模板" }).click(); + await page.getByRole("heading", { name: "工作流模板" }).waitFor(); + await page.getByPlaceholder("搜索模板...").fill(names.template); + await page.getByText(names.template).waitFor(); + const templateCard = page.locator("div").filter({ hasText: names.template }).first(); + await templateCard + .locator("button") + .filter({ hasText: /^使用此模板$/ }) + .click(); + await page.getByText("选择目标项目").waitFor(); + await page.getByRole("button", { name: names.project }).click(); + await page.getByRole("button", { name: "确认并进入编辑器" }).click(); + await page.waitForURL(/\/workflow\//); + + log("project detail save as template"); + await gotoWithRetry(page, `${baseURL}/project/${project.id}`); + await page.getByRole("button", { name: "保存为模板" }).click(); + await page.getByRole("heading", { name: "保存为模板" }).waitFor(); + await page.locator("select").last().selectOption(workflow.id); + await page.locator("input").last().fill(names.projectTemplate); + await page.getByRole("button", { name: "保存模板" }).click(); + await page.waitForURL(/type=template/); + + log("users create edit delete"); + await page.getByRole("link", { name: "用户管理" }).click(); + await page.getByRole("heading", { name: "用户管理" }).waitFor(); + await clickWithRetry(page, page.locator("button").filter({ hasText: /^创建用户$/ }).first()); + await page.getByRole("heading", { name: "创建用户" }).waitFor(); + await fillLabeledInput(page, "用户名", names.extraUser); + await fillLabeledInput(page, "邮箱地址", names.extraUserEmail); + await page.locator('label:has-text("角色权限")').locator("..").locator("select").selectOption("operator"); + await clickWithRetry(page, page.locator("button").filter({ hasText: /^创建用户$/ }).last()); + await page.getByText(names.extraUser).waitFor(); + const userRow = page.locator("tr").filter({ hasText: names.extraUser }).first(); + await userRow.getByRole("button", { name: "编辑" }).click(); + await page.locator('label:has-text("角色权限")').locator("..").locator("select").selectOption("auditor"); + await page.getByRole("button", { name: "保存修改" }).click(); + await userRow.getByRole("button", { name: "删除" }).click(); + await page.waitForTimeout(500); + + log("audit approve and reject"); + const auditOne = await createAuditWorkflow(`${names.auditProject} A`, `PW Audit Workflow A ${suffix}`); + const auditTwo = await createAuditWorkflow(`${names.auditProject} B`, `PW Audit Workflow B ${suffix}`); + await gotoWithRetry(page, `${baseURL}/audit`); + await page.getByRole("heading", { name: /任务审核/ }).waitFor(); + await page.getByText(`${names.auditProject} A`).click(); + await page.getByRole("button", { name: /审核通过/ }).click(); + await page.getByText(`${names.auditProject} B`).click(); + await page.getByRole("button", { name: /审核拒绝/ }).click(); + await page.getByRole("button", { name: "已处理" }).click(); + + await poll( + async () => api(`/api/runs/${auditOne.run.id}`), + (item) => item.status === "completed", + `audit run ${auditOne.run.id} approved completion`, + ); + await poll( + async () => api(`/api/runs/${auditTwo.run.id}`), + (item) => item.status === "completed", + `audit run ${auditTwo.run.id} rejected completion`, + ); + + log("functional check completed"); + } catch (error) { + await page.screenshot({ path: `/tmp/droboticflow-playwright-failure-${suffix}.png`, fullPage: true }).catch(() => {}); + throw error; + } finally { + await context.close(); + await browser.close(); + } +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/playwright-preview-operators-check.mjs b/scripts/playwright-preview-operators-check.mjs new file mode 100644 index 00000000..ff9e4aba --- /dev/null +++ b/scripts/playwright-preview-operators-check.mjs @@ -0,0 +1,605 @@ +import { chromium } from "playwright"; + +const baseURL = (process.env.BASE_URL || process.argv[2] || "http://114.111.24.223:30080").replace(/\/+$/, ""); +const suffix = Date.now().toString(36); +const prefix = `PWX ${suffix}`; + +const names = { + user: `PWX Admin ${suffix}`, + email: `pwx-admin-${suffix}@playwright.local`, + project: `${prefix} Project`, + previewWorkflow: `${prefix} Preview`, + dataOpsWorkflow: `${prefix} DataOps`, + csvWorkflow: `${prefix} CSV`, + scanWorkflow: `${prefix} Scan`, + rosbagWorkflow: `${prefix} ROSBag`, + lerobotWorkflow: `${prefix} LeRobot`, +}; + +const requiredOperators = [ + "目录扫描", + "CSV 转 JSON", + "JSON 转 CSV", + "JSON 字段提取", + "JSON 过滤", + "JSON 映射重命名", + "文本模板渲染", + "条件分流", + "LeRobot 数据集概览", + "ROSBag 数据概览", + "Shell 命令执行", +]; + +function log(step) { + process.stdout.write(`[preview-check] ${step}\n`); +} + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +async function api(path, init = {}) { + let lastError; + for (let attempt = 1; attempt <= 6; attempt += 1) { + try { + const response = await fetch(`${baseURL}${path}`, { + ...init, + headers: { + "Content-Type": "application/json", + ...(init.headers || {}), + }, + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`${init.method || "GET"} ${path} failed: ${response.status} ${text}`); + } + if (!text) { + return null; + } + return JSON.parse(text); + } catch (error) { + lastError = error; + if (attempt === 6) { + break; + } + await new Promise((resolve) => setTimeout(resolve, Math.min(attempt, 3) * 1000)); + } + } + throw lastError; +} + +async function poll(fn, predicate, label, timeoutMs = 180000, intervalMs = 2000) { + const deadline = Date.now() + timeoutMs; + let lastValue; + while (Date.now() < deadline) { + lastValue = await fn(); + if (predicate(lastValue)) { + return lastValue; + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error(`Timed out waiting for ${label}. Last value: ${JSON.stringify(lastValue)}`); +} + +async function gotoWithRetry(page, url) { + let lastError; + for (let attempt = 1; attempt <= 6; attempt += 1) { + try { + await page.goto(url, { waitUntil: "networkidle" }); + return; + } catch (error) { + lastError = error; + if (attempt === 6) { + break; + } + await page.waitForTimeout(Math.min(attempt, 3) * 1000); + } + } + throw lastError; +} + +async function cleanupTestData() { + log("cleanup previous PWX test data"); + const projects = (await api("/api/projects")).items; + for (const project of projects) { + if (String(project.name || "").startsWith("PWX ")) { + await api(`/api/projects/${encodeURIComponent(project.id)}`, { method: "DELETE" }); + } + } +} + +async function ensureUIReady(page) { + await gotoWithRetry(page, baseURL); + const bootstrapVisible = await page.getByText("新增真实平台用户").isVisible({ timeout: 5000 }).catch(() => false); + if (bootstrapVisible) { + await page.getByPlaceholder("输入真实姓名或账号名").fill(names.user); + await page.getByPlaceholder("name@example.com").fill(names.email); + await page.locator("select").first().selectOption("admin"); + await page.getByRole("button", { name: "创建并进入系统" }).click(); + } + await page.getByRole("heading", { name: "DroboticFlow 仪表盘" }).waitFor(); +} + +async function createProject() { + return api("/api/projects", { + method: "POST", + body: JSON.stringify({ + name: names.project, + description: "Created by preview/operator Playwright check", + }), + }); +} + +async function createWorkflow(projectId, name, nodes, edges) { + return api(`/api/projects/${projectId}/workflows`, { + method: "POST", + body: JSON.stringify({ + name, + description: `Created by ${name}`, + type: "batch", + nodes, + edges, + }), + }); +} + +async function runWorkflow(workflowId) { + const queued = await api(`/api/workflows/${workflowId}/run`, { + method: "POST", + body: JSON.stringify({ trigger: "manual" }), + }); + const finished = await poll( + async () => api(`/api/runs/${queued.id}`), + (run) => ["completed", "failed", "cancelled", "waiting_for_audit"].includes(run.status), + `run ${queued.id} to finish`, + 180000, + 3000, + ); + assert(finished.status === "completed", `Run ${queued.id} finished with ${finished.status}: ${finished.error || "unknown error"}`); + return finished; +} + +async function createWorkflows(projectId) { + const previewWorkflow = await createWorkflow( + projectId, + names.previewWorkflow, + [ + { + id: "generate", + type: "operator", + data: { + label: "Shell 命令执行", + category: "UTIL", + description: "Generate preview artifacts", + inputs: [], + outputs: [{ id: "out-1", name: "result", type: "file" }], + hyperparams: [ + { + key: "COMMAND", + value: "printf 'name,score\\nalpha,1\\nbeta,2\\n' > report.csv && printf 'hello\\nworld\\n' > notes.txt", + }, + ], + }, + position: { x: 0, y: 0 }, + }, + ], + [], + ); + + const dataOpsWorkflow = await createWorkflow( + projectId, + names.dataOpsWorkflow, + [ + { + id: "generate", + type: "operator", + data: { + label: "Shell 命令执行", + category: "UTIL", + description: "Generate json payload", + inputs: [], + outputs: [{ id: "out-1", name: "payload", type: "json" }], + hyperparams: [ + { + key: "COMMAND", + value: "printf '{\"device\":{\"id\":\"robot-1\",\"status\":\"ok\"},\"items\":[{\"kind\":\"keep\",\"value\":1},{\"kind\":\"drop\",\"value\":2}]}' > payload.json", + }, + ], + }, + position: { x: 0, y: 0 }, + }, + { + id: "extract", + type: "operator", + data: { + label: "JSON 字段提取", + category: "DATA", + description: "Extract device id", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "extracted", type: "json" }], + hyperparams: [{ key: "FIELD_PATH", value: "device.id" }], + }, + position: { x: 220, y: -160 }, + }, + { + id: "filter", + type: "operator", + data: { + label: "JSON 过滤", + category: "DATA", + description: "Filter keep rows", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "filtered", type: "json" }], + hyperparams: [ + { key: "FIELD_PATH", value: "kind" }, + { key: "OPERATOR", value: "eq" }, + { key: "EXPECTED_JSON", value: "\"keep\"" }, + ], + }, + position: { x: 220, y: -20 }, + }, + { + id: "map", + type: "operator", + data: { + label: "JSON 映射重命名", + category: "DATA", + description: "Rename fields", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "mapped", type: "json" }], + hyperparams: [{ key: "FIELD_MAP_JSON", value: "{\"kind\":\"label\",\"value\":\"score\"}" }], + }, + position: { x: 220, y: 120 }, + }, + { + id: "csv", + type: "operator", + data: { + label: "JSON 转 CSV", + category: "DATA", + description: "Export csv", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "csv", type: "file" }], + }, + position: { x: 460, y: 120 }, + }, + { + id: "route", + type: "operator", + data: { + label: "条件分流", + category: "LOGIC", + description: "Route rows", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [ + { id: "matched", name: "matched", type: "json" }, + { id: "unmatched", name: "unmatched", type: "json" }, + ], + hyperparams: [ + { key: "FIELD_PATH", value: "kind" }, + { key: "OPERATOR", value: "eq" }, + { key: "EXPECTED_JSON", value: "\"keep\"" }, + ], + }, + position: { x: 220, y: 260 }, + }, + { + id: "keep-upload", + type: "operator", + data: { + label: "结果上传", + category: "IO", + description: "matched sink", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [], + }, + position: { x: 500, y: 220 }, + }, + { + id: "drop-upload", + type: "operator", + data: { + label: "结果上传", + category: "IO", + description: "unmatched sink", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [], + }, + position: { x: 500, y: 320 }, + }, + ], + [ + { id: "generate-extract", source: "generate", sourceHandle: "out-1", target: "extract", targetHandle: "in-1" }, + { id: "generate-filter", source: "generate", sourceHandle: "out-1", target: "filter", targetHandle: "in-1" }, + { id: "generate-map", source: "generate", sourceHandle: "out-1", target: "map", targetHandle: "in-1" }, + { id: "generate-route", source: "generate", sourceHandle: "out-1", target: "route", targetHandle: "in-1" }, + { id: "map-csv", source: "map", sourceHandle: "out-1", target: "csv", targetHandle: "in-1" }, + { id: "route-keep", source: "route", sourceHandle: "matched", target: "keep-upload", targetHandle: "in-1" }, + { id: "route-drop", source: "route", sourceHandle: "unmatched", target: "drop-upload", targetHandle: "in-1" }, + ], + ); + + const csvWorkflow = await createWorkflow( + projectId, + names.csvWorkflow, + [ + { + id: "generate", + type: "operator", + data: { + label: "Shell 命令执行", + category: "UTIL", + description: "Generate csv", + inputs: [], + outputs: [{ id: "out-1", name: "table", type: "file" }], + hyperparams: [ + { + key: "COMMAND", + value: "printf 'col_a,col_b\\n1,2\\n3,4\\n' > table.csv", + }, + ], + }, + position: { x: 0, y: 0 }, + }, + { + id: "convert", + type: "operator", + data: { + label: "CSV 转 JSON", + category: "DATA", + description: "csv to json", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "table", type: "json" }], + }, + position: { x: 240, y: 0 }, + }, + ], + [{ id: "generate-convert", source: "generate", sourceHandle: "out-1", target: "convert", targetHandle: "in-1" }], + ); + + const scanWorkflow = await createWorkflow( + projectId, + names.scanWorkflow, + [ + { + id: "generate", + type: "operator", + data: { + label: "Shell 命令执行", + category: "UTIL", + description: "Generate directory tree", + inputs: [], + outputs: [{ id: "out-1", name: "dir", type: "json" }], + hyperparams: [ + { + key: "COMMAND", + value: "mkdir -p scan-root && printf 'alpha' > scan-root/a.txt && printf '{\"beta\":1}' > scan-root/b.json && printf '{\"sourcePath\":\"'\"$OUTPUT_DIR\"'/scan-root\"}' > manifest.json", + }, + ], + }, + position: { x: 0, y: 0 }, + }, + { + id: "scan", + type: "operator", + data: { + label: "目录扫描", + category: "IO", + description: "directory scan", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "summary", type: "json" }], + }, + position: { x: 240, y: 0 }, + }, + ], + [{ id: "generate-scan", source: "generate", sourceHandle: "out-1", target: "scan", targetHandle: "in-1" }], + ); + + const rosbagWorkflow = await createWorkflow( + projectId, + names.rosbagWorkflow, + [ + { + id: "generate", + type: "operator", + data: { + label: "Shell 命令执行", + category: "UTIL", + description: "Generate rosbag db3", + inputs: [], + outputs: [{ id: "out-1", name: "db3", type: "file" }], + hyperparams: [ + { + key: "COMMAND", + value: "python3 - <<'PY'\nimport sqlite3\nconn = sqlite3.connect('bag_0.db3')\ncur = conn.cursor()\ncur.execute('create table topics (id integer primary key, name text, type text, serialization_format text)')\ncur.execute('create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)')\ncur.execute(\"insert into topics (id, name, type, serialization_format) values (1, '/camera', 'sensor_msgs/msg/Image', 'cdr')\")\ncur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 100, X'01')\")\ncur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 200, X'02')\")\nconn.commit()\nconn.close()\nPY", + }, + ], + }, + position: { x: 0, y: 0 }, + }, + { + id: "summary", + type: "operator", + data: { + label: "ROSBag 数据概览", + category: "ROBOTICS", + description: "rosbag summary", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "summary", type: "json" }], + }, + position: { x: 240, y: 0 }, + }, + ], + [{ id: "generate-summary", source: "generate", sourceHandle: "out-1", target: "summary", targetHandle: "in-1" }], + ); + + const lerobotWorkflow = await createWorkflow( + projectId, + names.lerobotWorkflow, + [ + { + id: "generate", + type: "operator", + data: { + label: "Shell 命令执行", + category: "UTIL", + description: "Generate lerobot dataset", + inputs: [], + outputs: [{ id: "out-1", name: "dataset", type: "file" }], + hyperparams: [ + { + key: "COMMAND", + value: "mkdir -p dataset/meta dataset/data dataset/videos && printf '{\"robot_type\":\"aloha\",\"fps\":30}' > dataset/meta/info.json && printf '{\"episodes\":2}' > dataset/meta/stats.json && printf '{\"task\":\"pick\"}\\n{\"task\":\"place\"}\\n' > dataset/meta/tasks.jsonl && printf 'PAR1' > dataset/data/episode_000.parquet && printf 'video' > dataset/videos/episode_000.mp4", + }, + ], + }, + position: { x: 0, y: 0 }, + }, + { + id: "summary", + type: "operator", + data: { + label: "LeRobot 数据集概览", + category: "ROBOTICS", + description: "lerobot summary", + inputs: [{ id: "in-1", name: "input", type: "json" }], + outputs: [{ id: "out-1", name: "summary", type: "json" }], + }, + position: { x: 240, y: 0 }, + }, + ], + [{ id: "generate-summary", source: "generate", sourceHandle: "out-1", target: "summary", targetHandle: "in-1" }], + ); + + return { + previewWorkflow, + dataOpsWorkflow, + csvWorkflow, + scanWorkflow, + rosbagWorkflow, + lerobotWorkflow, + }; +} + +function artifactByName(items, name) { + const item = items.find((artifact) => artifact.name === name); + assert(item, `Missing artifact ${name}`); + return item; +} + +async function openArtifactModal(page, workflowName) { + await page.getByRole("link", { name: "执行监控" }).click(); + await page.getByRole("heading", { name: "执行监控" }).waitFor(); + const search = page.getByPlaceholder("搜索运行 ID、工作流或项目..."); + await search.fill(workflowName); + const row = page.locator("tr").filter({ hasText: workflowName }).first(); + await row.waitFor({ state: "visible" }); + await row.locator('button[title="查看数据"]').click(); + await page.getByText("产出数据").waitFor(); +} + +async function closeArtifactModal(page) { + await page.getByRole("button", { name: "关闭" }).click(); + await page.getByRole("button", { name: "关闭" }).waitFor({ state: "hidden" }); +} + +async function main() { + await cleanupTestData(); + + log("verify operator catalog"); + const operators = (await api("/api/operators")).items; + for (const name of requiredOperators) { + assert(operators.some((item) => item.name === name), `Missing built-in operator ${name}`); + } + + log("create workflows"); + const project = await createProject(); + const workflows = await createWorkflows(project.id); + + log("run preview workflow"); + const previewRun = await runWorkflow(workflows.previewWorkflow.id); + const previewArtifacts = (await api(`/api/runs/${previewRun.id}/artifacts`)).items; + assert(artifactByName(previewArtifacts, "report.csv").data.preview.kind === "table", "report.csv preview should be table"); + assert(artifactByName(previewArtifacts, "notes.txt").data.preview.kind === "text", "notes.txt preview should be text"); + + log("run data ops workflow"); + const dataOpsRun = await runWorkflow(workflows.dataOpsWorkflow.id); + const dataOpsArtifacts = (await api(`/api/runs/${dataOpsRun.id}/artifacts`)).items; + assert(artifactByName(dataOpsArtifacts, "extracted.json").data.value === "robot-1", "JSON extract should return robot-1"); + assert(artifactByName(dataOpsArtifacts, "filtered.json").data.count === 1, "JSON filter should keep one row"); + const mapped = artifactByName(dataOpsArtifacts, "mapped.json").data.items; + assert(Array.isArray(mapped) && mapped[0].label === "keep" && mapped[0].score === 1, "JSON map should rename fields"); + assert(artifactByName(dataOpsArtifacts, "rows.csv").data.preview.kind === "table", "JSON to CSV should produce previewable csv"); + assert(artifactByName(dataOpsArtifacts, "matched.json").portId === "matched", "matched artifact should carry portId"); + assert(artifactByName(dataOpsArtifacts, "unmatched.json").portId === "unmatched", "unmatched artifact should carry portId"); + + log("run csv conversion workflow"); + const csvRun = await runWorkflow(workflows.csvWorkflow.id); + const csvArtifacts = (await api(`/api/runs/${csvRun.id}/artifacts`)).items; + assert(artifactByName(csvArtifacts, "table.json").data.rowCount === 2, "CSV to JSON should convert two rows"); + + log("run directory scan workflow"); + const scanRun = await runWorkflow(workflows.scanWorkflow.id); + const scanArtifacts = (await api(`/api/runs/${scanRun.id}/artifacts`)).items; + assert(artifactByName(scanArtifacts, "directory-scan.json").data.summary.fileCount >= 2, "Directory scan should report files"); + + log("run rosbag workflow"); + const rosbagRun = await runWorkflow(workflows.rosbagWorkflow.id); + const rosbagArtifacts = (await api(`/api/runs/${rosbagRun.id}/artifacts`)).items; + const rosbagPreview = artifactByName(rosbagArtifacts, "rosbag-summary.json").data.preview; + assert(rosbagPreview.format === "rosbag2", "ROSBag summary should detect rosbag2 sqlite"); + assert(Array.isArray(rosbagPreview.topics) && rosbagPreview.topics[0].name === "/camera", "ROSBag summary should include topics"); + + log("run lerobot workflow"); + const lerobotRun = await runWorkflow(workflows.lerobotWorkflow.id); + const lerobotArtifacts = (await api(`/api/runs/${lerobotRun.id}/artifacts`)).items; + const lerobotPreview = artifactByName(lerobotArtifacts, "lerobot-summary.json").data.preview; + assert(lerobotPreview.format === "lerobot", "LeRobot summary should detect dataset"); + assert(lerobotPreview.info.robot_type === "aloha", "LeRobot summary should expose info.json"); + + log("open browser and verify UI previews"); + const browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ viewport: { width: 1440, height: 1100 } }); + const page = await context.newPage(); + try { + await ensureUIReady(page); + + await page.getByRole("link", { name: "算子管理" }).click(); + await page.getByRole("heading", { name: "算子管理" }).waitFor(); + await page.getByPlaceholder("搜索算子...").fill("ROSBag 数据概览"); + await page.getByText("ROSBag 数据概览").waitFor(); + await page.getByPlaceholder("搜索算子...").fill("LeRobot 数据集概览"); + await page.getByText("LeRobot 数据集概览").waitFor(); + + await openArtifactModal(page, names.previewWorkflow); + await page.getByText("report.csv").click(); + await page.getByText("alpha").waitFor(); + await page.getByText("score").waitFor(); + await page.getByText("notes.txt").click(); + await page.getByText("hello").waitFor(); + await closeArtifactModal(page); + + await openArtifactModal(page, names.rosbagWorkflow); + await page.getByText("rosbag-summary.json").click(); + await page.locator("pre").filter({ hasText: '"/camera"' }).first().waitFor(); + await page.locator("pre").filter({ hasText: '"datasetType": "rosbag2"' }).first().waitFor(); + await closeArtifactModal(page); + + await openArtifactModal(page, names.lerobotWorkflow); + await page.getByText("lerobot-summary.json").click(); + await page.locator("pre").filter({ hasText: '"robot_type": "aloha"' }).first().waitFor(); + await page.locator("pre").filter({ hasText: '"format": "lerobot"' }).first().waitFor(); + await closeArtifactModal(page); + } finally { + await browser.close(); + } + + log("preview and operator check completed"); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/reconcile-system-data.sh b/scripts/reconcile-system-data.sh new file mode 100755 index 00000000..f167ae9f --- /dev/null +++ b/scripts/reconcile-system-data.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash +set -euo pipefail + +API_BASE_URL="${1:-${API_BASE_URL:-http://127.0.0.1:3001}}" +K8S_ENDPOINT="${K8S_ENDPOINT:-https://120.48.161.230:6443}" +K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" +OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-}" + +if [[ -z "$OPERATOR_RUNNER_IMAGE" && -f /tmp/droboticflow-images-latest.env ]]; then + # shellcheck disable=SC1091 + source /tmp/droboticflow-images-latest.env +fi + +: "${OPERATOR_RUNNER_IMAGE:?OPERATOR_RUNNER_IMAGE is required}" + +python3 - "$API_BASE_URL" "$K8S_ENDPOINT" "$K8S_NAMESPACE" "$OPERATOR_RUNNER_IMAGE" <<'PY' +import http.client +import json +import time +import sys +import urllib.error +import urllib.parse +import urllib.request + +api_base_url = sys.argv[1].rstrip("/") +k8s_endpoint = sys.argv[2] +k8s_namespace = sys.argv[3] +operator_runner_image = sys.argv[4] + + +def request(method, path, payload=None, expected=(200, 201, 202, 204)): + body = None + headers = {"Content-Type": "application/json"} + if payload is not None: + body = json.dumps(payload).encode("utf-8") + last_error = None + for attempt in range(1, 7): + req = urllib.request.Request(f"{api_base_url}{path}", data=body, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=20) as response: + status = response.getcode() + raw = response.read().decode("utf-8") + if status not in expected: + raise SystemExit(f"{method} {path} returned unexpected status {status}: {raw}") + if not raw: + return None + return json.loads(raw) + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8", errors="replace") + raise SystemExit(f"{method} {path} failed with {error.code}: {raw}") from error + except (urllib.error.URLError, http.client.RemoteDisconnected, TimeoutError) as error: + last_error = error + if attempt == 6: + break + time.sleep(min(attempt, 3)) + raise SystemExit(f"{method} {path} failed after retries: {last_error}") + + +def delete_each(path_prefix, items, predicate): + for item in items: + if not predicate(item): + continue + print(f"delete {path_prefix}/{item['id']}") + request("DELETE", f"{path_prefix}/{urllib.parse.quote(item['id'])}", expected=(204, 404)) + + +users = request("GET", "/api/users")["items"] +projects = request("GET", "/api/projects")["items"] +templates = request("GET", "/api/templates")["items"] +operators = request("GET", "/api/operators")["items"] +settings = request("GET", "/api/settings") + +delete_each( + "/api/users", + users, + lambda item: item.get("id") in {"1", "2", "3"} or item.get("email", "").endswith("@emboflow.ai"), +) +delete_each( + "/api/projects", + projects, + lambda item: item.get("id") in {"1", "2", "3"} or item.get("description") == "Created by scripts/smoke-test.sh", +) +delete_each( + "/api/templates", + templates, + lambda item: item.get("id", "").startswith("tpl-"), +) +delete_each( + "/api/operators", + operators, + lambda item: item.get("id", "").startswith("op-") or str(item.get("image", "")).endswith(":latest"), +) + +request( + "PUT", + "/api/settings", + { + "k8s": { + "enabled": True, + "endpoint": k8s_endpoint, + "namespace": k8s_namespace, + }, + "storage": { + "endpoint": "", + "buckets": [], + }, + "worker": { + "pollIntervalMs": settings["worker"]["pollIntervalMs"], + "maxConcurrentRuns": settings["worker"]["maxConcurrentRuns"], + }, + "defaults": settings["defaults"], + }, +) +print("updated system settings") + +builtin_operators = [ + { + "name": "真实数据源", + "type": "builtin", + "category": "IO", + "description": "从 SOURCE_TEXT、SOURCE_JSON、SOURCE_PATH 或上游输入生成真实源数据清单。", + "inputs": [], + "outputs": [{"id": "out-1", "name": "manifest", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "工作流起点。读取真实输入源并生成结构化清单,供下游节点消费。", + "hyperparams": [ + {"key": "SOURCE_TEXT", "value": "", "description": "直接注入文本内容"}, + {"key": "SOURCE_JSON", "value": "", "description": "直接注入 JSON 内容"}, + {"key": "SOURCE_PATH", "value": "", "description": "从本地路径读取真实数据"}, + ], + }, + { + "name": "模型推理", + "type": "builtin", + "category": "AI", + "description": "执行真实节点输入汇总、命令调用或模型镜像推理。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "result", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "执行通用推理节点;若提供 COMMAND 或 SCRIPT,会真实执行命令并收集输出。", + "hyperparams": [ + {"key": "COMMAND", "value": "", "description": "容器内执行的命令"}, + {"key": "SCRIPT", "value": "", "description": "可执行脚本内容"}, + ], + }, + { + "name": "数据合并", + "type": "builtin", + "category": "MATH", + "description": "将多个 JSON 输入合并成统一结果。", + "inputs": [ + {"id": "in-1", "name": "left", "type": "json", "transmission": "bus"}, + {"id": "in-2", "name": "right", "type": "json", "transmission": "bus"}, + ], + "outputs": [{"id": "out-1", "name": "merged", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "对多个 JSON 输入执行真实合并。", + }, + { + "name": "结果上传", + "type": "builtin", + "category": "IO", + "description": "收集上游真实输出并生成上传结果清单。", + "inputs": [{"id": "in-1", "name": "result", "type": "json", "transmission": "bus"}], + "outputs": [], + "image": operator_runner_image, + "skillDefinition": "复制或汇总上游产物,生成上传结果清单。", + }, + { + "name": "人工审核", + "type": "builtin", + "category": "HUMAN", + "description": "将真实输入送入人工审核并等待审核动作恢复流程。", + "inputs": [{"id": "in-1", "name": "candidate", "type": "json", "transmission": "bus"}], + "outputs": [ + {"id": "approved", "name": "approved", "type": "json", "transmission": "bus"}, + {"id": "rejected", "name": "rejected", "type": "json", "transmission": "bus"}, + ], + "image": operator_runner_image, + "skillDefinition": "基于真实输入生成审核任务并等待人工处理。", + }, + { + "name": "目录扫描", + "type": "builtin", + "category": "IO", + "description": "对真实目录做文件统计、扩展名分布和样例文件扫描。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "读取 SOURCE_PATH 或上游 source manifest,输出真实目录概览。", + "hyperparams": [ + {"key": "SOURCE_PATH", "value": "", "description": "待扫描目录"}, + ], + }, + { + "name": "CSV 转 JSON", + "type": "builtin", + "category": "DATA", + "description": "将真实 CSV/TSV 文件转换成结构化 JSON 行列表。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "table", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "读取 SOURCE_PATH 或上游 source manifest 指向的 csv/tsv 文件,输出完整 JSON 行数据。", + "hyperparams": [ + {"key": "SOURCE_PATH", "value": "", "description": "csv/tsv 文件路径"}, + ], + }, + { + "name": "JSON 转 CSV", + "type": "builtin", + "category": "DATA", + "description": "将 JSON 行数据导出为真实 CSV 文件。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "csv", "type": "file", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "把上游 JSON items/entries 或单条对象导出为真实 CSV 文件,并附带摘要。", + }, + { + "name": "JSON 字段提取", + "type": "builtin", + "category": "DATA", + "description": "按字段路径从 JSON 输入中提取指定值。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "extracted", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "根据 FIELD_PATH 从 JSON 输入中提取字段。", + "hyperparams": [ + {"key": "FIELD_PATH", "value": "", "description": "点分字段路径,例如 metrics.score"}, + ], + }, + { + "name": "JSON 过滤", + "type": "builtin", + "category": "DATA", + "description": "按字段条件过滤 JSON 列表。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "filtered", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "按 FIELD_PATH + OPERATOR + EXPECTED 条件真实过滤 JSON items。", + "hyperparams": [ + {"key": "FIELD_PATH", "value": "", "description": "待判断字段路径"}, + {"key": "OPERATOR", "value": "eq", "description": "eq/ne/contains/gt/gte/lt/lte/exists/missing"}, + {"key": "EXPECTED", "value": "", "description": "字符串比较值"}, + {"key": "EXPECTED_JSON", "value": "", "description": "JSON 格式比较值,优先于 EXPECTED"}, + ], + }, + { + "name": "JSON 映射重命名", + "type": "builtin", + "category": "DATA", + "description": "按映射表批量重命名 JSON 字段。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "mapped", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "根据 FIELD_MAP_JSON 真实执行字段名重映射。", + "hyperparams": [ + {"key": "FIELD_MAP_JSON", "value": "{}", "description": "JSON 字段映射,例如 {\"old\":\"new\"}"}, + ], + }, + { + "name": "文本模板渲染", + "type": "builtin", + "category": "UTIL", + "description": "使用上游 JSON 上下文渲染真实文本模板文件。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "rendered", "type": "file", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "支持 {{field}} 和 Python format 风格占位符,输出真实文本文件。", + "hyperparams": [ + {"key": "TEMPLATE_TEXT", "value": "", "description": "模板文本"}, + {"key": "OUTPUT_NAME", "value": "rendered.txt", "description": "输出文件名"}, + ], + }, + { + "name": "条件分流", + "type": "builtin", + "category": "LOGIC", + "description": "按字段条件将输入拆分到 matched / unmatched 两个真实输出端口。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [ + {"id": "matched", "name": "matched", "type": "json", "transmission": "bus"}, + {"id": "unmatched", "name": "unmatched", "type": "json", "transmission": "bus"}, + ], + "image": operator_runner_image, + "skillDefinition": "按 FIELD_PATH + OPERATOR + EXPECTED 条件把真实数据分流到两个输出端口。", + "hyperparams": [ + {"key": "FIELD_PATH", "value": "", "description": "待判断字段路径"}, + {"key": "OPERATOR", "value": "eq", "description": "eq/ne/contains/gt/gte/lt/lte/exists/missing"}, + {"key": "EXPECTED", "value": "", "description": "字符串比较值"}, + {"key": "EXPECTED_JSON", "value": "", "description": "JSON 格式比较值,优先于 EXPECTED"}, + ], + }, + { + "name": "LeRobot 数据集概览", + "type": "builtin", + "category": "ROBOTICS", + "description": "读取真实 LeRobot 数据集目录,输出 meta/tasks/parquet/videos 概览。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "面向 LeRobot 数据集目录的真实概览算子,适合机器人示教数据检查。", + "hyperparams": [ + {"key": "SOURCE_PATH", "value": "", "description": "LeRobot 数据集根目录"}, + ], + }, + { + "name": "ROSBag 数据概览", + "type": "builtin", + "category": "ROBOTICS", + "description": "读取真实 rosbag1 / rosbag2 / mcap / db3 数据并输出概览。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "支持 rosbag db3 topic/message 概览,以及 bag/mcap 文件摘要。", + "hyperparams": [ + {"key": "SOURCE_PATH", "value": "", "description": "bag/db3/mcap 文件或 rosbag2 目录"}, + ], + }, + { + "name": "Shell 命令执行", + "type": "builtin", + "category": "UTIL", + "description": "在 operator-runner 容器中真实执行 shell 命令并收集输出文件。", + "inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}], + "outputs": [{"id": "out-1", "name": "result", "type": "file", "transmission": "bus"}], + "image": operator_runner_image, + "skillDefinition": "执行 COMMAND 或 SCRIPT,自动收集输出目录产物并生成预览元数据。", + "hyperparams": [ + {"key": "COMMAND", "value": "", "description": "容器内执行的命令"}, + {"key": "SCRIPT", "value": "", "description": "可执行脚本内容"}, + ], + }, +] + +operators = request("GET", "/api/operators")["items"] +operators_by_name = {item["name"]: item for item in operators} + +for operator in builtin_operators: + existing = operators_by_name.get(operator["name"]) + if existing: + print(f"update operator {operator['name']}") + request("PUT", f"/api/operators/{urllib.parse.quote(existing['id'])}", operator) + else: + print(f"create operator {operator['name']}") + request("POST", "/api/operators", operator, expected=(201,)) + +print("reconciled system data") +PY diff --git a/scripts/render-manifests.sh b/scripts/render-manifests.sh new file mode 100755 index 00000000..539f81c9 --- /dev/null +++ b/scripts/render-manifests.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEMPLATE="$ROOT_DIR/k8s/droboticflow.template.yaml" + +K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" +MONGO_DB="${MONGO_DB:-droboticflow}" +MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}" +MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}" +WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}" +K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}" +API_NODE_PORT="${API_NODE_PORT:-30080}" +API_REPLICAS="${API_REPLICAS:-1}" +WORKER_REPLICAS="${WORKER_REPLICAS:-1}" +AI_API_BASE_URL="${AI_API_BASE_URL:-https://coding.dashscope.aliyuncs.com/v1}" +AI_MODEL="${AI_MODEL:-glm-5}" + +API_IMAGE="${API_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-api:latest}" +WORKER_IMAGE="${WORKER_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-worker:latest}" +OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:latest}" +MONGO_IMAGE="${MONGO_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-mongo:7}" + +rendered="$(cat "$TEMPLATE")" +rendered="${rendered//__NAMESPACE__/$K8S_NAMESPACE}" +rendered="${rendered//__MONGO_DB__/$MONGO_DB}" +rendered="${rendered//__MONGO_CONNECT_RETRIES__/$MONGO_CONNECT_RETRIES}" +rendered="${rendered//__MONGO_CONNECT_RETRY_DELAY_MS__/$MONGO_CONNECT_RETRY_DELAY_MS}" +rendered="${rendered//__WORKER_POLL_INTERVAL_MS__/$WORKER_POLL_INTERVAL_MS}" +rendered="${rendered//__K8S_IMAGE_PULL_SECRET_NAME__/$K8S_IMAGE_PULL_SECRET_NAME}" +rendered="${rendered//__API_NODE_PORT__/$API_NODE_PORT}" +rendered="${rendered//__API_REPLICAS__/$API_REPLICAS}" +rendered="${rendered//__WORKER_REPLICAS__/$WORKER_REPLICAS}" +rendered="${rendered//__AI_API_BASE_URL__/$AI_API_BASE_URL}" +rendered="${rendered//__AI_MODEL__/$AI_MODEL}" +rendered="${rendered//__API_IMAGE__/$API_IMAGE}" +rendered="${rendered//__WORKER_IMAGE__/$WORKER_IMAGE}" +rendered="${rendered//__OPERATOR_RUNNER_IMAGE__/$OPERATOR_RUNNER_IMAGE}" +rendered="${rendered//__MONGO_IMAGE__/$MONGO_IMAGE}" + +printf '%s\n' "$rendered" diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh new file mode 100755 index 00000000..146e1fe8 --- /dev/null +++ b/scripts/smoke-test.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -gt 0 ]]; then + API_BASE_URL="$1" + WORKER_BASE_URL="${2:-${WORKER_BASE_URL:-}}" +else + API_BASE_URL="${API_BASE_URL:-http://127.0.0.1:3001}" + WORKER_BASE_URL="${WORKER_BASE_URL:-http://127.0.0.1:3002}" +fi + +json_get() { + local expression="$1" + python3 -c "import json,sys; print(${expression})" +} + +api_curl() { + curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS "$@" +} + +api_health="$(api_curl "$API_BASE_URL/api/health")" +printf 'api health: %s\n' "$api_health" +printf '%s' "$api_health" | grep -q '"status":"ok"' + +project_payload="$(python3 - <<'PY' +import json +print(json.dumps({ + "name": "Smoke Test Project", + "description": "Created by scripts/smoke-test.sh", +})) +PY +)" +project_response="$(api_curl -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")" +project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')" +printf 'created project: %s\n' "$project_response" + +workflow_payload="$(python3 - <<'PY' +import json +print(json.dumps({ + "name": "Smoke Test Workflow", + "description": "End-to-end workflow created by smoke test", + "type": "batch", + "nodes": [ + { + "id": "source-1", + "type": "operator", + "data": { + "label": "真实数据源", + "category": "IO", + "description": "Use SOURCE_TEXT to emit a real manifest artifact", + "inputs": [], + "outputs": [{"id": "out-1", "name": "manifest", "type": "json"}], + "hyperparams": [{"key": "SOURCE_TEXT", "value": "smoke-test-payload"}], + }, + "position": {"x": 0, "y": 0}, + }, + { + "id": "upload-1", + "type": "operator", + "data": { + "label": "结果上传", + "category": "IO", + "description": "Persist artifacts produced by upstream nodes", + "inputs": [{"id": "in-1", "name": "manifest", "type": "json"}], + "outputs": [], + }, + "position": {"x": 240, "y": 0}, + }, + ], + "edges": [ + { + "id": "edge-source-upload", + "source": "source-1", + "sourceHandle": "out-1", + "target": "upload-1", + "targetHandle": "in-1", + } + ], +})) +PY +)" +workflow_response="$(api_curl -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")" +workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')" +printf 'created workflow: %s\n' "$workflow_response" + +queued_run="$(api_curl -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')" +run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')" +printf 'queued run: %s\n' "$queued_run" + +maybe_process_once() { + if [[ -n "${WORKER_BASE_URL}" ]] && api_curl "$WORKER_BASE_URL/worker-api/health" >/dev/null 2>&1; then + api_curl -X POST "$WORKER_BASE_URL/worker-api/process-once" >/dev/null + fi +} + +if [[ -n "${WORKER_BASE_URL}" ]] && api_curl "$WORKER_BASE_URL/worker-api/health" >/dev/null 2>&1; then + worker_health="$(api_curl "$WORKER_BASE_URL/worker-api/health")" + printf 'worker health: %s\n' "$worker_health" +fi + +deadline=$((SECONDS + 180)) +final_run="" +final_status="" + +while (( SECONDS < deadline )); do + maybe_process_once + final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")" + final_status="$(printf '%s' "$final_run" | json_get 'json.load(sys.stdin)["status"]')" + printf 'run status: %s\n' "$final_status" + + case "$final_status" in + completed) + break + ;; + failed|cancelled|waiting_for_audit) + printf 'unexpected terminal status: %s\n' "$final_run" >&2 + exit 1 + ;; + esac + + sleep 2 +done + +if [[ "$final_status" != "completed" ]]; then + printf 'run did not complete before timeout: %s\n' "$final_run" >&2 + exit 1 +fi + +logs_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/logs")" +artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")" +printf 'logs: %s\n' "$logs_response" +printf 'artifacts: %s\n' "$artifacts_response" + +printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); +if not items: raise SystemExit("expected at least one log entry")' + +printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); +if not items: raise SystemExit("expected at least one artifact")' diff --git a/server.ts b/server.ts deleted file mode 100644 index 43842ac5..00000000 --- a/server.ts +++ /dev/null @@ -1,105 +0,0 @@ -import express from "express"; -import { createServer as createViteServer } from "vite"; -import path from "path"; -import { fileURLToPath } from "url"; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -/** - * EmboFlow Backend Architecture Simulation - * - * API: Express HTTP Server (this file) - * Worker: Background process polling "Mongo" - * MongoDB: Simulated in-memory store - */ - -// Simulated MongoDB -const db = { - projects: [ - { id: "1", name: "具身智能抓取任务", description: "处理机械臂抓取过程中的视觉数据", createdAt: "2024-03-20" }, - { id: "2", name: "自动驾驶感知流程", description: "多传感器融合数据清洗与标注", createdAt: "2024-03-21" }, - ], - operators: [ - { id: "op-1", name: "数据下载", type: "builtin", category: "IO", description: "从 OSS 下载原始数据" }, - { id: "op-2", name: "图像预处理", type: "custom", category: "CV", description: "缩放、归一化图像数据" }, - { id: "op-3", name: "模型推理", type: "builtin", category: "AI", description: "运行目标检测模型" }, - { id: "op-4", name: "结果上传", type: "builtin", category: "IO", description: "将处理结果上传至 OSS" }, - ], - tasks: [] as any[], -}; - -// Worker Simulation -async function runWorker() { - console.log("[Worker] Started polling tasks from Mongo..."); - - while (true) { - const nextTask = db.tasks.find(t => t.status === "pending"); - if (nextTask) { - nextTask.status = "running"; - console.log(`[Worker] Executing task: ${nextTask.id} (${nextTask.workflowName})`); - - // Simulate long-running job (Docker/K8s) - await new Promise(resolve => setTimeout(resolve, 5000)); - - nextTask.status = "completed"; - console.log(`[Worker] Task completed: ${nextTask.id}`); - } - await new Promise(resolve => setTimeout(resolve, 2000)); // Poll every 2s - } -} - -async function startServer() { - const app = express(); - const PORT = 3000; - - app.use(express.json()); - - // API Routes - app.get("/api/health", (req, res) => { - res.json({ status: "ok", service: "EmboFlow API" }); - }); - - app.get("/api/projects", (req, res) => { - res.json(db.projects); - }); - - app.get("/api/operators", (req, res) => { - res.json(db.operators); - }); - - // Run Workflow (Push to Mongo for Worker) - app.post("/api/workflow/:id/run", (req, res) => { - const taskId = `run-${Date.now()}`; - db.tasks.push({ - id: taskId, - workflowId: req.params.id, - workflowName: req.body.name || "Unknown Workflow", - status: "pending", - createdAt: new Date().toISOString() - }); - res.json({ success: true, taskId }); - }); - - // Vite middleware for development - if (process.env.NODE_ENV !== "production") { - const vite = await createViteServer({ - server: { middlewareMode: true }, - appType: "spa", - }); - app.use(vite.middlewares); - } else { - const distPath = path.join(process.cwd(), "dist"); - app.use(express.static(distPath)); - app.get("*", (req, res) => { - res.sendFile(path.join(distPath, "index.html")); - }); - } - - app.listen(PORT, "0.0.0.0", () => { - console.log(`[API] EmboFlow Server running on http://localhost:${PORT}`); - runWorker(); // Start worker in background - }); -} - -startServer(); diff --git a/src/App.tsx b/src/App.tsx index 77bde7e7..5b3fdc81 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -3,41 +3,54 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from "react"; +import React, { Suspense, lazy } from "react"; import { BrowserRouter as Router, Routes, Route } from "react-router-dom"; import { Layout } from "./components/Layout"; +import { TimeZoneProvider } from "./context/TimeZoneContext"; import { UserProvider } from "./context/UserContext"; -import Dashboard from "./pages/Dashboard"; -import Projects from "./pages/Projects"; -import ProjectDetail from "./pages/ProjectDetail"; -import Operators from "./pages/Operators"; -import WorkflowEditor from "./pages/WorkflowEditor"; -import Monitoring from "./pages/Monitoring"; -import Settings from "./pages/Settings"; -import Templates from "./pages/Templates"; -import Users from "./pages/Users"; -import Audit from "./pages/Audit"; + +const Dashboard = lazy(() => import("./pages/Dashboard")); +const Projects = lazy(() => import("./pages/Projects")); +const ProjectDetail = lazy(() => import("./pages/ProjectDetail")); +const Operators = lazy(() => import("./pages/Operators")); +const WorkflowEditor = lazy(() => import("./pages/WorkflowEditor")); +const Monitoring = lazy(() => import("./pages/Monitoring")); +const Settings = lazy(() => import("./pages/Settings")); +const Templates = lazy(() => import("./pages/Templates")); +const Users = lazy(() => import("./pages/Users")); +const Audit = lazy(() => import("./pages/Audit")); + +function RouteFallback() { + return ( +
+ 正在加载页面... +
+ ); +} export default function App() { return ( - - - - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - - - + + + + }> + + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + + + ); } - diff --git a/src/components/AIGenerateModal.tsx b/src/components/AIGenerateModal.tsx index 4f8dcf63..6ececfe9 100644 --- a/src/components/AIGenerateModal.tsx +++ b/src/components/AIGenerateModal.tsx @@ -1,8 +1,10 @@ -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import { X, Sparkles, Loader2, AlertCircle } from "lucide-react"; import { generateWorkflowFromDescription, GeneratedWorkflow } from "../services/aiService"; -import { MOCK_OPERATORS } from "../constants"; import { cn } from "../lib/utils"; +import type { Operator } from "../types"; +import { operatorsApi } from "../services/api/operators"; +import { useAIStatus } from "../hooks/useAIStatus"; interface AIGenerateModalProps { isOpen: boolean; @@ -11,19 +13,32 @@ interface AIGenerateModalProps { } export default function AIGenerateModal({ isOpen, onClose, onGenerate }: AIGenerateModalProps) { + const aiStatus = useAIStatus(); const [description, setDescription] = useState(""); const [isGenerating, setIsGenerating] = useState(false); const [error, setError] = useState(null); + const [availableOperators, setAvailableOperators] = useState([]); + + useEffect(() => { + if (!isOpen) { + return; + } + operatorsApi.list().then(setAvailableOperators).catch((err) => { + console.error("Failed to load operators for AI generation", err); + setAvailableOperators([]); + setError("无法加载真实算子列表,暂时不能生成工作流。"); + }); + }, [isOpen]); const handleGenerate = async () => { - if (!description.trim()) return; + if (!description.trim() || availableOperators.length === 0) return; setIsGenerating(true); setError(null); try { - const workflow = await generateWorkflowFromDescription(description, MOCK_OPERATORS as any); - onGenerate(workflow, "AI 生成工作流", description); + const workflow = await generateWorkflowFromDescription(description); + onGenerate(workflow, workflow.name || "AI 生成工作流", workflow.description || description); onClose(); } catch (err) { setError(err instanceof Error ? err.message : "生成失败,请稍后重试。"); @@ -68,6 +83,20 @@ export default function AIGenerateModal({ isOpen, onClose, onGenerate }: AIGener {error} )} + + {!aiStatus.enabled && ( +
+ + {aiStatus.reason || "当前服务端未配置 AI 服务,AI 生成功能已禁用。"} +
+ )} + + {aiStatus.enabled && ( +
+ + 当前由服务端模型 `{aiStatus.model || "glm-5"}` 生成,浏览器侧不会持有 API Key。 +
+ )}
@@ -80,10 +109,10 @@ export default function AIGenerateModal({ isOpen, onClose, onGenerate }: AIGener
diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index 3ca90416..c09f5d41 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -1,28 +1,27 @@ -import React, { useState } from "react"; +import React, { useEffect, useMemo, useState } from "react"; import { NavLink } from "react-router-dom"; import { Logo } from "./Logo"; import { useUser } from "../context/UserContext"; -import { - LayoutDashboard, - FolderKanban, - FileCode, - Cpu, - Activity, - Settings, +import { + LayoutDashboard, + FolderKanban, + FileCode, + Cpu, + Activity, + Settings, Users, - Workflow, ChevronLeft, ChevronRight, User, X, Mail, Shield, - Edit3, ClipboardCheck, - LogOut + LogOut, + Loader2, } from "lucide-react"; import { cn } from "../lib/utils"; -import { UserRole } from "../types"; +import type { UserRole } from "../types"; const navItems = [ { icon: LayoutDashboard, label: "仪表盘", path: "/", roles: ["admin", "auditor", "operator"] }, @@ -33,36 +32,83 @@ const navItems = [ { icon: Activity, label: "执行监控", path: "/monitoring", roles: ["admin", "auditor", "operator"] }, { icon: Users, label: "用户管理", path: "/users", roles: ["admin"] }, { icon: Settings, label: "系统配置", path: "/settings", roles: ["admin"] }, -]; +] as const; interface SidebarProps { isCollapsed: boolean; onToggle: () => void; } +type ProfileDraft = { + name: string; + email: string; + role: UserRole; +}; + export function Sidebar({ isCollapsed, onToggle }: SidebarProps) { const [isProfileModalOpen, setIsProfileModalOpen] = useState(false); - const { role, setRole, name, setName, email, setEmail, logout } = useUser(); + const [profileDraft, setProfileDraft] = useState(null); + const [isSavingProfile, setIsSavingProfile] = useState(false); + const { + currentUser, + users, + role, + name, + email, + selectUser, + updateCurrentUser, + logout, + } = useUser(); - const handleSaveProfile = (e: React.FormEvent) => { - e.preventDefault(); - setIsProfileModalOpen(false); + useEffect(() => { + if (!isProfileModalOpen || !currentUser) { + return; + } + setProfileDraft({ + name: currentUser.name, + email: currentUser.email, + role: currentUser.role, + }); + }, [currentUser, isProfileModalOpen]); + + const filteredNavItems = useMemo(() => { + if (!role) { + return []; + } + return navItems.filter((item) => item.roles.includes(role)); + }, [role]); + + const handleSaveProfile = async (event: React.FormEvent) => { + event.preventDefault(); + if (!profileDraft) { + return; + } + setIsSavingProfile(true); + try { + await updateCurrentUser(profileDraft); + setIsProfileModalOpen(false); + } catch (error) { + console.error("Failed to update current user", error); + } finally { + setIsSavingProfile(false); + } }; const handleLogout = () => { logout(); + setIsProfileModalOpen(false); }; - const filteredNavItems = navItems.filter(item => - item.roles.includes(role) - ); + const avatarLabel = currentUser?.name?.slice(0, 2).toUpperCase() || "??"; return ( -