🚀 Support production GPU operator containers

This commit is contained in:
eust-w 2026-06-29 17:02:51 +08:00
parent b5155246d2
commit 37fd97fcf3
9 changed files with 465 additions and 18 deletions

View File

@ -144,24 +144,26 @@ type HyperParam struct {
}
type WorkflowNodeData struct {
Label string `json:"label" bson:"label"`
Category string `json:"category" bson:"category"`
Description string `json:"description" bson:"description"`
Inputs []PortDefinition `json:"inputs" bson:"inputs"`
Outputs []PortDefinition `json:"outputs" bson:"outputs"`
OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"`
OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"`
OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"`
Image string `json:"image,omitempty" bson:"image,omitempty"`
SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"`
Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"`
ConnectionID string `json:"connectionId,omitempty" bson:"connectionId,omitempty"`
Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"`
Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"`
Pattern string `json:"pattern,omitempty" bson:"pattern,omitempty"`
CPU string `json:"cpu,omitempty" bson:"cpu,omitempty"`
Memory string `json:"memory,omitempty" bson:"memory,omitempty"`
GPU string `json:"gpu,omitempty" bson:"gpu,omitempty"`
Label string `json:"label" bson:"label"`
Category string `json:"category" bson:"category"`
Description string `json:"description" bson:"description"`
Inputs []PortDefinition `json:"inputs" bson:"inputs"`
Outputs []PortDefinition `json:"outputs" bson:"outputs"`
OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"`
OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"`
OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"`
Image string `json:"image,omitempty" bson:"image,omitempty"`
ContainerCommand []string `json:"containerCommand,omitempty" bson:"containerCommand,omitempty"`
ContainerArgs []string `json:"containerArgs,omitempty" bson:"containerArgs,omitempty"`
SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"`
Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"`
ConnectionID string `json:"connectionId,omitempty" bson:"connectionId,omitempty"`
Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"`
Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"`
Pattern string `json:"pattern,omitempty" bson:"pattern,omitempty"`
CPU string `json:"cpu,omitempty" bson:"cpu,omitempty"`
Memory string `json:"memory,omitempty" bson:"memory,omitempty"`
GPU string `json:"gpu,omitempty" bson:"gpu,omitempty"`
}
type WorkflowNode struct {

View File

@ -90,6 +90,15 @@ func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact
return cloned
}
func cloneStringSlice(items []string) []string {
if items == nil {
return nil
}
cloned := make([]string, len(items))
copy(cloned, items)
return cloned
}
func cloneArtifact(item domain.ExecutionArtifact) domain.ExecutionArtifact {
cloned := item
if item.Metadata != nil {
@ -559,6 +568,8 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord,
}
launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{
Image: chooseImage(node, runtime.operatorRunnerImage),
Command: cloneStringSlice(node.Data.ContainerCommand),
Args: cloneStringSlice(node.Data.ContainerArgs),
RunID: run.ID,
WorkflowID: run.WorkflowID,
NodeID: node.ID,

View File

@ -1304,6 +1304,79 @@ func TestRuntimeStopsGracefullyWithoutDispatchingNewNodes(t *testing.T) {
}
}
func TestRuntimePassesContainerCommandAndArgsToLauncher(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "镜像项目", Description: "command args"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "自定义容器入口",
Description: "verify command args",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "custom",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "ACT 策略训练",
Description: "custom entrypoint",
Image: "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime",
ContainerCommand: []string{"/bin/bash", "-lc"},
ContainerArgs: []string{"python /workspace/operators/act.py"},
},
Position: domain.Position{X: 0, Y: 0},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
var captured k8s.LaunchInput
runtime := &Runtime{
store: dataStore,
operatorRunnerImage: "operator-runner:latest",
workspaceRoot: t.TempDir(),
launcher: testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
captured = input
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
return nil, err
}
}
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: "custom", Namespace: "droboticflow"},
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
}, nil
},
},
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
if captured.Image != "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime" {
t.Fatalf("image = %q", captured.Image)
}
if strings.Join(captured.Command, " ") != "/bin/bash -lc" {
t.Fatalf("command = %#v", captured.Command)
}
if len(captured.Args) != 1 || captured.Args[0] != "python /workspace/operators/act.py" {
t.Fatalf("args = %#v", captured.Args)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed run, got %#v", completed)
}
}
func TestRuntimePausesForAuditAndResumes(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()

View File

@ -0,0 +1,8 @@
FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
WORKDIR /app
ENV PYTHONUNBUFFERED=1
COPY operator-images/gpu-production/src/gpu_policy_operator.py /app/gpu_policy_operator.py
ENTRYPOINT ["python", "/app/gpu_policy_operator.py"]

View File

@ -0,0 +1,10 @@
FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
WORKDIR /app
ENV PYTHONUNBUFFERED=1
RUN pip install --no-cache-dir ultralytics opencv-python-headless
COPY operator-images/gpu-production/src/yolo_operator.py /app/yolo_operator.py
ENTRYPOINT ["python", "/app/yolo_operator.py"]

View File

@ -0,0 +1,181 @@
from __future__ import annotations
import datetime as dt
import hashlib
import json
import os
import pathlib
import shlex
import subprocess
import time
from typing import Any
import torch
def now_iso() -> str:
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
def env_text(name: str, default: str = "") -> str:
return os.getenv(name, default).strip()
def output_dir() -> pathlib.Path:
path = pathlib.Path(env_text("OUTPUT_DIR") or f"/workspace/runs/{env_text('RUN_ID')}/nodes/{env_text('NODE_ID')}/outputs")
path.mkdir(parents=True, exist_ok=True)
return path
def sha256(path: pathlib.Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def require_path_env(name: str, *, file_required: bool = False) -> pathlib.Path:
value = env_text(name)
if not value:
raise RuntimeError(f"{name} is required for production GPU operators")
path = pathlib.Path(value)
if file_required and not path.is_file():
raise RuntimeError(f"{name} must point to an existing file: {path}")
if not file_required and not path.exists():
raise RuntimeError(f"{name} must point to an existing path: {path}")
return path
def require_cuda() -> dict[str, Any]:
if not torch.cuda.is_available():
raise RuntimeError("production GPU operator requires torch.cuda.is_available()")
device = torch.device("cuda:0")
probe = torch.randn(4096, 4096, device=device)
_ = probe @ probe
torch.cuda.synchronize(device)
props = torch.cuda.get_device_properties(device)
allocated = int(torch.cuda.max_memory_allocated(device) / 1024 / 1024)
if allocated <= 0:
raise RuntimeError("CUDA probe did not allocate GPU memory")
return {
"torchVersion": torch.__version__,
"cudaVersion": torch.version.cuda,
"gpuName": torch.cuda.get_device_name(device),
"gpuMemoryTotalMiB": int(props.total_memory / 1024 / 1024),
"gpuMemoryAllocatedMiB": allocated,
}
def artifact(run_id: str, node_id: str, path: pathlib.Path, artifact_type: str = "file", data: dict[str, Any] | None = None) -> dict[str, Any]:
stat = path.stat()
return {
"id": f"artifact-{node_id}-{path.name}",
"runId": run_id,
"nodeId": node_id,
"name": path.name,
"type": artifact_type,
"size": str(stat.st_size),
"path": str(path),
"metadata": {"sha256": sha256(path), "mtime": int(stat.st_mtime)},
"data": data or {},
}
def load_json(path: pathlib.Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def collect_artifacts(run_id: str, node_id: str, expected_manifest: pathlib.Path, metrics: dict[str, Any]) -> list[dict[str, Any]]:
manifest = load_json(expected_manifest)
artifacts: list[dict[str, Any]] = []
for item in manifest.get("artifacts", []):
path_value = str(item.get("path", "")).strip()
if not path_value:
raise RuntimeError(f"artifact entry is missing path in {expected_manifest}")
path = pathlib.Path(path_value)
if not path.exists():
raise RuntimeError(f"expected artifact does not exist: {path}")
artifact_type = str(item.get("type") or "file")
data = item.get("data")
artifacts.append(artifact(run_id, node_id, path, artifact_type, data if isinstance(data, dict) else None))
metrics_artifact = pathlib.Path(metrics["metricsPath"])
artifacts.append(artifact(run_id, node_id, metrics_artifact, "json", metrics))
if not artifacts:
raise RuntimeError(f"{expected_manifest} did not declare any artifacts")
return artifacts
def run_shell(command: str, cwd: pathlib.Path) -> tuple[int, str, float]:
started = time.time()
completed = subprocess.run(["/bin/bash", "-lc", command], cwd=cwd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
return completed.returncode, completed.stdout, time.time() - started
def main() -> None:
started = now_iso()
run_id = env_text("RUN_ID")
node_id = env_text("NODE_ID")
out = output_dir()
task = env_text("GPU_PRODUCTION_TASK") or env_text("NODE_LABEL")
if not task:
raise RuntimeError("GPU_PRODUCTION_TASK or NODE_LABEL is required")
dataset_manifest = require_path_env("PRODUCTION_DATASET_MANIFEST", file_required=True)
expected_artifacts = require_path_env("PRODUCTION_EXPECTED_ARTIFACTS", file_required=True)
command = env_text("PRODUCTION_TRAIN_COMMAND") or env_text("PRODUCTION_INFERENCE_COMMAND")
if not command:
raise RuntimeError("PRODUCTION_TRAIN_COMMAND or PRODUCTION_INFERENCE_COMMAND is required; lightweight built-in training is not allowed")
cuda = require_cuda()
workdir = pathlib.Path(env_text("PRODUCTION_WORKDIR", str(out)))
workdir.mkdir(parents=True, exist_ok=True)
return_code, output, duration = run_shell(command, workdir)
log_path = out / "production-command.log"
log_path.write_text(output, encoding="utf-8")
metrics_path = out / "production-gpu-metrics.json"
metrics: dict[str, Any] = {
"task": task,
"officialFramework": True,
"productionAcceptance": return_code == 0,
"datasetManifest": str(dataset_manifest),
"datasetManifestSha256": sha256(dataset_manifest),
"command": shlex.join(["/bin/bash", "-lc", command]),
"returnCode": return_code,
"durationSeconds": round(duration, 3),
"logPath": str(log_path),
"metricsPath": str(metrics_path),
"startedAt": started,
"endedAt": now_iso(),
**cuda,
}
metrics_path.write_text(json.dumps(metrics, ensure_ascii=False, indent=2), encoding="utf-8")
if return_code != 0:
raise RuntimeError(f"production command failed ({return_code}); see {log_path}")
artifacts = collect_artifacts(run_id, node_id, expected_artifacts, metrics)
artifacts.append(artifact(run_id, node_id, log_path, "file"))
result = {
"logs": [
{"timestamp": started, "level": "INFO", "message": f"started production GPU task {task}", "operator": env_text("NODE_LABEL")},
{"timestamp": now_iso(), "level": "INFO", "message": f"finished production GPU task {task}", "operator": env_text("NODE_LABEL")},
],
"artifacts": artifacts,
"resourceSeries": [
{
"timestamp": now_iso(),
"gpu": 100 if cuda["gpuMemoryAllocatedMiB"] > 0 else 0,
"gpuMemoryUsedMiB": cuda["gpuMemoryAllocatedMiB"],
"gpuMemoryTotalMiB": cuda["gpuMemoryTotalMiB"],
"gpuName": cuda["gpuName"],
"gpuCount": 1,
}
],
}
result_path = env_text("RESULT_OUTPUT_PATH")
if result_path:
pathlib.Path(result_path).write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8")
print(f"DROBOTICFLOW_RESULT_PATH={result_path}", flush=True)
else:
print("DROBOTICFLOW_RESULT=" + json.dumps(result, ensure_ascii=False), flush=True)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,158 @@
from __future__ import annotations
import datetime as dt
import hashlib
import json
import os
import pathlib
import subprocess
from typing import Any
import torch
def now_iso() -> str:
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
def env_text(name: str, default: str = "") -> str:
return os.getenv(name, default).strip()
def sha256(path: pathlib.Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def output_dir() -> pathlib.Path:
path = pathlib.Path(env_text("OUTPUT_DIR") or f"/workspace/runs/{env_text('RUN_ID')}/nodes/{env_text('NODE_ID')}/outputs")
path.mkdir(parents=True, exist_ok=True)
return path
def require_existing_path(name: str, *, file_required: bool = False) -> str:
value = env_text(name)
if not value:
raise RuntimeError(f"{name} is required for production YOLO operators")
path = pathlib.Path(value)
if file_required and not path.is_file():
raise RuntimeError(f"{name} must point to an existing file: {path}")
if not file_required and not path.exists():
raise RuntimeError(f"{name} must point to an existing path: {path}")
return str(path)
def artifact(run_id: str, node_id: str, path: pathlib.Path, artifact_type: str = "file", data: dict[str, Any] | None = None) -> dict[str, Any]:
return {
"id": f"artifact-{node_id}-{path.name}",
"runId": run_id,
"nodeId": node_id,
"name": path.name,
"type": artifact_type,
"size": str(path.stat().st_size),
"path": str(path),
"metadata": {"sha256": sha256(path)},
"data": data or {},
}
def run_command(args: list[str], cwd: pathlib.Path) -> str:
completed = subprocess.run(args, cwd=cwd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
if completed.returncode != 0:
raise RuntimeError(f"command failed ({completed.returncode}): {' '.join(args)}\n{completed.stdout}")
return completed.stdout
def main() -> None:
started = now_iso()
if not torch.cuda.is_available():
raise RuntimeError("YOLO production validation requires CUDA")
device_name = torch.cuda.get_device_name(0)
probe = torch.randn(2048, 2048, device="cuda")
_ = probe @ probe
torch.cuda.synchronize()
out = output_dir()
task = env_text("GPU_PRODUCTION_TASK") or ("clip" if "裁剪" in env_text("NODE_LABEL") else "train")
artifacts: list[dict[str, Any]] = []
metrics: dict[str, Any] = {
"task": task,
"framework": "ultralytics",
"officialFramework": True,
"productionAcceptance": True,
"torchVersion": torch.__version__,
"cudaVersion": torch.version.cuda,
"gpuName": device_name,
"gpuMemoryAllocatedMiB": int(torch.cuda.max_memory_allocated(0) / 1024 / 1024),
"startedAt": started,
}
if "clip" in task:
model = env_text("YOLO_MODEL", "yolov8n.pt")
source = require_existing_path("YOLO_SOURCE")
yolo_out = out / "predict"
log = run_command(["yolo", "predict", f"model={model}", f"source={source}", "device=0", "save=True", "conf=0.20", f"project={yolo_out}", "name=run"], out)
produced = sorted((yolo_out / "run").rglob("*"))
videos = [path for path in produced if path.is_file() and path.suffix.lower() in {".mp4", ".avi", ".mov"}]
metrics.update({"mode": "predict", "source": source, "videoCount": len(videos), "logTail": log[-4000:]})
for video in videos:
artifacts.append(artifact(env_text("RUN_ID"), env_text("NODE_ID"), video, "video"))
else:
dataset = require_existing_path("YOLO_DATASET", file_required=True)
yolo_out = out / "train"
log = run_command(
[
"yolo",
"detect",
"train",
"model=yolov8n.pt",
f"data={dataset}",
"epochs=1",
"imgsz=320",
"batch=8",
"device=0",
f"project={yolo_out}",
"name=run",
],
out,
)
weights = yolo_out / "run" / "weights" / "best.pt"
results_csv = yolo_out / "run" / "results.csv"
if not weights.exists():
raise RuntimeError("Ultralytics training did not produce best.pt")
metrics.update({"mode": "train", "checkpoint": str(weights), "checkpointSha256": sha256(weights), "logTail": log[-4000:]})
artifacts.append(artifact(env_text("RUN_ID"), env_text("NODE_ID"), weights, "file"))
if results_csv.exists():
artifacts.append(artifact(env_text("RUN_ID"), env_text("NODE_ID"), results_csv, "file"))
metrics["endedAt"] = now_iso()
metrics_path = out / f"yolo-{task}-metrics.json"
metrics_path.write_text(json.dumps(metrics, ensure_ascii=False, indent=2), encoding="utf-8")
artifacts.append(artifact(env_text("RUN_ID"), env_text("NODE_ID"), metrics_path, "json", metrics))
result = {
"logs": [
{"timestamp": started, "level": "INFO", "message": f"started YOLO {task} production validation", "operator": env_text("NODE_LABEL")},
{"timestamp": now_iso(), "level": "INFO", "message": f"finished YOLO {task} on {device_name}", "operator": env_text("NODE_LABEL")},
],
"artifacts": artifacts,
"resourceSeries": [
{
"timestamp": now_iso(),
"gpu": 100,
"gpuMemoryUsedMiB": metrics["gpuMemoryAllocatedMiB"],
"gpuMemoryTotalMiB": int(torch.cuda.get_device_properties(0).total_memory / 1024 / 1024),
"gpuName": device_name,
"gpuCount": 1,
}
],
}
result_path = env_text("RESULT_OUTPUT_PATH")
if result_path:
pathlib.Path(result_path).write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8")
print(f"DROBOTICFLOW_RESULT_PATH={result_path}", flush=True)
else:
print("DROBOTICFLOW_RESULT=" + json.dumps(result, ensure_ascii=False), flush=True)
if __name__ == "__main__":
main()

View File

@ -179,6 +179,8 @@ interface OperatorNodeData {
operatorVersion?: string;
operatorImageId?: string;
image?: string;
containerCommand?: string[];
containerArgs?: string[];
hyperparams?: HyperParam[];
cpu?: string;
memory?: string;

View File

@ -233,6 +233,8 @@ export interface Operator {
version?: string; // Operator business/runtime version
image?: string; // Docker image for custom operators
imageId?: string; // Digest-pinned image reference when available
containerCommand?: string[]; // Optional Kubernetes container command override
containerArgs?: string[]; // Optional Kubernetes container args override
skillDefinition?: string; // AI-readable description for workflow generation
hyperparams?: HyperParam[]; // K8s Job environment variables
resources?: {