forked from longtao.wu/droboticflow
🚀 Support production GPU operator containers
This commit is contained in:
parent
b5155246d2
commit
37fd97fcf3
@ -144,24 +144,26 @@ type HyperParam struct {
|
||||
}
|
||||
|
||||
type WorkflowNodeData struct {
|
||||
Label string `json:"label" bson:"label"`
|
||||
Category string `json:"category" bson:"category"`
|
||||
Description string `json:"description" bson:"description"`
|
||||
Inputs []PortDefinition `json:"inputs" bson:"inputs"`
|
||||
Outputs []PortDefinition `json:"outputs" bson:"outputs"`
|
||||
OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"`
|
||||
OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"`
|
||||
OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"`
|
||||
Image string `json:"image,omitempty" bson:"image,omitempty"`
|
||||
SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"`
|
||||
Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"`
|
||||
ConnectionID string `json:"connectionId,omitempty" bson:"connectionId,omitempty"`
|
||||
Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"`
|
||||
Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"`
|
||||
Pattern string `json:"pattern,omitempty" bson:"pattern,omitempty"`
|
||||
CPU string `json:"cpu,omitempty" bson:"cpu,omitempty"`
|
||||
Memory string `json:"memory,omitempty" bson:"memory,omitempty"`
|
||||
GPU string `json:"gpu,omitempty" bson:"gpu,omitempty"`
|
||||
Label string `json:"label" bson:"label"`
|
||||
Category string `json:"category" bson:"category"`
|
||||
Description string `json:"description" bson:"description"`
|
||||
Inputs []PortDefinition `json:"inputs" bson:"inputs"`
|
||||
Outputs []PortDefinition `json:"outputs" bson:"outputs"`
|
||||
OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"`
|
||||
OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"`
|
||||
OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"`
|
||||
Image string `json:"image,omitempty" bson:"image,omitempty"`
|
||||
ContainerCommand []string `json:"containerCommand,omitempty" bson:"containerCommand,omitempty"`
|
||||
ContainerArgs []string `json:"containerArgs,omitempty" bson:"containerArgs,omitempty"`
|
||||
SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"`
|
||||
Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"`
|
||||
ConnectionID string `json:"connectionId,omitempty" bson:"connectionId,omitempty"`
|
||||
Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"`
|
||||
Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"`
|
||||
Pattern string `json:"pattern,omitempty" bson:"pattern,omitempty"`
|
||||
CPU string `json:"cpu,omitempty" bson:"cpu,omitempty"`
|
||||
Memory string `json:"memory,omitempty" bson:"memory,omitempty"`
|
||||
GPU string `json:"gpu,omitempty" bson:"gpu,omitempty"`
|
||||
}
|
||||
|
||||
type WorkflowNode struct {
|
||||
|
||||
@ -90,6 +90,15 @@ func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact
|
||||
return cloned
|
||||
}
|
||||
|
||||
func cloneStringSlice(items []string) []string {
|
||||
if items == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := make([]string, len(items))
|
||||
copy(cloned, items)
|
||||
return cloned
|
||||
}
|
||||
|
||||
func cloneArtifact(item domain.ExecutionArtifact) domain.ExecutionArtifact {
|
||||
cloned := item
|
||||
if item.Metadata != nil {
|
||||
@ -559,6 +568,8 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord,
|
||||
}
|
||||
launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{
|
||||
Image: chooseImage(node, runtime.operatorRunnerImage),
|
||||
Command: cloneStringSlice(node.Data.ContainerCommand),
|
||||
Args: cloneStringSlice(node.Data.ContainerArgs),
|
||||
RunID: run.ID,
|
||||
WorkflowID: run.WorkflowID,
|
||||
NodeID: node.ID,
|
||||
|
||||
@ -1304,6 +1304,79 @@ func TestRuntimeStopsGracefullyWithoutDispatchingNewNodes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimePassesContainerCommandAndArgsToLauncher(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "镜像项目", Description: "command args"})
|
||||
if err != nil {
|
||||
t.Fatalf("create project: %v", err)
|
||||
}
|
||||
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
||||
ProjectID: project.ID,
|
||||
Name: "自定义容器入口",
|
||||
Description: "verify command args",
|
||||
Type: domain.WorkflowTypeBatch,
|
||||
Nodes: []domain.WorkflowNode{{
|
||||
ID: "custom",
|
||||
Type: "operator",
|
||||
Data: domain.WorkflowNodeData{
|
||||
Label: "ACT 策略训练",
|
||||
Description: "custom entrypoint",
|
||||
Image: "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime",
|
||||
ContainerCommand: []string{"/bin/bash", "-lc"},
|
||||
ContainerArgs: []string{"python /workspace/operators/act.py"},
|
||||
},
|
||||
Position: domain.Position{X: 0, Y: 0},
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create workflow: %v", err)
|
||||
}
|
||||
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
||||
if err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
var captured k8s.LaunchInput
|
||||
runtime := &Runtime{
|
||||
store: dataStore,
|
||||
operatorRunnerImage: "operator-runner:latest",
|
||||
workspaceRoot: t.TempDir(),
|
||||
launcher: testLauncher{
|
||||
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
||||
captured = input
|
||||
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
|
||||
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &k8s.LaunchResult{
|
||||
Reference: &k8s.JobReference{JobName: "custom", Namespace: "droboticflow"},
|
||||
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
||||
t.Fatalf("process once: %v", err)
|
||||
}
|
||||
if captured.Image != "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime" {
|
||||
t.Fatalf("image = %q", captured.Image)
|
||||
}
|
||||
if strings.Join(captured.Command, " ") != "/bin/bash -lc" {
|
||||
t.Fatalf("command = %#v", captured.Command)
|
||||
}
|
||||
if len(captured.Args) != 1 || captured.Args[0] != "python /workspace/operators/act.py" {
|
||||
t.Fatalf("args = %#v", captured.Args)
|
||||
}
|
||||
completed, err := dataStore.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if completed == nil || completed.Status != domain.RunStatusCompleted {
|
||||
t.Fatalf("expected completed run, got %#v", completed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimePausesForAuditAndResumes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
|
||||
8
operator-images/gpu-production/Dockerfile.torch-policy
Normal file
8
operator-images/gpu-production/Dockerfile.torch-policy
Normal file
@ -0,0 +1,8 @@
|
||||
FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
|
||||
|
||||
WORKDIR /app
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
COPY operator-images/gpu-production/src/gpu_policy_operator.py /app/gpu_policy_operator.py
|
||||
|
||||
ENTRYPOINT ["python", "/app/gpu_policy_operator.py"]
|
||||
10
operator-images/gpu-production/Dockerfile.yolo
Normal file
10
operator-images/gpu-production/Dockerfile.yolo
Normal file
@ -0,0 +1,10 @@
|
||||
FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
|
||||
|
||||
WORKDIR /app
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
RUN pip install --no-cache-dir ultralytics opencv-python-headless
|
||||
|
||||
COPY operator-images/gpu-production/src/yolo_operator.py /app/yolo_operator.py
|
||||
|
||||
ENTRYPOINT ["python", "/app/yolo_operator.py"]
|
||||
181
operator-images/gpu-production/src/gpu_policy_operator.py
Normal file
181
operator-images/gpu-production/src/gpu_policy_operator.py
Normal file
@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def env_text(name: str, default: str = "") -> str:
|
||||
return os.getenv(name, default).strip()
|
||||
|
||||
|
||||
def output_dir() -> pathlib.Path:
|
||||
path = pathlib.Path(env_text("OUTPUT_DIR") or f"/workspace/runs/{env_text('RUN_ID')}/nodes/{env_text('NODE_ID')}/outputs")
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def sha256(path: pathlib.Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def require_path_env(name: str, *, file_required: bool = False) -> pathlib.Path:
|
||||
value = env_text(name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{name} is required for production GPU operators")
|
||||
path = pathlib.Path(value)
|
||||
if file_required and not path.is_file():
|
||||
raise RuntimeError(f"{name} must point to an existing file: {path}")
|
||||
if not file_required and not path.exists():
|
||||
raise RuntimeError(f"{name} must point to an existing path: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def require_cuda() -> dict[str, Any]:
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("production GPU operator requires torch.cuda.is_available()")
|
||||
device = torch.device("cuda:0")
|
||||
probe = torch.randn(4096, 4096, device=device)
|
||||
_ = probe @ probe
|
||||
torch.cuda.synchronize(device)
|
||||
props = torch.cuda.get_device_properties(device)
|
||||
allocated = int(torch.cuda.max_memory_allocated(device) / 1024 / 1024)
|
||||
if allocated <= 0:
|
||||
raise RuntimeError("CUDA probe did not allocate GPU memory")
|
||||
return {
|
||||
"torchVersion": torch.__version__,
|
||||
"cudaVersion": torch.version.cuda,
|
||||
"gpuName": torch.cuda.get_device_name(device),
|
||||
"gpuMemoryTotalMiB": int(props.total_memory / 1024 / 1024),
|
||||
"gpuMemoryAllocatedMiB": allocated,
|
||||
}
|
||||
|
||||
|
||||
def artifact(run_id: str, node_id: str, path: pathlib.Path, artifact_type: str = "file", data: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
stat = path.stat()
|
||||
return {
|
||||
"id": f"artifact-{node_id}-{path.name}",
|
||||
"runId": run_id,
|
||||
"nodeId": node_id,
|
||||
"name": path.name,
|
||||
"type": artifact_type,
|
||||
"size": str(stat.st_size),
|
||||
"path": str(path),
|
||||
"metadata": {"sha256": sha256(path), "mtime": int(stat.st_mtime)},
|
||||
"data": data or {},
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: pathlib.Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def collect_artifacts(run_id: str, node_id: str, expected_manifest: pathlib.Path, metrics: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
manifest = load_json(expected_manifest)
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
for item in manifest.get("artifacts", []):
|
||||
path_value = str(item.get("path", "")).strip()
|
||||
if not path_value:
|
||||
raise RuntimeError(f"artifact entry is missing path in {expected_manifest}")
|
||||
path = pathlib.Path(path_value)
|
||||
if not path.exists():
|
||||
raise RuntimeError(f"expected artifact does not exist: {path}")
|
||||
artifact_type = str(item.get("type") or "file")
|
||||
data = item.get("data")
|
||||
artifacts.append(artifact(run_id, node_id, path, artifact_type, data if isinstance(data, dict) else None))
|
||||
metrics_artifact = pathlib.Path(metrics["metricsPath"])
|
||||
artifacts.append(artifact(run_id, node_id, metrics_artifact, "json", metrics))
|
||||
if not artifacts:
|
||||
raise RuntimeError(f"{expected_manifest} did not declare any artifacts")
|
||||
return artifacts
|
||||
|
||||
|
||||
def run_shell(command: str, cwd: pathlib.Path) -> tuple[int, str, float]:
|
||||
started = time.time()
|
||||
completed = subprocess.run(["/bin/bash", "-lc", command], cwd=cwd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
|
||||
return completed.returncode, completed.stdout, time.time() - started
|
||||
|
||||
|
||||
def main() -> None:
|
||||
started = now_iso()
|
||||
run_id = env_text("RUN_ID")
|
||||
node_id = env_text("NODE_ID")
|
||||
out = output_dir()
|
||||
task = env_text("GPU_PRODUCTION_TASK") or env_text("NODE_LABEL")
|
||||
if not task:
|
||||
raise RuntimeError("GPU_PRODUCTION_TASK or NODE_LABEL is required")
|
||||
dataset_manifest = require_path_env("PRODUCTION_DATASET_MANIFEST", file_required=True)
|
||||
expected_artifacts = require_path_env("PRODUCTION_EXPECTED_ARTIFACTS", file_required=True)
|
||||
command = env_text("PRODUCTION_TRAIN_COMMAND") or env_text("PRODUCTION_INFERENCE_COMMAND")
|
||||
if not command:
|
||||
raise RuntimeError("PRODUCTION_TRAIN_COMMAND or PRODUCTION_INFERENCE_COMMAND is required; lightweight built-in training is not allowed")
|
||||
cuda = require_cuda()
|
||||
workdir = pathlib.Path(env_text("PRODUCTION_WORKDIR", str(out)))
|
||||
workdir.mkdir(parents=True, exist_ok=True)
|
||||
return_code, output, duration = run_shell(command, workdir)
|
||||
log_path = out / "production-command.log"
|
||||
log_path.write_text(output, encoding="utf-8")
|
||||
metrics_path = out / "production-gpu-metrics.json"
|
||||
metrics: dict[str, Any] = {
|
||||
"task": task,
|
||||
"officialFramework": True,
|
||||
"productionAcceptance": return_code == 0,
|
||||
"datasetManifest": str(dataset_manifest),
|
||||
"datasetManifestSha256": sha256(dataset_manifest),
|
||||
"command": shlex.join(["/bin/bash", "-lc", command]),
|
||||
"returnCode": return_code,
|
||||
"durationSeconds": round(duration, 3),
|
||||
"logPath": str(log_path),
|
||||
"metricsPath": str(metrics_path),
|
||||
"startedAt": started,
|
||||
"endedAt": now_iso(),
|
||||
**cuda,
|
||||
}
|
||||
metrics_path.write_text(json.dumps(metrics, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
if return_code != 0:
|
||||
raise RuntimeError(f"production command failed ({return_code}); see {log_path}")
|
||||
artifacts = collect_artifacts(run_id, node_id, expected_artifacts, metrics)
|
||||
artifacts.append(artifact(run_id, node_id, log_path, "file"))
|
||||
result = {
|
||||
"logs": [
|
||||
{"timestamp": started, "level": "INFO", "message": f"started production GPU task {task}", "operator": env_text("NODE_LABEL")},
|
||||
{"timestamp": now_iso(), "level": "INFO", "message": f"finished production GPU task {task}", "operator": env_text("NODE_LABEL")},
|
||||
],
|
||||
"artifacts": artifacts,
|
||||
"resourceSeries": [
|
||||
{
|
||||
"timestamp": now_iso(),
|
||||
"gpu": 100 if cuda["gpuMemoryAllocatedMiB"] > 0 else 0,
|
||||
"gpuMemoryUsedMiB": cuda["gpuMemoryAllocatedMiB"],
|
||||
"gpuMemoryTotalMiB": cuda["gpuMemoryTotalMiB"],
|
||||
"gpuName": cuda["gpuName"],
|
||||
"gpuCount": 1,
|
||||
}
|
||||
],
|
||||
}
|
||||
result_path = env_text("RESULT_OUTPUT_PATH")
|
||||
if result_path:
|
||||
pathlib.Path(result_path).write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"DROBOTICFLOW_RESULT_PATH={result_path}", flush=True)
|
||||
else:
|
||||
print("DROBOTICFLOW_RESULT=" + json.dumps(result, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
158
operator-images/gpu-production/src/yolo_operator.py
Normal file
158
operator-images/gpu-production/src/yolo_operator.py
Normal file
@ -0,0 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def env_text(name: str, default: str = "") -> str:
|
||||
return os.getenv(name, default).strip()
|
||||
|
||||
|
||||
def sha256(path: pathlib.Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def output_dir() -> pathlib.Path:
|
||||
path = pathlib.Path(env_text("OUTPUT_DIR") or f"/workspace/runs/{env_text('RUN_ID')}/nodes/{env_text('NODE_ID')}/outputs")
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def require_existing_path(name: str, *, file_required: bool = False) -> str:
|
||||
value = env_text(name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{name} is required for production YOLO operators")
|
||||
path = pathlib.Path(value)
|
||||
if file_required and not path.is_file():
|
||||
raise RuntimeError(f"{name} must point to an existing file: {path}")
|
||||
if not file_required and not path.exists():
|
||||
raise RuntimeError(f"{name} must point to an existing path: {path}")
|
||||
return str(path)
|
||||
|
||||
|
||||
def artifact(run_id: str, node_id: str, path: pathlib.Path, artifact_type: str = "file", data: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return {
|
||||
"id": f"artifact-{node_id}-{path.name}",
|
||||
"runId": run_id,
|
||||
"nodeId": node_id,
|
||||
"name": path.name,
|
||||
"type": artifact_type,
|
||||
"size": str(path.stat().st_size),
|
||||
"path": str(path),
|
||||
"metadata": {"sha256": sha256(path)},
|
||||
"data": data or {},
|
||||
}
|
||||
|
||||
|
||||
def run_command(args: list[str], cwd: pathlib.Path) -> str:
|
||||
completed = subprocess.run(args, cwd=cwd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(f"command failed ({completed.returncode}): {' '.join(args)}\n{completed.stdout}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def main() -> None:
|
||||
started = now_iso()
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("YOLO production validation requires CUDA")
|
||||
device_name = torch.cuda.get_device_name(0)
|
||||
probe = torch.randn(2048, 2048, device="cuda")
|
||||
_ = probe @ probe
|
||||
torch.cuda.synchronize()
|
||||
out = output_dir()
|
||||
task = env_text("GPU_PRODUCTION_TASK") or ("clip" if "裁剪" in env_text("NODE_LABEL") else "train")
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
metrics: dict[str, Any] = {
|
||||
"task": task,
|
||||
"framework": "ultralytics",
|
||||
"officialFramework": True,
|
||||
"productionAcceptance": True,
|
||||
"torchVersion": torch.__version__,
|
||||
"cudaVersion": torch.version.cuda,
|
||||
"gpuName": device_name,
|
||||
"gpuMemoryAllocatedMiB": int(torch.cuda.max_memory_allocated(0) / 1024 / 1024),
|
||||
"startedAt": started,
|
||||
}
|
||||
if "clip" in task:
|
||||
model = env_text("YOLO_MODEL", "yolov8n.pt")
|
||||
source = require_existing_path("YOLO_SOURCE")
|
||||
yolo_out = out / "predict"
|
||||
log = run_command(["yolo", "predict", f"model={model}", f"source={source}", "device=0", "save=True", "conf=0.20", f"project={yolo_out}", "name=run"], out)
|
||||
produced = sorted((yolo_out / "run").rglob("*"))
|
||||
videos = [path for path in produced if path.is_file() and path.suffix.lower() in {".mp4", ".avi", ".mov"}]
|
||||
metrics.update({"mode": "predict", "source": source, "videoCount": len(videos), "logTail": log[-4000:]})
|
||||
for video in videos:
|
||||
artifacts.append(artifact(env_text("RUN_ID"), env_text("NODE_ID"), video, "video"))
|
||||
else:
|
||||
dataset = require_existing_path("YOLO_DATASET", file_required=True)
|
||||
yolo_out = out / "train"
|
||||
log = run_command(
|
||||
[
|
||||
"yolo",
|
||||
"detect",
|
||||
"train",
|
||||
"model=yolov8n.pt",
|
||||
f"data={dataset}",
|
||||
"epochs=1",
|
||||
"imgsz=320",
|
||||
"batch=8",
|
||||
"device=0",
|
||||
f"project={yolo_out}",
|
||||
"name=run",
|
||||
],
|
||||
out,
|
||||
)
|
||||
weights = yolo_out / "run" / "weights" / "best.pt"
|
||||
results_csv = yolo_out / "run" / "results.csv"
|
||||
if not weights.exists():
|
||||
raise RuntimeError("Ultralytics training did not produce best.pt")
|
||||
metrics.update({"mode": "train", "checkpoint": str(weights), "checkpointSha256": sha256(weights), "logTail": log[-4000:]})
|
||||
artifacts.append(artifact(env_text("RUN_ID"), env_text("NODE_ID"), weights, "file"))
|
||||
if results_csv.exists():
|
||||
artifacts.append(artifact(env_text("RUN_ID"), env_text("NODE_ID"), results_csv, "file"))
|
||||
metrics["endedAt"] = now_iso()
|
||||
metrics_path = out / f"yolo-{task}-metrics.json"
|
||||
metrics_path.write_text(json.dumps(metrics, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
artifacts.append(artifact(env_text("RUN_ID"), env_text("NODE_ID"), metrics_path, "json", metrics))
|
||||
result = {
|
||||
"logs": [
|
||||
{"timestamp": started, "level": "INFO", "message": f"started YOLO {task} production validation", "operator": env_text("NODE_LABEL")},
|
||||
{"timestamp": now_iso(), "level": "INFO", "message": f"finished YOLO {task} on {device_name}", "operator": env_text("NODE_LABEL")},
|
||||
],
|
||||
"artifacts": artifacts,
|
||||
"resourceSeries": [
|
||||
{
|
||||
"timestamp": now_iso(),
|
||||
"gpu": 100,
|
||||
"gpuMemoryUsedMiB": metrics["gpuMemoryAllocatedMiB"],
|
||||
"gpuMemoryTotalMiB": int(torch.cuda.get_device_properties(0).total_memory / 1024 / 1024),
|
||||
"gpuName": device_name,
|
||||
"gpuCount": 1,
|
||||
}
|
||||
],
|
||||
}
|
||||
result_path = env_text("RESULT_OUTPUT_PATH")
|
||||
if result_path:
|
||||
pathlib.Path(result_path).write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"DROBOTICFLOW_RESULT_PATH={result_path}", flush=True)
|
||||
else:
|
||||
print("DROBOTICFLOW_RESULT=" + json.dumps(result, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -179,6 +179,8 @@ interface OperatorNodeData {
|
||||
operatorVersion?: string;
|
||||
operatorImageId?: string;
|
||||
image?: string;
|
||||
containerCommand?: string[];
|
||||
containerArgs?: string[];
|
||||
hyperparams?: HyperParam[];
|
||||
cpu?: string;
|
||||
memory?: string;
|
||||
|
||||
@ -233,6 +233,8 @@ export interface Operator {
|
||||
version?: string; // Operator business/runtime version
|
||||
image?: string; // Docker image for custom operators
|
||||
imageId?: string; // Digest-pinned image reference when available
|
||||
containerCommand?: string[]; // Optional Kubernetes container command override
|
||||
containerArgs?: string[]; // Optional Kubernetes container args override
|
||||
skillDefinition?: string; // AI-readable description for workflow generation
|
||||
hyperparams?: HyperParam[]; // K8s Job environment variables
|
||||
resources?: {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user