1827 lines
59 KiB
Go

package k8s
import (
"bytes"
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/remotecommand"
)
const (
defaultJobWaitTimeout = 2 * time.Hour
longRunningJobWaitTimeout = 7 * 24 * time.Hour
defaultJobBackoffLimit int32 = 0
)
type LaunchInput struct {
Image string
Command []string
Args []string
RunID string
WorkflowID string
NodeID string
NodeLabel string
Attempt int
TimeoutSeconds int
Env map[string]string
SensitiveEnv map[string]string
CPURequest string
CPULimit string
MemoryRequest string
MemoryLimit string
GPULimit string
GPUNodeSelector map[string]string
GPUTolerations []corev1.Toleration
GPURuntimeClass string
JobTTLSeconds int32
RegistryAuth bool
LongRunning bool
CodeSubPath string
CodeMountPath string
CodeReadOnly bool
PVCMounts []PVCMount
// OperatorRunnerImage is the image the platform resolves builtin operators to.
// When the resolved Image equals it, the pod runs operator-runner's main.py,
// which already self-reports resourceSeries via its ResourceSampler — so the
// launcher skips platform-side GPU sampling for those pods to avoid duplicate
// work. Custom-container operators (own image) bypass main.py and would report
// no resourceSeries at all; for those the launcher samples GPU usage itself.
OperatorRunnerImage string
// SharedMemorySize controls the size of the in-memory /dev/shm volume. It is
// populated by Launcher from DROBOTICFLOW_SHARED_MEMORY_SIZE (default 8Gi)
// for production jobs; keeping it on the launch input makes the pod contract
// explicit and easy to test.
SharedMemorySize string
InferenceSidecar *InferenceSidecar
}
type InferenceSidecar struct {
Name string
Image string
Port int32
HealthPath string
Env map[string]string
CPURequest string
CPULimit string
MemoryRequest string
MemoryLimit string
GPULimit string
}
type PVCMount struct {
Name string
PVCName string
SubPath string
MountPath string
// ReadOnly is opt-in. An omitted readOnly field in a workflow contract
// becomes false and keeps the PVC mount writable by default.
ReadOnly bool
}
type JobReference struct {
JobName string
Namespace string
}
type LaunchResult struct {
Reference *JobReference
Output string
// ResourceSeries holds GPU samples collected by the platform while the pod was
// running. Populated only for custom-container operators that do not
// self-report resourceSeries (see LaunchInput.OperatorRunnerImage). Empty for
// builtin operator-runner pods, which emit their own series in result.json.
ResourceSeries []GPUSample
}
// GPUSample mirrors the GPU-relevant fields of domain.ResourceSample. Kept in the
// k8s package so the launcher does not depend on internal/domain; the worker maps
// it to domain.ResourceSample when the operator did not self-report a series.
type GPUSample struct {
Timestamp string
GPU int
GPUMemoryUsedMiB int
GPUMemoryTotalMiB int
GPUName string
GPUCount int
}
type JobState string
const (
JobStateMissing JobState = "missing"
JobStatePending JobState = "pending"
JobStateActive JobState = "active"
JobStateSucceeded JobState = "succeeded"
JobStateFailed JobState = "failed"
)
type JobInspection struct {
Reference *JobReference
Image string
State JobState
Output string
CreatedAt string
BlockingReason string
BlockingMessage string
}
// JobNotFoundError marks a Job that disappeared after it was created. Keeping
// this distinct from ordinary execution failures lets the worker re-queue the
// node through its bounded missing-Job recovery path instead of immediately
// terminalizing a healthy long-running training Run.
type JobNotFoundError struct {
JobName string
}
func (err *JobNotFoundError) Error() string {
return fmt.Sprintf("kubernetes job %q not found", strings.TrimSpace(err.JobName))
}
func IsJobNotFound(err error) bool {
if err == nil {
return false
}
var missing *JobNotFoundError
return errors.As(err, &missing) || apierrors.IsNotFound(err)
}
type OrphanRuntime struct {
Kind string `json:"kind"`
Name string `json:"name"`
Namespace string `json:"namespace"`
RunID string `json:"runId"`
CreatedAt string `json:"createdAt"`
Cleaned bool `json:"cleaned"`
CleanupError string `json:"cleanupError,omitempty"`
}
type GPUCapacityNode struct {
Name string `json:"name"`
Allocatable int64 `json:"allocatable"`
SelectorMatched bool `json:"selectorMatched"`
Unschedulable bool `json:"unschedulable"`
}
type GPUCapacitySnapshot struct {
ResourceName string `json:"resourceName"`
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
SelectorSummary string `json:"selectorSummary"`
Nodes []GPUCapacityNode `json:"nodes"`
TotalAllocatable int64 `json:"totalAllocatable"`
SchedulableAllocatable int64 `json:"schedulableAllocatable"`
MaxAllocatable int64 `json:"maxAllocatable"`
SchedulableNodeCount int `json:"schedulableNodeCount"`
Available bool `json:"available"`
CheckedAt string `json:"checkedAt"`
}
type Launcher struct {
namespace string
imagePullSecretName string
workspacePVCName string
workspaceMountPath string
gpuNodeSelector map[string]string
gpuTolerations []corev1.Toleration
gpuRuntimeClass string
jobTTLSeconds int32
sharedMemorySize string
clientset kubernetes.Interface
restConfig *rest.Config
}
var invalidNameCharacters = regexp.MustCompile(`[^a-z0-9-]+`)
var launcherSystemEnvKeys = map[string]struct{}{
"RUN_ID": {},
"WORKFLOW_ID": {},
"NODE_ID": {},
"NODE_LABEL": {},
}
const defaultSharedMemorySize = "8Gi"
func sanitizeName(value string) string {
normalized := strings.ToLower(value)
normalized = invalidNameCharacters.ReplaceAllString(normalized, "-")
normalized = strings.Trim(normalized, "-")
normalized = regexp.MustCompile(`-+`).ReplaceAllString(normalized, "-")
return normalized
}
func launchAttempt(input LaunchInput) int {
if input.Attempt <= 0 {
return 1
}
return input.Attempt
}
func launchJobName(input LaunchInput) string {
attemptSuffix := fmt.Sprintf("-a%d", launchAttempt(input))
base := sanitizeName(fmt.Sprintf("%s-%s", input.RunID, input.NodeID))
if base == "" {
return "droboticflow-node-a1"
}
maxBaseLength := 63 - len(attemptSuffix)
if maxBaseLength < 1 {
maxBaseLength = 1
}
if len(base) > maxBaseLength {
sum := sha1.Sum([]byte(base))
hashSuffix := "-" + hex.EncodeToString(sum[:])[:8]
prefixLength := maxBaseLength - len(hashSuffix)
if prefixLength < 1 {
prefixLength = 1
}
base = strings.Trim(base[:prefixLength], "-") + hashSuffix
}
base = strings.Trim(base, "-")
if base == "" {
return "droboticflow-node-a1"
}
return strings.Trim(base+attemptSuffix, "-")
}
func launchSensitiveEnvSecretName(input LaunchInput) string {
base := launchJobName(input) + "-env"
if len(base) <= 63 {
return base
}
sum := sha1.Sum([]byte(base))
hashSuffix := "-" + hex.EncodeToString(sum[:])[:8]
prefixLength := 63 - len(hashSuffix)
return strings.Trim(base[:prefixLength], "-") + hashSuffix
}
func parseResourceQuantity(field string, value string) (*resource.Quantity, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return nil, nil
}
quantity, err := resource.ParseQuantity(trimmed)
if err != nil {
return nil, fmt.Errorf("%s must be a valid Kubernetes resource quantity: %w", field, err)
}
if quantity.Sign() <= 0 {
return nil, fmt.Errorf("%s must be greater than zero", field)
}
return &quantity, nil
}
func addResourceQuantity(resources corev1.ResourceList, name corev1.ResourceName, field string, value string) error {
quantity, err := parseResourceQuantity(field, value)
if err != nil {
return err
}
if quantity == nil {
return nil
}
resources[name] = *quantity
return nil
}
func ValidateResourceQuantity(field string, value string) error {
_, err := parseResourceQuantity(field, value)
return err
}
func addGPUQuantity(resources corev1.ResourceList, value string) {
if !hasPositiveGPU(value) {
return
}
resources[corev1.ResourceName("nvidia.com/gpu")] = resource.MustParse(strings.TrimSpace(value))
}
func hasPositiveGPU(value string) bool {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return false
}
count, err := strconv.Atoi(trimmed)
return err == nil && count > 0
}
func ValidateGPURequest(value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return nil
}
count, err := strconv.Atoi(trimmed)
if err != nil || count <= 0 {
return fmt.Errorf("gpu limit must be a positive integer")
}
return nil
}
func resourceRequirements(input LaunchInput) (corev1.ResourceRequirements, error) {
requests := corev1.ResourceList{}
limits := corev1.ResourceList{}
if err := addResourceQuantity(requests, corev1.ResourceCPU, "cpu request", input.CPURequest); err != nil {
return corev1.ResourceRequirements{}, err
}
if err := addResourceQuantity(requests, corev1.ResourceMemory, "memory request", input.MemoryRequest); err != nil {
return corev1.ResourceRequirements{}, err
}
if err := addResourceQuantity(limits, corev1.ResourceCPU, "cpu limit", input.CPULimit); err != nil {
return corev1.ResourceRequirements{}, err
}
if err := addResourceQuantity(limits, corev1.ResourceMemory, "memory limit", input.MemoryLimit); err != nil {
return corev1.ResourceRequirements{}, err
}
addGPUQuantity(requests, input.GPULimit)
addGPUQuantity(limits, input.GPULimit)
requirements := corev1.ResourceRequirements{}
if len(requests) > 0 {
requirements.Requests = requests
}
if len(limits) > 0 {
requirements.Limits = limits
}
return requirements, nil
}
func boolPtr(value bool) *bool {
return &value
}
func int64Ptr(value int64) *int64 {
return &value
}
func cleanVolumeSubPath(value string) (string, error) {
trimmed := strings.Trim(strings.TrimSpace(value), "/")
if trimmed == "" {
return "", nil
}
cleaned := pathClean(trimmed)
if cleaned == "." || strings.HasPrefix(cleaned, "../") || cleaned == ".." || filepath.IsAbs(cleaned) {
return "", fmt.Errorf("code sub path must be a relative workspace path")
}
return cleaned, nil
}
func pathClean(value string) string {
return filepath.ToSlash(filepath.Clean(filepath.FromSlash(value)))
}
func cleanMountPath(value string) (string, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return "", fmt.Errorf("code mount path is required")
}
if !strings.HasPrefix(trimmed, "/") {
return "", fmt.Errorf("code mount path must be absolute")
}
if trimmed == "/" || trimmed == strings.TrimRight(trimmed, "/") {
return trimmed, nil
}
return strings.TrimRight(trimmed, "/"), nil
}
func parseNodeSelector(value string) (map[string]string, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return nil, nil
}
result := map[string]string{}
for _, item := range strings.Split(trimmed, ",") {
trimmedItem := strings.TrimSpace(item)
if trimmedItem == "" {
continue
}
parts := strings.SplitN(trimmedItem, "=", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("invalid gpu node selector %q: expected key=value", trimmedItem)
}
key := strings.TrimSpace(parts[0])
val := strings.TrimSpace(parts[1])
if key == "" || val == "" {
return nil, fmt.Errorf("invalid gpu node selector %q: key and value are required", trimmedItem)
}
result[key] = val
}
if len(result) == 0 {
return nil, nil
}
return result, nil
}
// ParseNodeSelector exposes the launcher-compatible selector parser to the
// worker's narrowly scoped runtime configuration. Keeping one parser avoids a
// cache-only selector accepting syntax that the job launcher would reject.
func ParseNodeSelector(value string) (map[string]string, error) {
return parseNodeSelector(value)
}
func parseTolerations(value string) ([]corev1.Toleration, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return nil, nil
}
var items []corev1.Toleration
if err := json.Unmarshal([]byte(trimmed), &items); err != nil {
return nil, fmt.Errorf("invalid gpu tolerations json: %w", err)
}
if len(items) == 0 {
return nil, nil
}
return items, nil
}
func cloneStringMap(items map[string]string) map[string]string {
if len(items) == 0 {
return nil
}
cloned := make(map[string]string, len(items))
for key, value := range items {
cloned[key] = value
}
return cloned
}
func cloneTolerations(items []corev1.Toleration) []corev1.Toleration {
if len(items) == 0 {
return nil
}
cloned := make([]corev1.Toleration, len(items))
copy(cloned, items)
return cloned
}
func applyGPUScheduling(podSpec *corev1.PodSpec, input LaunchInput) {
if !hasPositiveGPU(input.GPULimit) {
return
}
if len(input.GPUNodeSelector) > 0 {
podSpec.NodeSelector = cloneStringMap(input.GPUNodeSelector)
}
if len(input.GPUTolerations) > 0 {
podSpec.Tolerations = cloneTolerations(input.GPUTolerations)
}
if trimmed := strings.TrimSpace(input.GPURuntimeClass); trimmed != "" {
podSpec.RuntimeClassName = &trimmed
}
}
func nodeMatchesSelector(node corev1.Node, selector map[string]string) bool {
if len(selector) == 0 {
return true
}
for key, expected := range selector {
if node.Labels[key] != expected {
return false
}
}
return true
}
func gpuAllocatableCount(node corev1.Node) int64 {
quantity, ok := node.Status.Allocatable[corev1.ResourceName("nvidia.com/gpu")]
if !ok {
return 0
}
value := quantity.Value()
if value < 0 {
return 0
}
return value
}
func gpuSelectorSummary(selector map[string]string) string {
if len(selector) == 0 {
return "<none>"
}
keys := make([]string, 0, len(selector))
for key := range selector {
keys = append(keys, key)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, key := range keys {
parts = append(parts, key+"="+selector[key])
}
return strings.Join(parts, ",")
}
func (launcher *Launcher) gpuCapacity(ctx context.Context, selector map[string]string) (*GPUCapacitySnapshot, error) {
nodes, err := launcher.clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("list kubernetes nodes: %w", err)
}
snapshot := &GPUCapacitySnapshot{
ResourceName: "nvidia.com/gpu",
NodeSelector: cloneStringMap(selector),
SelectorSummary: gpuSelectorSummary(selector),
Nodes: make([]GPUCapacityNode, 0, len(nodes.Items)),
CheckedAt: time.Now().UTC().Format(time.RFC3339Nano),
}
for _, node := range nodes.Items {
allocatable := gpuAllocatableCount(node)
selectorMatched := nodeMatchesSelector(node, selector)
snapshot.Nodes = append(snapshot.Nodes, GPUCapacityNode{
Name: node.Name,
Allocatable: allocatable,
SelectorMatched: selectorMatched,
Unschedulable: node.Spec.Unschedulable,
})
snapshot.TotalAllocatable += allocatable
if selectorMatched && !node.Spec.Unschedulable {
snapshot.SchedulableAllocatable += allocatable
if allocatable > snapshot.MaxAllocatable {
snapshot.MaxAllocatable = allocatable
}
if allocatable > 0 {
snapshot.SchedulableNodeCount++
}
}
}
snapshot.Available = snapshot.MaxAllocatable > 0
return snapshot, nil
}
func (launcher *Launcher) GPUCapacity(ctx context.Context) (*GPUCapacitySnapshot, error) {
return launcher.gpuCapacity(ctx, launcher.gpuNodeSelector)
}
func (launcher *Launcher) ensureGPUCapacity(ctx context.Context, input LaunchInput) error {
if !hasPositiveGPU(input.GPULimit) {
return nil
}
requested, err := strconv.ParseInt(strings.TrimSpace(input.GPULimit), 10, 64)
if err != nil || requested <= 0 {
return ValidateGPURequest(input.GPULimit)
}
snapshot, err := launcher.gpuCapacity(ctx, input.GPUNodeSelector)
if err != nil {
return fmt.Errorf("gpu preflight failed: %w", err)
}
if snapshot.MaxAllocatable >= requested {
return nil
}
checked := make([]string, 0, len(snapshot.Nodes))
for _, node := range snapshot.Nodes {
checked = append(
checked,
fmt.Sprintf("%s nvidia.com/gpu=%d selectorMatched=%t unschedulable=%t", node.Name, node.Allocatable, node.SelectorMatched, node.Unschedulable),
)
}
if len(checked) == 0 {
checked = append(checked, "none")
}
return fmt.Errorf(
"gpu request cannot be scheduled: requested nvidia.com/gpu=%d, selector=%s; checked nodes: %s",
requested,
snapshot.SelectorSummary,
strings.Join(checked, "; "),
)
}
func CreateJobManifest(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, input LaunchInput) (*batchv1.Job, error) {
if err := ValidateGPURequest(input.GPULimit); err != nil {
return nil, err
}
resources, err := resourceRequirements(input)
if err != nil {
return nil, err
}
var sidecarResources corev1.ResourceRequirements
if input.InferenceSidecar != nil {
sidecar := input.InferenceSidecar
if strings.TrimSpace(sidecar.Name) == "" || strings.TrimSpace(sidecar.Image) == "" || strings.ContainsAny(sidecar.Image, " \t\r\n") || sidecar.Port <= 0 || sidecar.Port > 65535 || !strings.HasPrefix(strings.TrimSpace(sidecar.HealthPath), "/") {
return nil, fmt.Errorf("inference sidecar requires a name, image, valid port and absolute health path")
}
if err := ValidateGPURequest(sidecar.GPULimit); err != nil {
return nil, fmt.Errorf("inference sidecar: %w", err)
}
sidecarResources, err = resourceRequirements(LaunchInput{CPURequest: sidecar.CPURequest, CPULimit: sidecar.CPULimit, MemoryRequest: sidecar.MemoryRequest, MemoryLimit: sidecar.MemoryLimit, GPULimit: sidecar.GPULimit})
if err != nil {
return nil, err
}
if hasPositiveGPU(sidecar.GPULimit) {
delete(resources.Requests, corev1.ResourceName("nvidia.com/gpu"))
delete(resources.Limits, corev1.ResourceName("nvidia.com/gpu"))
}
}
sharedMemorySize := strings.TrimSpace(input.SharedMemorySize)
if sharedMemorySize != "" {
quantity, quantityErr := resource.ParseQuantity(sharedMemorySize)
if quantityErr != nil || quantity.Value() <= 0 {
return nil, fmt.Errorf("shared memory size must be a positive Kubernetes resource quantity")
}
sharedMemorySize = quantity.String()
}
codeSubPath, err := cleanVolumeSubPath(input.CodeSubPath)
if err != nil {
return nil, err
}
codeMountPath := ""
if codeSubPath != "" {
codeMountPath, err = cleanMountPath(input.CodeMountPath)
if err != nil {
return nil, err
}
}
cleanPVCMounts := make([]PVCMount, 0, len(input.PVCMounts))
for _, item := range input.PVCMounts {
item.Name = strings.TrimSpace(item.Name)
item.PVCName = strings.TrimSpace(item.PVCName)
item.SubPath, err = cleanVolumeSubPath(item.SubPath)
if err != nil {
return nil, fmt.Errorf("PVC mount %s: %w", item.Name, err)
}
item.MountPath, err = cleanMountPath(item.MountPath)
if err != nil {
return nil, fmt.Errorf("PVC mount %s: %w", item.Name, err)
}
if item.Name == "" || item.PVCName == "" {
return nil, fmt.Errorf("PVC mount name and pvcName are required")
}
cleanPVCMounts = append(cleanPVCMounts, item)
}
jobName := launchJobName(input)
envVars := []corev1.EnvVar{
{Name: "RUN_ID", Value: input.RunID},
{Name: "WORKFLOW_ID", Value: input.WorkflowID},
{Name: "NODE_ID", Value: input.NodeID},
{Name: "NODE_LABEL", Value: input.NodeLabel},
{Name: "RUN_ATTEMPT", Value: strconv.Itoa(launchAttempt(input))},
}
envKeys := make([]string, 0, len(input.Env))
for key := range input.Env {
if _, reserved := launcherSystemEnvKeys[key]; reserved {
continue
}
envKeys = append(envKeys, key)
}
sort.Strings(envKeys)
for _, key := range envKeys {
if _, sensitive := input.SensitiveEnv[key]; sensitive {
continue
}
value := input.Env[key]
envVars = append(envVars, corev1.EnvVar{Name: key, Value: value})
}
sensitiveKeys := make([]string, 0, len(input.SensitiveEnv))
for key := range input.SensitiveEnv {
if _, reserved := launcherSystemEnvKeys[key]; reserved {
continue
}
sensitiveKeys = append(sensitiveKeys, key)
}
sort.Strings(sensitiveKeys)
sensitiveSecretName := launchSensitiveEnvSecretName(input)
for _, key := range sensitiveKeys {
envVars = append(envVars, corev1.EnvVar{
Name: key,
ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: sensitiveSecretName},
Key: key,
}},
})
}
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: jobName,
Namespace: namespace,
Labels: map[string]string{
"app": "droboticflow",
"component": "operator-runner",
"runId": input.RunID,
"workflowId": input.WorkflowID,
"nodeId": input.NodeID,
"attempt": strconv.Itoa(launchAttempt(input)),
},
},
Spec: batchv1.JobSpec{
BackoffLimit: func() *int32 { value := defaultJobBackoffLimit; return &value }(),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "droboticflow",
"component": "operator-runner",
"runId": input.RunID,
"nodeId": input.NodeID,
"attempt": strconv.Itoa(launchAttempt(input)),
},
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
SecurityContext: &corev1.PodSecurityContext{
RunAsNonRoot: boolPtr(true),
RunAsUser: int64Ptr(10001),
RunAsGroup: int64Ptr(10001),
FSGroup: int64Ptr(10001),
SeccompProfile: &corev1.SeccompProfile{
Type: corev1.SeccompProfileTypeRuntimeDefault,
},
},
Containers: []corev1.Container{
{
Name: "operator-runner",
Image: input.Image,
ImagePullPolicy: corev1.PullAlways,
Command: input.Command,
Args: input.Args,
Env: envVars,
Resources: resources,
SecurityContext: &corev1.SecurityContext{
AllowPrivilegeEscalation: boolPtr(false),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"ALL"},
},
},
},
},
},
},
},
}
if input.InferenceSidecar != nil {
sidecar := input.InferenceSidecar
restartAlways := corev1.ContainerRestartPolicyAlways
sidecarEnv := make([]corev1.EnvVar, 0, len(sidecar.Env))
sidecarKeys := make([]string, 0, len(sidecar.Env))
for key := range sidecar.Env {
sidecarKeys = append(sidecarKeys, key)
}
sort.Strings(sidecarKeys)
for _, key := range sidecarKeys {
sidecarEnv = append(sidecarEnv, corev1.EnvVar{Name: key, Value: sidecar.Env[key]})
}
job.Spec.Template.Spec.InitContainers = []corev1.Container{{
Name: sidecar.Name, Image: sidecar.Image, ImagePullPolicy: corev1.PullAlways,
RestartPolicy: &restartAlways, Env: sidecarEnv, Resources: sidecarResources,
Ports: []corev1.ContainerPort{{Name: "inference-http", ContainerPort: sidecar.Port, Protocol: corev1.ProtocolTCP}},
StartupProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: sidecar.HealthPath, Port: intstr.FromInt32(sidecar.Port)}},
InitialDelaySeconds: 2, PeriodSeconds: 5, TimeoutSeconds: 3, FailureThreshold: 120,
},
SecurityContext: &corev1.SecurityContext{RunAsNonRoot: boolPtr(false), RunAsUser: int64Ptr(0), AllowPrivilegeEscalation: boolPtr(false)},
}}
}
if sharedMemorySize != "" {
quantity := resource.MustParse(sharedMemorySize)
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "droboticflow-shm",
VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{
Medium: corev1.StorageMediumMemory,
SizeLimit: &quantity,
}},
})
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "droboticflow-shm",
MountPath: "/dev/shm",
})
if len(job.Spec.Template.Spec.InitContainers) > 0 {
job.Spec.Template.Spec.InitContainers[0].VolumeMounts = append(job.Spec.Template.Spec.InitContainers[0].VolumeMounts, corev1.VolumeMount{Name: "droboticflow-shm", MountPath: "/dev/shm"})
}
}
if input.JobTTLSeconds > 0 {
job.Spec.TTLSecondsAfterFinished = func() *int32 {
value := input.JobTTLSeconds
return &value
}()
}
if input.TimeoutSeconds > 0 {
job.Spec.ActiveDeadlineSeconds = func() *int64 {
value := int64(input.TimeoutSeconds)
return &value
}()
}
if imagePullSecretName != "" {
job.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: imagePullSecretName}}
}
applyGPUScheduling(&job.Spec.Template.Spec, input)
if strings.TrimSpace(workspacePVCName) != "" {
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "workspace",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: workspacePVCName,
},
},
})
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "workspace",
MountPath: workspaceMountPath,
ReadOnly: input.CodeReadOnly,
})
if len(job.Spec.Template.Spec.InitContainers) > 0 {
job.Spec.Template.Spec.InitContainers[0].VolumeMounts = append(job.Spec.Template.Spec.InitContainers[0].VolumeMounts, corev1.VolumeMount{Name: "workspace", MountPath: workspaceMountPath})
}
if codeSubPath != "" {
runSubPath, err := cleanVolumeSubPath(pathClean("system/runs/" + input.RunID))
if err != nil {
return nil, err
}
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "workspace",
MountPath: strings.TrimRight(workspaceMountPath, "/") + "/system/runs/" + input.RunID,
SubPath: runSubPath,
})
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "workspace",
MountPath: codeMountPath,
SubPath: codeSubPath,
ReadOnly: input.CodeReadOnly,
})
}
}
volumeNames := map[string]string{workspacePVCName: "workspace"}
for index, item := range cleanPVCMounts {
volumeName, exists := volumeNames[item.PVCName]
if !exists {
volumeName = fmt.Sprintf("lightweight-pvc-%d", index)
volumeNames[item.PVCName] = volumeName
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: volumeName,
VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: item.PVCName}},
})
}
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: volumeName, MountPath: item.MountPath, SubPath: item.SubPath, ReadOnly: item.ReadOnly,
})
}
if input.RegistryAuth && imagePullSecretName != "" {
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "registry-auth",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: imagePullSecretName,
Items: []corev1.KeyToPath{
{Key: ".dockerconfigjson", Path: "config.json"},
},
},
},
})
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "registry-auth",
MountPath: "/kaniko/.docker",
})
}
return job, nil
}
func loadRestConfig() (*rest.Config, error) {
if config, err := rest.InClusterConfig(); err == nil {
return config, nil
}
kubeconfig := os.Getenv("KUBECONFIG")
if kubeconfig == "" {
home, err := os.UserHomeDir()
if err != nil {
return nil, err
}
kubeconfig = filepath.Join(home, ".kube", "config")
}
return clientcmd.BuildConfigFromFlags("", kubeconfig)
}
func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int) (*Launcher, error) {
trimmedPVCName := strings.TrimSpace(workspacePVCName)
if trimmedPVCName == "" {
return nil, fmt.Errorf("workspace pvc name is required")
}
trimmedMountPath := strings.TrimSpace(workspaceMountPath)
if trimmedMountPath == "" {
trimmedMountPath = "/workspace"
}
if !strings.HasPrefix(trimmedMountPath, "/") {
return nil, fmt.Errorf("workspace mount path must be absolute")
}
parsedGPUNodeSelector, err := parseNodeSelector(gpuNodeSelector)
if err != nil {
return nil, err
}
parsedGPUTolerations, err := parseTolerations(gpuTolerationsJSON)
if err != nil {
return nil, err
}
if jobTTLSeconds < 0 {
jobTTLSeconds = 0
}
sharedMemorySize := strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHARED_MEMORY_SIZE"))
if sharedMemorySize == "" {
sharedMemorySize = strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHM_SIZE"))
}
if sharedMemorySize == "" {
sharedMemorySize = defaultSharedMemorySize
}
sharedMemoryQuantity, err := resource.ParseQuantity(sharedMemorySize)
if err != nil || sharedMemoryQuantity.Value() <= 0 {
return nil, fmt.Errorf("DROBOTICFLOW_SHARED_MEMORY_SIZE must be a positive Kubernetes resource quantity")
}
launcher := &Launcher{
namespace: namespace,
imagePullSecretName: imagePullSecretName,
workspacePVCName: trimmedPVCName,
workspaceMountPath: trimmedMountPath,
gpuNodeSelector: parsedGPUNodeSelector,
gpuTolerations: parsedGPUTolerations,
gpuRuntimeClass: strings.TrimSpace(gpuRuntimeClass),
jobTTLSeconds: int32(jobTTLSeconds),
sharedMemorySize: sharedMemoryQuantity.String(),
}
config, err := loadRestConfig()
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
launcher.clientset = clientset
launcher.restConfig = config
return launcher, nil
}
func waitDeadlineForLaunch(ctx context.Context, input LaunchInput, now time.Time) time.Time {
if deadline, ok := ctx.Deadline(); ok {
return deadline
}
if input.TimeoutSeconds > 0 {
return now.Add(time.Duration(input.TimeoutSeconds)*time.Second + 30*time.Second)
}
if input.LongRunning {
return now.Add(longRunningJobWaitTimeout)
}
return now.Add(defaultJobWaitTimeout)
}
func (launcher *Launcher) readJobOutput(ctx context.Context, jobName string) (string, error) {
var podName string
for attempt := 0; attempt < 30; attempt++ {
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{
LabelSelector: "job-name=" + jobName,
})
if err != nil {
return "", err
}
if len(pods.Items) > 0 {
podName = pods.Items[0].Name
break
}
time.Sleep(time.Second)
}
if podName == "" {
return "", fmt.Errorf("no pod found for job %s", jobName)
}
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, &corev1.PodLogOptions{}).Stream(ctx)
if err != nil {
return "", err
}
defer stream.Close()
output, err := io.ReadAll(stream)
if err != nil {
return "", err
}
return string(output), nil
}
func (launcher *Launcher) readJobOutputNow(ctx context.Context, jobName string) (string, error) {
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName})
if err != nil {
return "", err
}
if len(pods.Items) == 0 {
return "", fmt.Errorf("no pod found for job %s", jobName)
}
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}).Stream(ctx)
if err != nil {
return "", err
}
defer stream.Close()
output, err := io.ReadAll(stream)
if err != nil {
return "", err
}
return string(output), nil
}
// findJobPodName returns the name of the (first) pod backing jobName, waiting up
// to ~30s for it to exist. Returns "" if no pod appears in time.
func (launcher *Launcher) findJobPodName(ctx context.Context, jobName string) string {
for attempt := 0; attempt < 30; attempt++ {
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{
LabelSelector: "job-name=" + jobName,
})
if err == nil && len(pods.Items) > 0 {
return pods.Items[0].Name
}
select {
case <-ctx.Done():
return ""
case <-time.After(time.Second):
}
}
return ""
}
// podGPUSampleInterval controls how often the platform samples GPU usage inside a
// custom-container pod. Remote exec is heavier than operator-runner's in-process
// sampler, so the default is 10s (vs 1s). Override via
// DROBOTICFLOW_POD_GPU_SAMPLE_INTERVAL_SECONDS (clamped to [1, 300]).
func podGPUSampleInterval() time.Duration {
raw := strings.TrimSpace(os.Getenv("DROBOTICFLOW_POD_GPU_SAMPLE_INTERVAL_SECONDS"))
if raw == "" {
return 10 * time.Second
}
value, err := strconv.ParseFloat(raw, 64)
if err != nil || value < 1 {
return 10 * time.Second
}
if value > 300 {
value = 300
}
return time.Duration(value * float64(time.Second))
}
// parseGPUSampleOutput parses `nvidia-smi --query-gpu=utilization.gpu,memory.used,
// memory.total,name --format=csv,noheader,nounits` output into a single rolled-up
// GPUSample (max util across GPUs, summed memory, distinct names). Mirrors the
// rollup in backend/operator-runner/resource_monitor.py so platform-collected
// samples are directly comparable to operator-runner self-reported ones.
func parseGPUSampleOutput(output string) GPUSample {
type gpuLine struct {
util int
used int
total int
name string
}
var lines []gpuLine
for _, raw := range strings.Split(output, "\n") {
parts := strings.Split(strings.TrimSpace(raw), ",")
if len(parts) < 4 {
continue
}
util, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
used, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
total, err3 := strconv.Atoi(strings.TrimSpace(parts[2]))
if err1 != nil || err2 != nil || err3 != nil {
continue
}
lines = append(lines, gpuLine{util: util, used: used, total: total, name: strings.TrimSpace(parts[3])})
}
if len(lines) == 0 {
return GPUSample{}
}
maxUtil := 0
sumUsed := 0
sumTotal := 0
nameSet := make(map[string]struct{})
names := make([]string, 0)
for _, line := range lines {
if line.util > maxUtil {
maxUtil = line.util
}
sumUsed += line.used
sumTotal += line.total
if _, ok := nameSet[line.name]; !ok {
nameSet[line.name] = struct{}{}
names = append(names, line.name)
}
}
if maxUtil > 100 {
maxUtil = 100
}
if maxUtil < 0 {
maxUtil = 0
}
return GPUSample{
Timestamp: time.Now().UTC().Format("2006-01-02T15:04:05.000000Z"),
GPU: maxUtil,
GPUMemoryUsedMiB: sumUsed,
GPUMemoryTotalMiB: sumTotal,
GPUName: strings.Join(names, ", "),
GPUCount: len(lines),
}
}
// samplePodGPU execs nvidia-smi inside the pod's operator-runner container and
// returns one rolled-up GPUSample. Fails gracefully (returns error, zero sample)
// when the pod has no GPU, nvidia-smi is absent, restConfig is unavailable, or the
// worker lacks pods/exec RBAC — the caller simply skips that sample.
func (launcher *Launcher) samplePodGPU(ctx context.Context, podName string) (GPUSample, error) {
empty := GPUSample{}
if launcher.restConfig == nil {
return empty, fmt.Errorf("rest config unavailable")
}
request := launcher.clientset.CoreV1().RESTClient().Post().
Resource("pods").
Namespace(launcher.namespace).
Name(podName).
SubResource("exec").
VersionedParams(&corev1.PodExecOptions{
Container: "operator-runner",
Command: []string{
"nvidia-smi",
"--query-gpu=utilization.gpu,memory.used,memory.total,name",
"--format=csv,noheader,nounits",
},
Stdout: true,
Stderr: false,
}, scheme.ParameterCodec)
executor, err := remotecommand.NewSPDYExecutor(launcher.restConfig, http.MethodPost, request.URL())
if err != nil {
return empty, err
}
var stdout bytes.Buffer
if err := executor.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: &stdout}); err != nil {
return empty, err
}
return parseGPUSampleOutput(stdout.String()), nil
}
// findRunningNodePod returns the name of the first Running pod backing runID+nodeID,
// listed via the runId/nodeId labels every operator pod carries. Returns "" (no
// error) when no pod is Running yet — callers treat that as "not live right now".
func (launcher *Launcher) findRunningNodePod(ctx context.Context, runID string, nodeID string) (string, error) {
runID = strings.TrimSpace(runID)
nodeID = strings.TrimSpace(nodeID)
if runID == "" || nodeID == "" {
return "", fmt.Errorf("runID and nodeID are required")
}
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{
LabelSelector: "runId=" + runID + ",nodeId=" + nodeID,
})
if err != nil {
return "", err
}
for _, pod := range pods.Items {
if pod.Status.Phase == corev1.PodRunning {
return pod.Name, nil
}
}
return "", nil
}
// SamplePodGPUForRun samples GPU usage (nvidia-smi) inside the pod backing
// runID+nodeID. The worker uses it to surface live GPU metrics for custom-container
// operators DURING execution — those bypass operator-runner's in-process
// ResourceSampler, so otherwise the only GPU samples appear at node completion (when
// the launcher flushes its internal buffer). Fails gracefully (returns error) when
// no Running pod exists, the pod has no GPU, nvidia-smi is absent, or pods/exec
// RBAC is unavailable — the caller simply skips that sample.
func (launcher *Launcher) SamplePodGPUForRun(ctx context.Context, runID string, nodeID string) (GPUSample, error) {
empty := GPUSample{}
if launcher.restConfig == nil {
return empty, fmt.Errorf("rest config unavailable")
}
podName, err := launcher.findRunningNodePod(ctx, runID, nodeID)
if err != nil {
return empty, err
}
if podName == "" {
return empty, fmt.Errorf("no running pod for run %s node %s", runID, nodeID)
}
return launcher.samplePodGPU(ctx, podName)
}
// TailNodePodLogs returns the last tailLines of stdout from the operator-runner
// container of the pod backing runID+nodeID. The API uses it to surface live
// execution logs for a running node before the operator emits its structured
// result.json (which the worker persists only at node completion). Returns "" when
// no Running pod exists, so the caller falls back to the stored run logs.
func (launcher *Launcher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, false)
}
// TailNodePodLogsWithTimestamps keeps the Kubernetes source timestamp on each
// line so repeated API polls can deduplicate the same live log entry.
func (launcher *Launcher) TailNodePodLogsWithTimestamps(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, true)
}
func (launcher *Launcher) tailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64, timestamps bool) (string, error) {
if tailLines <= 0 {
tailLines = 300
}
podName, err := launcher.findRunningNodePod(ctx, runID, nodeID)
if err != nil {
return "", err
}
if podName == "" {
return "", nil
}
tail := tailLines
options := &corev1.PodLogOptions{
Container: "operator-runner",
TailLines: &tail,
}
if timestamps {
options.Timestamps = true
}
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, options).Stream(ctx)
if err != nil {
return "", err
}
defer stream.Close()
output, err := io.ReadAll(stream)
if err != nil {
return "", err
}
return string(output), nil
}
func appendDiagnosticLine(builder *strings.Builder, format string, args ...any) {
line := strings.TrimSpace(fmt.Sprintf(format, args...))
if line == "" {
return
}
if builder.Len() > 0 {
builder.WriteString("\n")
}
builder.WriteString(line)
}
func containerStateDiagnostic(name string, state corev1.ContainerState) string {
switch {
case state.Waiting != nil:
return fmt.Sprintf("container %s waiting reason=%s message=%s", name, state.Waiting.Reason, state.Waiting.Message)
case state.Terminated != nil:
return fmt.Sprintf("container %s terminated exitCode=%d reason=%s message=%s", name, state.Terminated.ExitCode, state.Terminated.Reason, state.Terminated.Message)
case state.Running != nil:
return fmt.Sprintf("container %s running startedAt=%s", name, state.Running.StartedAt.Time.Format(time.RFC3339))
default:
return ""
}
}
func (launcher *Launcher) readJobDiagnostics(ctx context.Context, job *batchv1.Job, logs string, logErr error) string {
var builder strings.Builder
if job != nil {
for _, condition := range job.Status.Conditions {
appendDiagnosticLine(&builder, "job condition type=%s status=%s reason=%s message=%s", condition.Type, condition.Status, condition.Reason, condition.Message)
}
}
jobName := ""
if job != nil {
jobName = job.Name
}
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{
LabelSelector: "job-name=" + jobName,
})
if err != nil {
appendDiagnosticLine(&builder, "pod inspection failed: %v", err)
} else if len(pods.Items) == 0 {
appendDiagnosticLine(&builder, "no pod found for job %s", jobName)
}
for _, pod := range pods.Items {
appendDiagnosticLine(&builder, "pod %s phase=%s reason=%s message=%s", pod.Name, pod.Status.Phase, pod.Status.Reason, pod.Status.Message)
for _, condition := range pod.Status.Conditions {
if condition.Status != corev1.ConditionTrue {
appendDiagnosticLine(&builder, "pod %s condition type=%s status=%s reason=%s message=%s", pod.Name, condition.Type, condition.Status, condition.Reason, condition.Message)
}
}
for _, status := range pod.Status.InitContainerStatuses {
if diagnostic := containerStateDiagnostic(status.Name, status.State); diagnostic != "" {
appendDiagnosticLine(&builder, "pod %s %s", pod.Name, diagnostic)
}
}
for _, status := range pod.Status.ContainerStatuses {
if diagnostic := containerStateDiagnostic(status.Name, status.State); diagnostic != "" {
appendDiagnosticLine(&builder, "pod %s %s", pod.Name, diagnostic)
}
}
events, eventErr := launcher.clientset.CoreV1().Events(launcher.namespace).List(ctx, metav1.ListOptions{
FieldSelector: "involvedObject.name=" + pod.Name,
})
if eventErr != nil {
appendDiagnosticLine(&builder, "pod %s event inspection failed: %v", pod.Name, eventErr)
continue
}
sort.Slice(events.Items, func(i int, j int) bool {
left := events.Items[i].LastTimestamp.Time
if left.IsZero() {
left = events.Items[i].EventTime.Time
}
right := events.Items[j].LastTimestamp.Time
if right.IsZero() {
right = events.Items[j].EventTime.Time
}
return left.Before(right)
})
for _, event := range events.Items {
appendDiagnosticLine(&builder, "pod %s event type=%s reason=%s message=%s", pod.Name, event.Type, event.Reason, event.Message)
}
}
if trimmed := strings.TrimSpace(logs); trimmed != "" {
appendDiagnosticLine(&builder, "pod logs:\n%s", trimmed)
} else if logErr != nil {
appendDiagnosticLine(&builder, "pod log read failed: %v", logErr)
}
return builder.String()
}
func (launcher *Launcher) TerminateJob(ctx context.Context, namespace string, jobName string) error {
targetNamespace := strings.TrimSpace(namespace)
if targetNamespace == "" {
targetNamespace = launcher.namespace
}
propagation := metav1.DeletePropagationBackground
err := launcher.clientset.BatchV1().Jobs(targetNamespace).Delete(ctx, jobName, metav1.DeleteOptions{
PropagationPolicy: &propagation,
})
if err != nil && strings.Contains(strings.ToLower(err.Error()), "not found") {
return nil
}
return err
}
func (launcher *Launcher) TerminateRun(ctx context.Context, namespace string, runID string) error {
if launcher.clientset == nil || strings.TrimSpace(runID) == "" {
return nil
}
targetNamespace := strings.TrimSpace(namespace)
if targetNamespace == "" {
targetNamespace = launcher.namespace
}
jobs, err := launcher.clientset.BatchV1().Jobs(targetNamespace).List(
ctx,
metav1.ListOptions{LabelSelector: "runId=" + strings.TrimSpace(runID)},
)
if err != nil {
return err
}
for _, job := range jobs.Items {
if err := launcher.TerminateJob(ctx, targetNamespace, job.Name); err != nil {
return err
}
}
pods, err := launcher.clientset.CoreV1().Pods(targetNamespace).List(
ctx,
metav1.ListOptions{LabelSelector: "runId=" + strings.TrimSpace(runID)},
)
if err != nil {
return err
}
propagation := metav1.DeletePropagationBackground
for _, pod := range pods.Items {
err := launcher.clientset.CoreV1().Pods(targetNamespace).Delete(ctx, pod.Name, metav1.DeleteOptions{PropagationPolicy: &propagation})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
}
return nil
}
// TerminateRunAndWait must finish before deleting a Run's writable workspace.
func (launcher *Launcher) TerminateRunAndWait(ctx context.Context, namespace, runID string) error {
if launcher.clientset == nil {
return fmt.Errorf("Kubernetes client unavailable")
}
ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
defer cancel()
if namespace == "" {
namespace = launcher.namespace
}
for {
if err := launcher.TerminateRun(ctx, namespace, runID); err != nil {
return err
}
jobs, err := launcher.clientset.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId=" + runID})
if err != nil {
return err
}
pods, err := launcher.clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId=" + runID})
if err != nil {
return err
}
if len(jobs.Items) == 0 && len(pods.Items) == 0 {
return nil
}
select {
case <-ctx.Done():
return fmt.Errorf("runtime termination not verified: %w", ctx.Err())
case <-time.After(time.Second):
}
}
}
func orphanRuntimeCreatedAt(timestamp metav1.Time) string {
if timestamp.IsZero() {
return ""
}
return timestamp.UTC().Format(time.RFC3339Nano)
}
func (launcher *Launcher) ReconcileOrphanedRuntimes(ctx context.Context, knownRunIDs map[string]struct{}, olderThan time.Time, cleanup bool) ([]OrphanRuntime, error) {
if launcher == nil || launcher.clientset == nil {
return nil, nil
}
result := make([]OrphanRuntime, 0)
jobs, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId"})
if err != nil {
return nil, err
}
for _, job := range jobs.Items {
runID := strings.TrimSpace(job.Labels["runId"])
if runID == "" || !job.CreationTimestamp.Time.Before(olderThan) {
continue
}
if _, exists := knownRunIDs[runID]; exists {
continue
}
item := OrphanRuntime{
Kind: "Job",
Name: job.Name,
Namespace: job.Namespace,
RunID: runID,
CreatedAt: orphanRuntimeCreatedAt(job.CreationTimestamp),
}
if cleanup {
if err := launcher.TerminateJob(ctx, job.Namespace, job.Name); err != nil {
item.CleanupError = err.Error()
} else {
item.Cleaned = true
}
}
result = append(result, item)
}
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId"})
if err != nil {
return nil, err
}
for _, pod := range pods.Items {
runID := strings.TrimSpace(pod.Labels["runId"])
if runID == "" || !pod.CreationTimestamp.Time.Before(olderThan) {
continue
}
if _, exists := knownRunIDs[runID]; exists {
continue
}
item := OrphanRuntime{
Kind: "Pod",
Name: pod.Name,
Namespace: pod.Namespace,
RunID: runID,
CreatedAt: orphanRuntimeCreatedAt(pod.CreationTimestamp),
}
if cleanup {
propagation := metav1.DeletePropagationBackground
err := launcher.clientset.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{PropagationPolicy: &propagation})
if err != nil && !apierrors.IsNotFound(err) {
item.CleanupError = err.Error()
} else {
item.Cleaned = true
}
}
result = append(result, item)
}
sort.Slice(result, func(i int, j int) bool {
if result[i].RunID == result[j].RunID {
if result[i].Kind == result[j].Kind {
return result[i].Name < result[j].Name
}
return result[i].Kind < result[j].Kind
}
return result[i].RunID < result[j].RunID
})
return result, nil
}
func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*LaunchResult, error) {
if err := ValidateGPURequest(input.GPULimit); err != nil {
return nil, err
}
if len(input.GPUNodeSelector) == 0 {
input.GPUNodeSelector = cloneStringMap(launcher.gpuNodeSelector)
}
if len(input.GPUTolerations) == 0 {
input.GPUTolerations = cloneTolerations(launcher.gpuTolerations)
}
if strings.TrimSpace(input.GPURuntimeClass) == "" {
input.GPURuntimeClass = launcher.gpuRuntimeClass
}
if input.JobTTLSeconds == 0 {
input.JobTTLSeconds = launcher.jobTTLSeconds
}
if strings.TrimSpace(input.SharedMemorySize) == "" {
input.SharedMemorySize = launcher.sharedMemorySize
}
job, err := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, launcher.workspacePVCName, launcher.workspaceMountPath, input)
if err != nil {
return nil, err
}
if err := launcher.ensureGPUCapacity(ctx, input); err != nil {
return nil, err
}
created, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Create(ctx, job, metav1.CreateOptions{})
jobCreated := err == nil
if err != nil {
if !apierrors.IsAlreadyExists(err) {
return nil, err
}
created, err = launcher.clientset.BatchV1().Jobs(launcher.namespace).Get(ctx, job.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}
}
if len(input.SensitiveEnv) > 0 {
secretData := make(map[string]string, len(input.SensitiveEnv))
for key, value := range input.SensitiveEnv {
if _, reserved := launcherSystemEnvKeys[key]; reserved {
continue
}
secretData[key] = value
}
controller := true
blockOwnerDeletion := true
sensitiveSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: launchSensitiveEnvSecretName(input), Namespace: launcher.namespace,
Labels: map[string]string{"app": "droboticflow", "component": "operator-sensitive-env", "runId": input.RunID, "nodeId": input.NodeID},
OwnerReferences: []metav1.OwnerReference{{
APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", Name: created.Name, UID: created.UID,
Controller: &controller, BlockOwnerDeletion: &blockOwnerDeletion,
}},
},
Type: corev1.SecretTypeOpaque, StringData: secretData,
}
if _, secretErr := launcher.clientset.CoreV1().Secrets(launcher.namespace).Create(ctx, sensitiveSecret, metav1.CreateOptions{}); secretErr != nil && !apierrors.IsAlreadyExists(secretErr) {
if jobCreated {
propagation := metav1.DeletePropagationBackground
_ = launcher.clientset.BatchV1().Jobs(launcher.namespace).Delete(ctx, created.Name, metav1.DeleteOptions{PropagationPolicy: &propagation})
}
return nil, fmt.Errorf("create owner-referenced operator sensitive environment secret: %w", secretErr)
}
}
jobName := created.Name
reference := &JobReference{JobName: jobName, Namespace: launcher.namespace}
waitParentCtx := ctx
stopCancellationForwarder := func() {}
if input.LongRunning {
// Long-running training deliberately ignores ordinary workflow/node deadlines,
// but it must still detach promptly when the worker shuts down or loses its
// execution lease. The Kubernetes Job is left in place for the next worker to
// reconcile; only short-lived jobs are deleted on cancellation below.
detachedCtx, cancelDetached := context.WithCancel(context.Background())
forwarderDone := make(chan struct{})
go func() {
select {
case <-ctx.Done():
if errors.Is(ctx.Err(), context.Canceled) {
cancelDetached()
}
case <-forwarderDone:
}
}()
waitParentCtx = detachedCtx
stopCancellationForwarder = func() {
close(forwarderDone)
cancelDetached()
}
}
defer stopCancellationForwarder()
waitCtx, cancel := context.WithDeadline(waitParentCtx, waitDeadlineForLaunch(waitParentCtx, input, time.Now()))
defer cancel()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
// Platform-side GPU sampler for custom-container operators. Builtin
// operator-runner pods self-report resourceSeries via main.py's
// ResourceSampler, so they are skipped (resolved Image == OperatorRunnerImage).
// Custom containers bypass main.py and would otherwise show GPU=0 on the
// resource monitor; the launcher samples nvidia-smi inside the pod instead.
samplerCtx, samplerCancel := context.WithCancel(context.Background())
defer samplerCancel()
var gpuSamples []GPUSample
var sampleMu sync.Mutex
samplerDone := make(chan struct{})
shouldSample := launcher.restConfig != nil &&
input.OperatorRunnerImage != "" &&
input.Image != input.OperatorRunnerImage
if shouldSample {
go func() {
defer close(samplerDone)
probeTicker := time.NewTicker(2 * time.Second)
defer probeTicker.Stop()
sampleTicker := time.NewTicker(podGPUSampleInterval())
defer sampleTicker.Stop()
podName := ""
running := false
for {
select {
case <-samplerCtx.Done():
return
case <-probeTicker.C:
if podName == "" {
podName = launcher.findJobPodName(samplerCtx, jobName)
if podName == "" {
continue
}
}
pod, err := launcher.clientset.CoreV1().Pods(launcher.namespace).Get(samplerCtx, podName, metav1.GetOptions{})
if err != nil {
continue
}
running = pod.Status.Phase == corev1.PodRunning
case <-sampleTicker.C:
if !running || podName == "" {
continue
}
sampleCallCtx, cancelCall := context.WithTimeout(samplerCtx, 5*time.Second)
sample, err := launcher.samplePodGPU(sampleCallCtx, podName)
cancelCall()
if err != nil {
continue
}
sampleMu.Lock()
gpuSamples = append(gpuSamples, sample)
if len(gpuSamples) > 7200 {
gpuSamples = gpuSamples[len(gpuSamples)-7200:]
}
sampleMu.Unlock()
}
}
}()
} else {
close(samplerDone)
}
stopSampler := func() []GPUSample {
samplerCancel()
<-samplerDone
sampleMu.Lock()
defer sampleMu.Unlock()
return append([]GPUSample(nil), gpuSamples...)
}
checkStatus := func() (*LaunchResult, error) {
status, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Get(waitCtx, jobName, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return nil, &JobNotFoundError{JobName: jobName}
}
return nil, err
}
if status.Status.Succeeded > 0 {
output, err := launcher.readJobOutput(waitCtx, jobName)
if err != nil {
return nil, err
}
return &LaunchResult{Reference: reference, Output: output}, nil
}
if status.Status.Failed > 0 {
output, outputErr := launcher.readJobOutput(waitCtx, jobName)
diagnostics := strings.TrimSpace(launcher.readJobDiagnostics(waitCtx, status, output, outputErr))
if diagnostics == "" {
return nil, fmt.Errorf("kubernetes job %s failed", jobName)
}
return nil, fmt.Errorf("kubernetes job %s failed:\n%s", jobName, diagnostics)
}
return nil, nil
}
for {
result, err := checkStatus()
if result != nil || err != nil {
if result != nil {
result.ResourceSeries = stopSampler()
} else {
stopSampler()
}
return result, err
}
select {
case <-waitCtx.Done():
stopSampler()
// A cancelled worker context is not proof that the business Job should
// be deleted. Worker rollouts and lease hand-offs cancel this waiter too;
// the surviving Job is intentionally left for reconciliation. Explicit
// user stops and terminal Run cleanup call TerminateRun separately.
if errors.Is(waitCtx.Err(), context.DeadlineExceeded) {
return nil, fmt.Errorf("timed out waiting for kubernetes job %s", jobName)
}
return nil, waitCtx.Err()
case <-ticker.C:
}
}
}
var terminalPendingReasons = map[string]struct{}{
"CreateContainerConfigError": {},
"CreateContainerError": {},
"ErrImagePull": {},
"ImagePullBackOff": {},
"InvalidImageName": {},
"Unschedulable": {},
}
func terminalPendingReason(reason string) bool {
_, ok := terminalPendingReasons[strings.TrimSpace(reason)]
return ok
}
func podBlockingState(pod corev1.Pod) (string, string) {
for _, condition := range pod.Status.Conditions {
if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && terminalPendingReason(condition.Reason) {
return condition.Reason, condition.Message
}
}
for _, status := range append(append([]corev1.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
if status.State.Waiting != nil && terminalPendingReason(status.State.Waiting.Reason) {
return status.State.Waiting.Reason, status.State.Waiting.Message
}
}
if terminalPendingReason(pod.Status.Reason) {
return pod.Status.Reason, pod.Status.Message
}
return "", ""
}
func (launcher *Launcher) inspectBlockingState(ctx context.Context, jobName string) (string, string, error) {
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName})
if err != nil {
return "", "", err
}
for _, pod := range pods.Items {
if reason, message := podBlockingState(pod); reason != "" {
return reason, message, nil
}
}
return "", "", nil
}
func (launcher *Launcher) Inspect(ctx context.Context, runID string, nodeID string) (*JobInspection, error) {
selectorParts := []string{"runId=" + strings.TrimSpace(runID)}
if trimmedNodeID := strings.TrimSpace(nodeID); trimmedNodeID != "" {
selectorParts = append(selectorParts, "nodeId="+trimmedNodeID)
}
jobs, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).List(ctx, metav1.ListOptions{
LabelSelector: strings.Join(selectorParts, ","),
})
if err != nil {
return nil, err
}
if len(jobs.Items) == 0 {
return &JobInspection{State: JobStateMissing}, nil
}
sort.Slice(jobs.Items, func(i int, j int) bool {
return jobs.Items[i].CreationTimestamp.Time.Before(jobs.Items[j].CreationTimestamp.Time)
})
job := jobs.Items[len(jobs.Items)-1]
reference := &JobReference{JobName: job.Name, Namespace: launcher.namespace}
createdAt := orphanRuntimeCreatedAt(job.CreationTimestamp)
image := ""
for _, container := range job.Spec.Template.Spec.Containers {
if image == "" || container.Name == "operator-runner" {
image = container.Image
}
if container.Name == "operator-runner" {
break
}
}
switch {
case job.Status.Succeeded > 0:
output, err := launcher.readJobOutputNow(ctx, job.Name)
if err != nil {
return nil, err
}
return &JobInspection{Reference: reference, Image: image, State: JobStateSucceeded, Output: output, CreatedAt: createdAt}, nil
case job.Status.Failed > 0:
output, outputErr := launcher.readJobOutputNow(ctx, job.Name)
diagnostics := launcher.readJobDiagnostics(ctx, &job, output, outputErr)
if strings.TrimSpace(diagnostics) != "" {
output = diagnostics
}
return &JobInspection{Reference: reference, Image: image, State: JobStateFailed, Output: output, CreatedAt: createdAt}, nil
}
reason, message, err := launcher.inspectBlockingState(ctx, job.Name)
if err != nil {
return nil, err
}
if reason != "" {
return &JobInspection{Reference: reference, Image: image, State: JobStatePending, CreatedAt: createdAt, BlockingReason: reason, BlockingMessage: message}, nil
}
if job.Status.Active > 0 {
return &JobInspection{Reference: reference, Image: image, State: JobStateActive, CreatedAt: createdAt}, nil
}
return &JobInspection{Reference: reference, Image: image, State: JobStatePending, CreatedAt: createdAt}, nil
}