diff --git a/internal/k8s/launcher.go b/internal/k8s/launcher.go index b4cc487b..7d05e3f5 100644 --- a/internal/k8s/launcher.go +++ b/internal/k8s/launcher.go @@ -70,6 +70,11 @@ type LaunchInput struct { // work. Custom-container operators (own image) bypass main.py and would report // no resourceSeries at all; for those the launcher samples GPU usage itself. OperatorRunnerImage string + // SharedMemorySize controls the size of the in-memory /dev/shm volume. It is + // populated by Launcher from DROBOTICFLOW_SHARED_MEMORY_SIZE (default 8Gi) + // for production jobs; keeping it on the launch input makes the pod contract + // explicit and easy to test. + SharedMemorySize string } type PVCMount struct { @@ -167,6 +172,7 @@ type Launcher struct { gpuTolerations []corev1.Toleration gpuRuntimeClass string jobTTLSeconds int32 + sharedMemorySize string clientset kubernetes.Interface restConfig *rest.Config } @@ -180,6 +186,8 @@ var launcherSystemEnvKeys = map[string]struct{}{ "NODE_LABEL": {}, } +const defaultSharedMemorySize = "8Gi" + func sanitizeName(value string) string { normalized := strings.ToLower(value) normalized = invalidNameCharacters.ReplaceAllString(normalized, "-") @@ -545,6 +553,14 @@ func CreateJobManifest(namespace string, imagePullSecretName string, workspacePV if err != nil { return nil, err } + sharedMemorySize := strings.TrimSpace(input.SharedMemorySize) + if sharedMemorySize != "" { + quantity, quantityErr := resource.ParseQuantity(sharedMemorySize) + if quantityErr != nil || quantity.Value() <= 0 { + return nil, fmt.Errorf("shared memory size must be a positive Kubernetes resource quantity") + } + sharedMemorySize = quantity.String() + } codeSubPath, err := cleanVolumeSubPath(input.CodeSubPath) if err != nil { return nil, err @@ -650,6 +666,20 @@ func CreateJobManifest(namespace string, imagePullSecretName string, workspacePV }, }, } + if sharedMemorySize != "" { + quantity := resource.MustParse(sharedMemorySize) + job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "droboticflow-shm", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + SizeLimit: &quantity, + }}, + }) + job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "droboticflow-shm", + MountPath: "/dev/shm", + }) + } if input.JobTTLSeconds > 0 { job.Spec.TTLSecondsAfterFinished = func() *int32 { value := input.JobTTLSeconds @@ -771,6 +801,17 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName if jobTTLSeconds < 0 { jobTTLSeconds = 0 } + sharedMemorySize := strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHARED_MEMORY_SIZE")) + if sharedMemorySize == "" { + sharedMemorySize = strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHM_SIZE")) + } + if sharedMemorySize == "" { + sharedMemorySize = defaultSharedMemorySize + } + sharedMemoryQuantity, err := resource.ParseQuantity(sharedMemorySize) + if err != nil || sharedMemoryQuantity.Value() <= 0 { + return nil, fmt.Errorf("DROBOTICFLOW_SHARED_MEMORY_SIZE must be a positive Kubernetes resource quantity") + } launcher := &Launcher{ namespace: namespace, imagePullSecretName: imagePullSecretName, @@ -780,6 +821,7 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName gpuTolerations: parsedGPUTolerations, gpuRuntimeClass: strings.TrimSpace(gpuRuntimeClass), jobTTLSeconds: int32(jobTTLSeconds), + sharedMemorySize: sharedMemoryQuantity.String(), } config, err := loadRestConfig() if err != nil { @@ -1323,6 +1365,9 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc if input.JobTTLSeconds == 0 { input.JobTTLSeconds = launcher.jobTTLSeconds } + if strings.TrimSpace(input.SharedMemorySize) == "" { + input.SharedMemorySize = launcher.sharedMemorySize + } job, err := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, launcher.workspacePVCName, launcher.workspaceMountPath, input) if err != nil { return nil, err diff --git a/internal/k8s/launcher_test.go b/internal/k8s/launcher_test.go index 50551cb1..812a1854 100644 --- a/internal/k8s/launcher_test.go +++ b/internal/k8s/launcher_test.go @@ -303,6 +303,48 @@ func TestCreateJobManifestSupportsCommandArgsAndRegistryAuth(t *testing.T) { } } +func TestCreateJobManifestAddsSharedMemoryVolume(t *testing.T) { + manifest, err := CreateJobManifest("droboticflow", "", "droboticflow-workspace", "/workspace", LaunchInput{ + Image: "runner:latest", + RunID: "run-shm", + NodeID: "train-1", + SharedMemorySize: "8Gi", + }) + if err != nil { + t.Fatalf("CreateJobManifest returned error: %v", err) + } + + var foundVolume, foundMount bool + for _, volume := range manifest.Spec.Template.Spec.Volumes { + if volume.Name == "droboticflow-shm" { + foundVolume = true + if volume.EmptyDir == nil || volume.EmptyDir.Medium != corev1.StorageMediumMemory || volume.EmptyDir.SizeLimit == nil || volume.EmptyDir.SizeLimit.String() != "8Gi" { + t.Fatalf("unexpected shared memory volume: %#v", volume) + } + } + } + for _, mount := range manifest.Spec.Template.Spec.Containers[0].VolumeMounts { + if mount.Name == "droboticflow-shm" && mount.MountPath == "/dev/shm" && !mount.ReadOnly { + foundMount = true + } + } + if !foundVolume || !foundMount { + t.Fatalf("expected writable /dev/shm memory volume, volumes=%#v mounts=%#v", manifest.Spec.Template.Spec.Volumes, manifest.Spec.Template.Spec.Containers[0].VolumeMounts) + } +} + +func TestCreateJobManifestRejectsInvalidSharedMemorySize(t *testing.T) { + _, err := CreateJobManifest("droboticflow", "", "workspace", "/workspace", LaunchInput{ + Image: "runner:latest", + RunID: "run-shm-invalid", + NodeID: "train-1", + SharedMemorySize: "not-a-quantity", + }) + if err == nil || !strings.Contains(err.Error(), "shared memory size") { + t.Fatalf("expected invalid shared memory error, got %v", err) + } +} + func TestCreateJobManifestOmitsTTLWhenDisabled(t *testing.T) { manifest, err := CreateJobManifest("droboticflow", "", "droboticflow-workspace", "/workspace", LaunchInput{ Image: "runner:latest", diff --git a/k8s/droboticflow.template.yaml b/k8s/droboticflow.template.yaml index af215d69..a650708c 100644 --- a/k8s/droboticflow.template.yaml +++ b/k8s/droboticflow.template.yaml @@ -120,6 +120,7 @@ data: MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__" MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__" WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__" + DROBOTICFLOW_SHARED_MEMORY_SIZE: "__DROBOTICFLOW_SHARED_MEMORY_SIZE__" DROBOTICFLOW_BUILD_COMMIT: "__DROBOTICFLOW_BUILD_COMMIT__" DROBOTICFLOW_BUILD_TIME: "__DROBOTICFLOW_BUILD_TIME__" OPERATOR_RUNNER_BUILD_COMMIT: "__OPERATOR_RUNNER_BUILD_COMMIT__" @@ -463,6 +464,11 @@ spec: configMapKeyRef: name: droboticflow-config key: WORKER_POLL_INTERVAL_MS + - name: DROBOTICFLOW_SHARED_MEMORY_SIZE + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: DROBOTICFLOW_SHARED_MEMORY_SIZE - name: WORKER_ORPHAN_CLEANUP_ENABLED valueFrom: configMapKeyRef: diff --git a/scripts/deploy-envs/poc.env b/scripts/deploy-envs/poc.env index 56ed7d86..8393804d 100644 --- a/scripts/deploy-envs/poc.env +++ b/scripts/deploy-envs/poc.env @@ -14,6 +14,7 @@ export MONGO_DB="${MONGO_DB:-poc}" export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}" export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}" export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}" +export DROBOTICFLOW_SHARED_MEMORY_SIZE="${DROBOTICFLOW_SHARED_MEMORY_SIZE:-8Gi}" export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}" export WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-true}" export WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}" diff --git a/scripts/render-manifests.sh b/scripts/render-manifests.sh index 81ee5869..aa4cfcc9 100755 --- a/scripts/render-manifests.sh +++ b/scripts/render-manifests.sh @@ -22,6 +22,7 @@ MONGO_DB="${MONGO_DB:-droboticflow}" MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}" MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}" WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}" +DROBOTICFLOW_SHARED_MEMORY_SIZE="${DROBOTICFLOW_SHARED_MEMORY_SIZE:-8Gi}" WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}" BOS_ENDPOINT="${BOS_ENDPOINT:-}" K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}" @@ -107,6 +108,7 @@ export \ MONGO_CONNECT_RETRIES \ MONGO_CONNECT_RETRY_DELAY_MS \ WORKER_POLL_INTERVAL_MS \ + DROBOTICFLOW_SHARED_MEMORY_SIZE \ WORKER_API_BASE_URL \ BOS_ENDPOINT \ K8S_IMAGE_PULL_SECRET_NAME \ @@ -158,6 +160,7 @@ replacements = { "__MONGO_CONNECT_RETRIES__": os.environ["MONGO_CONNECT_RETRIES"], "__MONGO_CONNECT_RETRY_DELAY_MS__": os.environ["MONGO_CONNECT_RETRY_DELAY_MS"], "__WORKER_POLL_INTERVAL_MS__": os.environ["WORKER_POLL_INTERVAL_MS"], + "__DROBOTICFLOW_SHARED_MEMORY_SIZE__": os.environ["DROBOTICFLOW_SHARED_MEMORY_SIZE"], "__WORKER_API_BASE_URL__": os.environ["WORKER_API_BASE_URL"], "__BOS_ENDPOINT__": os.environ["BOS_ENDPOINT"], "__K8S_IMAGE_PULL_SECRET_NAME__": os.environ["K8S_IMAGE_PULL_SECRET_NAME"],