🧠 add shared memory for training jobs

This commit is contained in:
eust-w 2026-07-13 13:57:46 +08:00
parent b59a692be8
commit c6f5a86b17
5 changed files with 97 additions and 0 deletions

View File

@ -70,6 +70,11 @@ type LaunchInput struct {
// work. Custom-container operators (own image) bypass main.py and would report
// no resourceSeries at all; for those the launcher samples GPU usage itself.
OperatorRunnerImage string
// SharedMemorySize controls the size of the in-memory /dev/shm volume. It is
// populated by Launcher from DROBOTICFLOW_SHARED_MEMORY_SIZE (default 8Gi)
// for production jobs; keeping it on the launch input makes the pod contract
// explicit and easy to test.
SharedMemorySize string
}
type PVCMount struct {
@ -167,6 +172,7 @@ type Launcher struct {
gpuTolerations []corev1.Toleration
gpuRuntimeClass string
jobTTLSeconds int32
sharedMemorySize string
clientset kubernetes.Interface
restConfig *rest.Config
}
@ -180,6 +186,8 @@ var launcherSystemEnvKeys = map[string]struct{}{
"NODE_LABEL": {},
}
const defaultSharedMemorySize = "8Gi"
func sanitizeName(value string) string {
normalized := strings.ToLower(value)
normalized = invalidNameCharacters.ReplaceAllString(normalized, "-")
@ -545,6 +553,14 @@ func CreateJobManifest(namespace string, imagePullSecretName string, workspacePV
if err != nil {
return nil, err
}
sharedMemorySize := strings.TrimSpace(input.SharedMemorySize)
if sharedMemorySize != "" {
quantity, quantityErr := resource.ParseQuantity(sharedMemorySize)
if quantityErr != nil || quantity.Value() <= 0 {
return nil, fmt.Errorf("shared memory size must be a positive Kubernetes resource quantity")
}
sharedMemorySize = quantity.String()
}
codeSubPath, err := cleanVolumeSubPath(input.CodeSubPath)
if err != nil {
return nil, err
@ -650,6 +666,20 @@ func CreateJobManifest(namespace string, imagePullSecretName string, workspacePV
},
},
}
if sharedMemorySize != "" {
quantity := resource.MustParse(sharedMemorySize)
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "droboticflow-shm",
VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{
Medium: corev1.StorageMediumMemory,
SizeLimit: &quantity,
}},
})
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "droboticflow-shm",
MountPath: "/dev/shm",
})
}
if input.JobTTLSeconds > 0 {
job.Spec.TTLSecondsAfterFinished = func() *int32 {
value := input.JobTTLSeconds
@ -771,6 +801,17 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
if jobTTLSeconds < 0 {
jobTTLSeconds = 0
}
sharedMemorySize := strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHARED_MEMORY_SIZE"))
if sharedMemorySize == "" {
sharedMemorySize = strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHM_SIZE"))
}
if sharedMemorySize == "" {
sharedMemorySize = defaultSharedMemorySize
}
sharedMemoryQuantity, err := resource.ParseQuantity(sharedMemorySize)
if err != nil || sharedMemoryQuantity.Value() <= 0 {
return nil, fmt.Errorf("DROBOTICFLOW_SHARED_MEMORY_SIZE must be a positive Kubernetes resource quantity")
}
launcher := &Launcher{
namespace: namespace,
imagePullSecretName: imagePullSecretName,
@ -780,6 +821,7 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
gpuTolerations: parsedGPUTolerations,
gpuRuntimeClass: strings.TrimSpace(gpuRuntimeClass),
jobTTLSeconds: int32(jobTTLSeconds),
sharedMemorySize: sharedMemoryQuantity.String(),
}
config, err := loadRestConfig()
if err != nil {
@ -1323,6 +1365,9 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc
if input.JobTTLSeconds == 0 {
input.JobTTLSeconds = launcher.jobTTLSeconds
}
if strings.TrimSpace(input.SharedMemorySize) == "" {
input.SharedMemorySize = launcher.sharedMemorySize
}
job, err := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, launcher.workspacePVCName, launcher.workspaceMountPath, input)
if err != nil {
return nil, err

View File

@ -303,6 +303,48 @@ func TestCreateJobManifestSupportsCommandArgsAndRegistryAuth(t *testing.T) {
}
}
func TestCreateJobManifestAddsSharedMemoryVolume(t *testing.T) {
manifest, err := CreateJobManifest("droboticflow", "", "droboticflow-workspace", "/workspace", LaunchInput{
Image: "runner:latest",
RunID: "run-shm",
NodeID: "train-1",
SharedMemorySize: "8Gi",
})
if err != nil {
t.Fatalf("CreateJobManifest returned error: %v", err)
}
var foundVolume, foundMount bool
for _, volume := range manifest.Spec.Template.Spec.Volumes {
if volume.Name == "droboticflow-shm" {
foundVolume = true
if volume.EmptyDir == nil || volume.EmptyDir.Medium != corev1.StorageMediumMemory || volume.EmptyDir.SizeLimit == nil || volume.EmptyDir.SizeLimit.String() != "8Gi" {
t.Fatalf("unexpected shared memory volume: %#v", volume)
}
}
}
for _, mount := range manifest.Spec.Template.Spec.Containers[0].VolumeMounts {
if mount.Name == "droboticflow-shm" && mount.MountPath == "/dev/shm" && !mount.ReadOnly {
foundMount = true
}
}
if !foundVolume || !foundMount {
t.Fatalf("expected writable /dev/shm memory volume, volumes=%#v mounts=%#v", manifest.Spec.Template.Spec.Volumes, manifest.Spec.Template.Spec.Containers[0].VolumeMounts)
}
}
func TestCreateJobManifestRejectsInvalidSharedMemorySize(t *testing.T) {
_, err := CreateJobManifest("droboticflow", "", "workspace", "/workspace", LaunchInput{
Image: "runner:latest",
RunID: "run-shm-invalid",
NodeID: "train-1",
SharedMemorySize: "not-a-quantity",
})
if err == nil || !strings.Contains(err.Error(), "shared memory size") {
t.Fatalf("expected invalid shared memory error, got %v", err)
}
}
func TestCreateJobManifestOmitsTTLWhenDisabled(t *testing.T) {
manifest, err := CreateJobManifest("droboticflow", "", "droboticflow-workspace", "/workspace", LaunchInput{
Image: "runner:latest",

View File

@ -120,6 +120,7 @@ data:
MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__"
MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__"
WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__"
DROBOTICFLOW_SHARED_MEMORY_SIZE: "__DROBOTICFLOW_SHARED_MEMORY_SIZE__"
DROBOTICFLOW_BUILD_COMMIT: "__DROBOTICFLOW_BUILD_COMMIT__"
DROBOTICFLOW_BUILD_TIME: "__DROBOTICFLOW_BUILD_TIME__"
OPERATOR_RUNNER_BUILD_COMMIT: "__OPERATOR_RUNNER_BUILD_COMMIT__"
@ -463,6 +464,11 @@ spec:
configMapKeyRef:
name: droboticflow-config
key: WORKER_POLL_INTERVAL_MS
- name: DROBOTICFLOW_SHARED_MEMORY_SIZE
valueFrom:
configMapKeyRef:
name: droboticflow-config
key: DROBOTICFLOW_SHARED_MEMORY_SIZE
- name: WORKER_ORPHAN_CLEANUP_ENABLED
valueFrom:
configMapKeyRef:

View File

@ -14,6 +14,7 @@ export MONGO_DB="${MONGO_DB:-poc}"
export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
export DROBOTICFLOW_SHARED_MEMORY_SIZE="${DROBOTICFLOW_SHARED_MEMORY_SIZE:-8Gi}"
export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
export WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-true}"
export WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"

View File

@ -22,6 +22,7 @@ MONGO_DB="${MONGO_DB:-droboticflow}"
MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
DROBOTICFLOW_SHARED_MEMORY_SIZE="${DROBOTICFLOW_SHARED_MEMORY_SIZE:-8Gi}"
WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
BOS_ENDPOINT="${BOS_ENDPOINT:-}"
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
@ -107,6 +108,7 @@ export \
MONGO_CONNECT_RETRIES \
MONGO_CONNECT_RETRY_DELAY_MS \
WORKER_POLL_INTERVAL_MS \
DROBOTICFLOW_SHARED_MEMORY_SIZE \
WORKER_API_BASE_URL \
BOS_ENDPOINT \
K8S_IMAGE_PULL_SECRET_NAME \
@ -158,6 +160,7 @@ replacements = {
"__MONGO_CONNECT_RETRIES__": os.environ["MONGO_CONNECT_RETRIES"],
"__MONGO_CONNECT_RETRY_DELAY_MS__": os.environ["MONGO_CONNECT_RETRY_DELAY_MS"],
"__WORKER_POLL_INTERVAL_MS__": os.environ["WORKER_POLL_INTERVAL_MS"],
"__DROBOTICFLOW_SHARED_MEMORY_SIZE__": os.environ["DROBOTICFLOW_SHARED_MEMORY_SIZE"],
"__WORKER_API_BASE_URL__": os.environ["WORKER_API_BASE_URL"],
"__BOS_ENDPOINT__": os.environ["BOS_ENDPOINT"],
"__K8S_IMAGE_PULL_SECRET_NAME__": os.environ["K8S_IMAGE_PULL_SECRET_NAME"],