forked from longtao.wu/droboticflow
🧠 add shared memory for training jobs
This commit is contained in:
parent
b59a692be8
commit
c6f5a86b17
@ -70,6 +70,11 @@ type LaunchInput struct {
|
||||
// work. Custom-container operators (own image) bypass main.py and would report
|
||||
// no resourceSeries at all; for those the launcher samples GPU usage itself.
|
||||
OperatorRunnerImage string
|
||||
// SharedMemorySize controls the size of the in-memory /dev/shm volume. It is
|
||||
// populated by Launcher from DROBOTICFLOW_SHARED_MEMORY_SIZE (default 8Gi)
|
||||
// for production jobs; keeping it on the launch input makes the pod contract
|
||||
// explicit and easy to test.
|
||||
SharedMemorySize string
|
||||
}
|
||||
|
||||
type PVCMount struct {
|
||||
@ -167,6 +172,7 @@ type Launcher struct {
|
||||
gpuTolerations []corev1.Toleration
|
||||
gpuRuntimeClass string
|
||||
jobTTLSeconds int32
|
||||
sharedMemorySize string
|
||||
clientset kubernetes.Interface
|
||||
restConfig *rest.Config
|
||||
}
|
||||
@ -180,6 +186,8 @@ var launcherSystemEnvKeys = map[string]struct{}{
|
||||
"NODE_LABEL": {},
|
||||
}
|
||||
|
||||
const defaultSharedMemorySize = "8Gi"
|
||||
|
||||
func sanitizeName(value string) string {
|
||||
normalized := strings.ToLower(value)
|
||||
normalized = invalidNameCharacters.ReplaceAllString(normalized, "-")
|
||||
@ -545,6 +553,14 @@ func CreateJobManifest(namespace string, imagePullSecretName string, workspacePV
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sharedMemorySize := strings.TrimSpace(input.SharedMemorySize)
|
||||
if sharedMemorySize != "" {
|
||||
quantity, quantityErr := resource.ParseQuantity(sharedMemorySize)
|
||||
if quantityErr != nil || quantity.Value() <= 0 {
|
||||
return nil, fmt.Errorf("shared memory size must be a positive Kubernetes resource quantity")
|
||||
}
|
||||
sharedMemorySize = quantity.String()
|
||||
}
|
||||
codeSubPath, err := cleanVolumeSubPath(input.CodeSubPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -650,6 +666,20 @@ func CreateJobManifest(namespace string, imagePullSecretName string, workspacePV
|
||||
},
|
||||
},
|
||||
}
|
||||
if sharedMemorySize != "" {
|
||||
quantity := resource.MustParse(sharedMemorySize)
|
||||
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
|
||||
Name: "droboticflow-shm",
|
||||
VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{
|
||||
Medium: corev1.StorageMediumMemory,
|
||||
SizeLimit: &quantity,
|
||||
}},
|
||||
})
|
||||
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
||||
Name: "droboticflow-shm",
|
||||
MountPath: "/dev/shm",
|
||||
})
|
||||
}
|
||||
if input.JobTTLSeconds > 0 {
|
||||
job.Spec.TTLSecondsAfterFinished = func() *int32 {
|
||||
value := input.JobTTLSeconds
|
||||
@ -771,6 +801,17 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
|
||||
if jobTTLSeconds < 0 {
|
||||
jobTTLSeconds = 0
|
||||
}
|
||||
sharedMemorySize := strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHARED_MEMORY_SIZE"))
|
||||
if sharedMemorySize == "" {
|
||||
sharedMemorySize = strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHM_SIZE"))
|
||||
}
|
||||
if sharedMemorySize == "" {
|
||||
sharedMemorySize = defaultSharedMemorySize
|
||||
}
|
||||
sharedMemoryQuantity, err := resource.ParseQuantity(sharedMemorySize)
|
||||
if err != nil || sharedMemoryQuantity.Value() <= 0 {
|
||||
return nil, fmt.Errorf("DROBOTICFLOW_SHARED_MEMORY_SIZE must be a positive Kubernetes resource quantity")
|
||||
}
|
||||
launcher := &Launcher{
|
||||
namespace: namespace,
|
||||
imagePullSecretName: imagePullSecretName,
|
||||
@ -780,6 +821,7 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
|
||||
gpuTolerations: parsedGPUTolerations,
|
||||
gpuRuntimeClass: strings.TrimSpace(gpuRuntimeClass),
|
||||
jobTTLSeconds: int32(jobTTLSeconds),
|
||||
sharedMemorySize: sharedMemoryQuantity.String(),
|
||||
}
|
||||
config, err := loadRestConfig()
|
||||
if err != nil {
|
||||
@ -1323,6 +1365,9 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc
|
||||
if input.JobTTLSeconds == 0 {
|
||||
input.JobTTLSeconds = launcher.jobTTLSeconds
|
||||
}
|
||||
if strings.TrimSpace(input.SharedMemorySize) == "" {
|
||||
input.SharedMemorySize = launcher.sharedMemorySize
|
||||
}
|
||||
job, err := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, launcher.workspacePVCName, launcher.workspaceMountPath, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@ -303,6 +303,48 @@ func TestCreateJobManifestSupportsCommandArgsAndRegistryAuth(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateJobManifestAddsSharedMemoryVolume(t *testing.T) {
|
||||
manifest, err := CreateJobManifest("droboticflow", "", "droboticflow-workspace", "/workspace", LaunchInput{
|
||||
Image: "runner:latest",
|
||||
RunID: "run-shm",
|
||||
NodeID: "train-1",
|
||||
SharedMemorySize: "8Gi",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateJobManifest returned error: %v", err)
|
||||
}
|
||||
|
||||
var foundVolume, foundMount bool
|
||||
for _, volume := range manifest.Spec.Template.Spec.Volumes {
|
||||
if volume.Name == "droboticflow-shm" {
|
||||
foundVolume = true
|
||||
if volume.EmptyDir == nil || volume.EmptyDir.Medium != corev1.StorageMediumMemory || volume.EmptyDir.SizeLimit == nil || volume.EmptyDir.SizeLimit.String() != "8Gi" {
|
||||
t.Fatalf("unexpected shared memory volume: %#v", volume)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, mount := range manifest.Spec.Template.Spec.Containers[0].VolumeMounts {
|
||||
if mount.Name == "droboticflow-shm" && mount.MountPath == "/dev/shm" && !mount.ReadOnly {
|
||||
foundMount = true
|
||||
}
|
||||
}
|
||||
if !foundVolume || !foundMount {
|
||||
t.Fatalf("expected writable /dev/shm memory volume, volumes=%#v mounts=%#v", manifest.Spec.Template.Spec.Volumes, manifest.Spec.Template.Spec.Containers[0].VolumeMounts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateJobManifestRejectsInvalidSharedMemorySize(t *testing.T) {
|
||||
_, err := CreateJobManifest("droboticflow", "", "workspace", "/workspace", LaunchInput{
|
||||
Image: "runner:latest",
|
||||
RunID: "run-shm-invalid",
|
||||
NodeID: "train-1",
|
||||
SharedMemorySize: "not-a-quantity",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "shared memory size") {
|
||||
t.Fatalf("expected invalid shared memory error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateJobManifestOmitsTTLWhenDisabled(t *testing.T) {
|
||||
manifest, err := CreateJobManifest("droboticflow", "", "droboticflow-workspace", "/workspace", LaunchInput{
|
||||
Image: "runner:latest",
|
||||
|
||||
@ -120,6 +120,7 @@ data:
|
||||
MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__"
|
||||
MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__"
|
||||
WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__"
|
||||
DROBOTICFLOW_SHARED_MEMORY_SIZE: "__DROBOTICFLOW_SHARED_MEMORY_SIZE__"
|
||||
DROBOTICFLOW_BUILD_COMMIT: "__DROBOTICFLOW_BUILD_COMMIT__"
|
||||
DROBOTICFLOW_BUILD_TIME: "__DROBOTICFLOW_BUILD_TIME__"
|
||||
OPERATOR_RUNNER_BUILD_COMMIT: "__OPERATOR_RUNNER_BUILD_COMMIT__"
|
||||
@ -463,6 +464,11 @@ spec:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: WORKER_POLL_INTERVAL_MS
|
||||
- name: DROBOTICFLOW_SHARED_MEMORY_SIZE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: DROBOTICFLOW_SHARED_MEMORY_SIZE
|
||||
- name: WORKER_ORPHAN_CLEANUP_ENABLED
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
|
||||
@ -14,6 +14,7 @@ export MONGO_DB="${MONGO_DB:-poc}"
|
||||
export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
|
||||
export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
|
||||
export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
|
||||
export DROBOTICFLOW_SHARED_MEMORY_SIZE="${DROBOTICFLOW_SHARED_MEMORY_SIZE:-8Gi}"
|
||||
export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
|
||||
export WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-true}"
|
||||
export WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
|
||||
|
||||
@ -22,6 +22,7 @@ MONGO_DB="${MONGO_DB:-droboticflow}"
|
||||
MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
|
||||
MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
|
||||
WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
|
||||
DROBOTICFLOW_SHARED_MEMORY_SIZE="${DROBOTICFLOW_SHARED_MEMORY_SIZE:-8Gi}"
|
||||
WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
|
||||
BOS_ENDPOINT="${BOS_ENDPOINT:-}"
|
||||
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
||||
@ -107,6 +108,7 @@ export \
|
||||
MONGO_CONNECT_RETRIES \
|
||||
MONGO_CONNECT_RETRY_DELAY_MS \
|
||||
WORKER_POLL_INTERVAL_MS \
|
||||
DROBOTICFLOW_SHARED_MEMORY_SIZE \
|
||||
WORKER_API_BASE_URL \
|
||||
BOS_ENDPOINT \
|
||||
K8S_IMAGE_PULL_SECRET_NAME \
|
||||
@ -158,6 +160,7 @@ replacements = {
|
||||
"__MONGO_CONNECT_RETRIES__": os.environ["MONGO_CONNECT_RETRIES"],
|
||||
"__MONGO_CONNECT_RETRY_DELAY_MS__": os.environ["MONGO_CONNECT_RETRY_DELAY_MS"],
|
||||
"__WORKER_POLL_INTERVAL_MS__": os.environ["WORKER_POLL_INTERVAL_MS"],
|
||||
"__DROBOTICFLOW_SHARED_MEMORY_SIZE__": os.environ["DROBOTICFLOW_SHARED_MEMORY_SIZE"],
|
||||
"__WORKER_API_BASE_URL__": os.environ["WORKER_API_BASE_URL"],
|
||||
"__BOS_ENDPOINT__": os.environ["BOS_ENDPOINT"],
|
||||
"__K8S_IMAGE_PULL_SECRET_NAME__": os.environ["K8S_IMAGE_PULL_SECRET_NAME"],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user