1827 lines
59 KiB
Go
1827 lines
59 KiB
Go
package k8s
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha1"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/rest"
|
|
"k8s.io/client-go/tools/clientcmd"
|
|
"k8s.io/client-go/tools/remotecommand"
|
|
)
|
|
|
|
const (
|
|
defaultJobWaitTimeout = 2 * time.Hour
|
|
longRunningJobWaitTimeout = 7 * 24 * time.Hour
|
|
defaultJobBackoffLimit int32 = 0
|
|
)
|
|
|
|
type LaunchInput struct {
|
|
Image string
|
|
Command []string
|
|
Args []string
|
|
RunID string
|
|
WorkflowID string
|
|
NodeID string
|
|
NodeLabel string
|
|
Attempt int
|
|
TimeoutSeconds int
|
|
Env map[string]string
|
|
SensitiveEnv map[string]string
|
|
CPURequest string
|
|
CPULimit string
|
|
MemoryRequest string
|
|
MemoryLimit string
|
|
GPULimit string
|
|
GPUNodeSelector map[string]string
|
|
GPUTolerations []corev1.Toleration
|
|
GPURuntimeClass string
|
|
JobTTLSeconds int32
|
|
RegistryAuth bool
|
|
LongRunning bool
|
|
CodeSubPath string
|
|
CodeMountPath string
|
|
CodeReadOnly bool
|
|
PVCMounts []PVCMount
|
|
// OperatorRunnerImage is the image the platform resolves builtin operators to.
|
|
// When the resolved Image equals it, the pod runs operator-runner's main.py,
|
|
// which already self-reports resourceSeries via its ResourceSampler — so the
|
|
// launcher skips platform-side GPU sampling for those pods to avoid duplicate
|
|
// work. Custom-container operators (own image) bypass main.py and would report
|
|
// no resourceSeries at all; for those the launcher samples GPU usage itself.
|
|
OperatorRunnerImage string
|
|
// SharedMemorySize controls the size of the in-memory /dev/shm volume. It is
|
|
// populated by Launcher from DROBOTICFLOW_SHARED_MEMORY_SIZE (default 8Gi)
|
|
// for production jobs; keeping it on the launch input makes the pod contract
|
|
// explicit and easy to test.
|
|
SharedMemorySize string
|
|
InferenceSidecar *InferenceSidecar
|
|
}
|
|
|
|
type InferenceSidecar struct {
|
|
Name string
|
|
Image string
|
|
Port int32
|
|
HealthPath string
|
|
Env map[string]string
|
|
CPURequest string
|
|
CPULimit string
|
|
MemoryRequest string
|
|
MemoryLimit string
|
|
GPULimit string
|
|
}
|
|
|
|
type PVCMount struct {
|
|
Name string
|
|
PVCName string
|
|
SubPath string
|
|
MountPath string
|
|
// ReadOnly is opt-in. An omitted readOnly field in a workflow contract
|
|
// becomes false and keeps the PVC mount writable by default.
|
|
ReadOnly bool
|
|
}
|
|
|
|
type JobReference struct {
|
|
JobName string
|
|
Namespace string
|
|
}
|
|
|
|
type LaunchResult struct {
|
|
Reference *JobReference
|
|
Output string
|
|
// ResourceSeries holds GPU samples collected by the platform while the pod was
|
|
// running. Populated only for custom-container operators that do not
|
|
// self-report resourceSeries (see LaunchInput.OperatorRunnerImage). Empty for
|
|
// builtin operator-runner pods, which emit their own series in result.json.
|
|
ResourceSeries []GPUSample
|
|
}
|
|
|
|
// GPUSample mirrors the GPU-relevant fields of domain.ResourceSample. Kept in the
|
|
// k8s package so the launcher does not depend on internal/domain; the worker maps
|
|
// it to domain.ResourceSample when the operator did not self-report a series.
|
|
type GPUSample struct {
|
|
Timestamp string
|
|
GPU int
|
|
GPUMemoryUsedMiB int
|
|
GPUMemoryTotalMiB int
|
|
GPUName string
|
|
GPUCount int
|
|
}
|
|
|
|
type JobState string
|
|
|
|
const (
|
|
JobStateMissing JobState = "missing"
|
|
JobStatePending JobState = "pending"
|
|
JobStateActive JobState = "active"
|
|
JobStateSucceeded JobState = "succeeded"
|
|
JobStateFailed JobState = "failed"
|
|
)
|
|
|
|
type JobInspection struct {
|
|
Reference *JobReference
|
|
Image string
|
|
State JobState
|
|
Output string
|
|
CreatedAt string
|
|
BlockingReason string
|
|
BlockingMessage string
|
|
}
|
|
|
|
// JobNotFoundError marks a Job that disappeared after it was created. Keeping
|
|
// this distinct from ordinary execution failures lets the worker re-queue the
|
|
// node through its bounded missing-Job recovery path instead of immediately
|
|
// terminalizing a healthy long-running training Run.
|
|
type JobNotFoundError struct {
|
|
JobName string
|
|
}
|
|
|
|
func (err *JobNotFoundError) Error() string {
|
|
return fmt.Sprintf("kubernetes job %q not found", strings.TrimSpace(err.JobName))
|
|
}
|
|
|
|
func IsJobNotFound(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
var missing *JobNotFoundError
|
|
return errors.As(err, &missing) || apierrors.IsNotFound(err)
|
|
}
|
|
|
|
type OrphanRuntime struct {
|
|
Kind string `json:"kind"`
|
|
Name string `json:"name"`
|
|
Namespace string `json:"namespace"`
|
|
RunID string `json:"runId"`
|
|
CreatedAt string `json:"createdAt"`
|
|
Cleaned bool `json:"cleaned"`
|
|
CleanupError string `json:"cleanupError,omitempty"`
|
|
}
|
|
|
|
type GPUCapacityNode struct {
|
|
Name string `json:"name"`
|
|
Allocatable int64 `json:"allocatable"`
|
|
SelectorMatched bool `json:"selectorMatched"`
|
|
Unschedulable bool `json:"unschedulable"`
|
|
}
|
|
|
|
type GPUCapacitySnapshot struct {
|
|
ResourceName string `json:"resourceName"`
|
|
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
|
|
SelectorSummary string `json:"selectorSummary"`
|
|
Nodes []GPUCapacityNode `json:"nodes"`
|
|
TotalAllocatable int64 `json:"totalAllocatable"`
|
|
SchedulableAllocatable int64 `json:"schedulableAllocatable"`
|
|
MaxAllocatable int64 `json:"maxAllocatable"`
|
|
SchedulableNodeCount int `json:"schedulableNodeCount"`
|
|
Available bool `json:"available"`
|
|
CheckedAt string `json:"checkedAt"`
|
|
}
|
|
|
|
type Launcher struct {
|
|
namespace string
|
|
imagePullSecretName string
|
|
workspacePVCName string
|
|
workspaceMountPath string
|
|
gpuNodeSelector map[string]string
|
|
gpuTolerations []corev1.Toleration
|
|
gpuRuntimeClass string
|
|
jobTTLSeconds int32
|
|
sharedMemorySize string
|
|
clientset kubernetes.Interface
|
|
restConfig *rest.Config
|
|
}
|
|
|
|
var invalidNameCharacters = regexp.MustCompile(`[^a-z0-9-]+`)
|
|
|
|
var launcherSystemEnvKeys = map[string]struct{}{
|
|
"RUN_ID": {},
|
|
"WORKFLOW_ID": {},
|
|
"NODE_ID": {},
|
|
"NODE_LABEL": {},
|
|
}
|
|
|
|
const defaultSharedMemorySize = "8Gi"
|
|
|
|
func sanitizeName(value string) string {
|
|
normalized := strings.ToLower(value)
|
|
normalized = invalidNameCharacters.ReplaceAllString(normalized, "-")
|
|
normalized = strings.Trim(normalized, "-")
|
|
normalized = regexp.MustCompile(`-+`).ReplaceAllString(normalized, "-")
|
|
return normalized
|
|
}
|
|
|
|
func launchAttempt(input LaunchInput) int {
|
|
if input.Attempt <= 0 {
|
|
return 1
|
|
}
|
|
return input.Attempt
|
|
}
|
|
|
|
func launchJobName(input LaunchInput) string {
|
|
attemptSuffix := fmt.Sprintf("-a%d", launchAttempt(input))
|
|
base := sanitizeName(fmt.Sprintf("%s-%s", input.RunID, input.NodeID))
|
|
if base == "" {
|
|
return "droboticflow-node-a1"
|
|
}
|
|
maxBaseLength := 63 - len(attemptSuffix)
|
|
if maxBaseLength < 1 {
|
|
maxBaseLength = 1
|
|
}
|
|
if len(base) > maxBaseLength {
|
|
sum := sha1.Sum([]byte(base))
|
|
hashSuffix := "-" + hex.EncodeToString(sum[:])[:8]
|
|
prefixLength := maxBaseLength - len(hashSuffix)
|
|
if prefixLength < 1 {
|
|
prefixLength = 1
|
|
}
|
|
base = strings.Trim(base[:prefixLength], "-") + hashSuffix
|
|
}
|
|
base = strings.Trim(base, "-")
|
|
if base == "" {
|
|
return "droboticflow-node-a1"
|
|
}
|
|
return strings.Trim(base+attemptSuffix, "-")
|
|
}
|
|
|
|
func launchSensitiveEnvSecretName(input LaunchInput) string {
|
|
base := launchJobName(input) + "-env"
|
|
if len(base) <= 63 {
|
|
return base
|
|
}
|
|
sum := sha1.Sum([]byte(base))
|
|
hashSuffix := "-" + hex.EncodeToString(sum[:])[:8]
|
|
prefixLength := 63 - len(hashSuffix)
|
|
return strings.Trim(base[:prefixLength], "-") + hashSuffix
|
|
}
|
|
|
|
func parseResourceQuantity(field string, value string) (*resource.Quantity, error) {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return nil, nil
|
|
}
|
|
quantity, err := resource.ParseQuantity(trimmed)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s must be a valid Kubernetes resource quantity: %w", field, err)
|
|
}
|
|
if quantity.Sign() <= 0 {
|
|
return nil, fmt.Errorf("%s must be greater than zero", field)
|
|
}
|
|
return &quantity, nil
|
|
}
|
|
|
|
func addResourceQuantity(resources corev1.ResourceList, name corev1.ResourceName, field string, value string) error {
|
|
quantity, err := parseResourceQuantity(field, value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if quantity == nil {
|
|
return nil
|
|
}
|
|
resources[name] = *quantity
|
|
return nil
|
|
}
|
|
|
|
func ValidateResourceQuantity(field string, value string) error {
|
|
_, err := parseResourceQuantity(field, value)
|
|
return err
|
|
}
|
|
|
|
func addGPUQuantity(resources corev1.ResourceList, value string) {
|
|
if !hasPositiveGPU(value) {
|
|
return
|
|
}
|
|
resources[corev1.ResourceName("nvidia.com/gpu")] = resource.MustParse(strings.TrimSpace(value))
|
|
}
|
|
|
|
func hasPositiveGPU(value string) bool {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return false
|
|
}
|
|
count, err := strconv.Atoi(trimmed)
|
|
return err == nil && count > 0
|
|
}
|
|
|
|
func ValidateGPURequest(value string) error {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return nil
|
|
}
|
|
count, err := strconv.Atoi(trimmed)
|
|
if err != nil || count <= 0 {
|
|
return fmt.Errorf("gpu limit must be a positive integer")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func resourceRequirements(input LaunchInput) (corev1.ResourceRequirements, error) {
|
|
requests := corev1.ResourceList{}
|
|
limits := corev1.ResourceList{}
|
|
if err := addResourceQuantity(requests, corev1.ResourceCPU, "cpu request", input.CPURequest); err != nil {
|
|
return corev1.ResourceRequirements{}, err
|
|
}
|
|
if err := addResourceQuantity(requests, corev1.ResourceMemory, "memory request", input.MemoryRequest); err != nil {
|
|
return corev1.ResourceRequirements{}, err
|
|
}
|
|
if err := addResourceQuantity(limits, corev1.ResourceCPU, "cpu limit", input.CPULimit); err != nil {
|
|
return corev1.ResourceRequirements{}, err
|
|
}
|
|
if err := addResourceQuantity(limits, corev1.ResourceMemory, "memory limit", input.MemoryLimit); err != nil {
|
|
return corev1.ResourceRequirements{}, err
|
|
}
|
|
addGPUQuantity(requests, input.GPULimit)
|
|
addGPUQuantity(limits, input.GPULimit)
|
|
requirements := corev1.ResourceRequirements{}
|
|
if len(requests) > 0 {
|
|
requirements.Requests = requests
|
|
}
|
|
if len(limits) > 0 {
|
|
requirements.Limits = limits
|
|
}
|
|
return requirements, nil
|
|
}
|
|
|
|
func boolPtr(value bool) *bool {
|
|
return &value
|
|
}
|
|
|
|
func int64Ptr(value int64) *int64 {
|
|
return &value
|
|
}
|
|
|
|
func cleanVolumeSubPath(value string) (string, error) {
|
|
trimmed := strings.Trim(strings.TrimSpace(value), "/")
|
|
if trimmed == "" {
|
|
return "", nil
|
|
}
|
|
cleaned := pathClean(trimmed)
|
|
if cleaned == "." || strings.HasPrefix(cleaned, "../") || cleaned == ".." || filepath.IsAbs(cleaned) {
|
|
return "", fmt.Errorf("code sub path must be a relative workspace path")
|
|
}
|
|
return cleaned, nil
|
|
}
|
|
|
|
func pathClean(value string) string {
|
|
return filepath.ToSlash(filepath.Clean(filepath.FromSlash(value)))
|
|
}
|
|
|
|
func cleanMountPath(value string) (string, error) {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return "", fmt.Errorf("code mount path is required")
|
|
}
|
|
if !strings.HasPrefix(trimmed, "/") {
|
|
return "", fmt.Errorf("code mount path must be absolute")
|
|
}
|
|
if trimmed == "/" || trimmed == strings.TrimRight(trimmed, "/") {
|
|
return trimmed, nil
|
|
}
|
|
return strings.TrimRight(trimmed, "/"), nil
|
|
}
|
|
|
|
func parseNodeSelector(value string) (map[string]string, error) {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return nil, nil
|
|
}
|
|
result := map[string]string{}
|
|
for _, item := range strings.Split(trimmed, ",") {
|
|
trimmedItem := strings.TrimSpace(item)
|
|
if trimmedItem == "" {
|
|
continue
|
|
}
|
|
parts := strings.SplitN(trimmedItem, "=", 2)
|
|
if len(parts) != 2 {
|
|
return nil, fmt.Errorf("invalid gpu node selector %q: expected key=value", trimmedItem)
|
|
}
|
|
key := strings.TrimSpace(parts[0])
|
|
val := strings.TrimSpace(parts[1])
|
|
if key == "" || val == "" {
|
|
return nil, fmt.Errorf("invalid gpu node selector %q: key and value are required", trimmedItem)
|
|
}
|
|
result[key] = val
|
|
}
|
|
if len(result) == 0 {
|
|
return nil, nil
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// ParseNodeSelector exposes the launcher-compatible selector parser to the
|
|
// worker's narrowly scoped runtime configuration. Keeping one parser avoids a
|
|
// cache-only selector accepting syntax that the job launcher would reject.
|
|
func ParseNodeSelector(value string) (map[string]string, error) {
|
|
return parseNodeSelector(value)
|
|
}
|
|
|
|
func parseTolerations(value string) ([]corev1.Toleration, error) {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return nil, nil
|
|
}
|
|
var items []corev1.Toleration
|
|
if err := json.Unmarshal([]byte(trimmed), &items); err != nil {
|
|
return nil, fmt.Errorf("invalid gpu tolerations json: %w", err)
|
|
}
|
|
if len(items) == 0 {
|
|
return nil, nil
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func cloneStringMap(items map[string]string) map[string]string {
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
cloned := make(map[string]string, len(items))
|
|
for key, value := range items {
|
|
cloned[key] = value
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func cloneTolerations(items []corev1.Toleration) []corev1.Toleration {
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
cloned := make([]corev1.Toleration, len(items))
|
|
copy(cloned, items)
|
|
return cloned
|
|
}
|
|
|
|
func applyGPUScheduling(podSpec *corev1.PodSpec, input LaunchInput) {
|
|
if !hasPositiveGPU(input.GPULimit) {
|
|
return
|
|
}
|
|
if len(input.GPUNodeSelector) > 0 {
|
|
podSpec.NodeSelector = cloneStringMap(input.GPUNodeSelector)
|
|
}
|
|
if len(input.GPUTolerations) > 0 {
|
|
podSpec.Tolerations = cloneTolerations(input.GPUTolerations)
|
|
}
|
|
if trimmed := strings.TrimSpace(input.GPURuntimeClass); trimmed != "" {
|
|
podSpec.RuntimeClassName = &trimmed
|
|
}
|
|
}
|
|
|
|
func nodeMatchesSelector(node corev1.Node, selector map[string]string) bool {
|
|
if len(selector) == 0 {
|
|
return true
|
|
}
|
|
for key, expected := range selector {
|
|
if node.Labels[key] != expected {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func gpuAllocatableCount(node corev1.Node) int64 {
|
|
quantity, ok := node.Status.Allocatable[corev1.ResourceName("nvidia.com/gpu")]
|
|
if !ok {
|
|
return 0
|
|
}
|
|
value := quantity.Value()
|
|
if value < 0 {
|
|
return 0
|
|
}
|
|
return value
|
|
}
|
|
|
|
func gpuSelectorSummary(selector map[string]string) string {
|
|
if len(selector) == 0 {
|
|
return "<none>"
|
|
}
|
|
keys := make([]string, 0, len(selector))
|
|
for key := range selector {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
parts := make([]string, 0, len(keys))
|
|
for _, key := range keys {
|
|
parts = append(parts, key+"="+selector[key])
|
|
}
|
|
return strings.Join(parts, ",")
|
|
}
|
|
|
|
func (launcher *Launcher) gpuCapacity(ctx context.Context, selector map[string]string) (*GPUCapacitySnapshot, error) {
|
|
nodes, err := launcher.clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list kubernetes nodes: %w", err)
|
|
}
|
|
snapshot := &GPUCapacitySnapshot{
|
|
ResourceName: "nvidia.com/gpu",
|
|
NodeSelector: cloneStringMap(selector),
|
|
SelectorSummary: gpuSelectorSummary(selector),
|
|
Nodes: make([]GPUCapacityNode, 0, len(nodes.Items)),
|
|
CheckedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
|
}
|
|
for _, node := range nodes.Items {
|
|
allocatable := gpuAllocatableCount(node)
|
|
selectorMatched := nodeMatchesSelector(node, selector)
|
|
snapshot.Nodes = append(snapshot.Nodes, GPUCapacityNode{
|
|
Name: node.Name,
|
|
Allocatable: allocatable,
|
|
SelectorMatched: selectorMatched,
|
|
Unschedulable: node.Spec.Unschedulable,
|
|
})
|
|
snapshot.TotalAllocatable += allocatable
|
|
if selectorMatched && !node.Spec.Unschedulable {
|
|
snapshot.SchedulableAllocatable += allocatable
|
|
if allocatable > snapshot.MaxAllocatable {
|
|
snapshot.MaxAllocatable = allocatable
|
|
}
|
|
if allocatable > 0 {
|
|
snapshot.SchedulableNodeCount++
|
|
}
|
|
}
|
|
}
|
|
snapshot.Available = snapshot.MaxAllocatable > 0
|
|
return snapshot, nil
|
|
}
|
|
|
|
func (launcher *Launcher) GPUCapacity(ctx context.Context) (*GPUCapacitySnapshot, error) {
|
|
return launcher.gpuCapacity(ctx, launcher.gpuNodeSelector)
|
|
}
|
|
|
|
func (launcher *Launcher) ensureGPUCapacity(ctx context.Context, input LaunchInput) error {
|
|
if !hasPositiveGPU(input.GPULimit) {
|
|
return nil
|
|
}
|
|
requested, err := strconv.ParseInt(strings.TrimSpace(input.GPULimit), 10, 64)
|
|
if err != nil || requested <= 0 {
|
|
return ValidateGPURequest(input.GPULimit)
|
|
}
|
|
snapshot, err := launcher.gpuCapacity(ctx, input.GPUNodeSelector)
|
|
if err != nil {
|
|
return fmt.Errorf("gpu preflight failed: %w", err)
|
|
}
|
|
if snapshot.MaxAllocatable >= requested {
|
|
return nil
|
|
}
|
|
checked := make([]string, 0, len(snapshot.Nodes))
|
|
for _, node := range snapshot.Nodes {
|
|
checked = append(
|
|
checked,
|
|
fmt.Sprintf("%s nvidia.com/gpu=%d selectorMatched=%t unschedulable=%t", node.Name, node.Allocatable, node.SelectorMatched, node.Unschedulable),
|
|
)
|
|
}
|
|
if len(checked) == 0 {
|
|
checked = append(checked, "none")
|
|
}
|
|
return fmt.Errorf(
|
|
"gpu request cannot be scheduled: requested nvidia.com/gpu=%d, selector=%s; checked nodes: %s",
|
|
requested,
|
|
snapshot.SelectorSummary,
|
|
strings.Join(checked, "; "),
|
|
)
|
|
}
|
|
|
|
func CreateJobManifest(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, input LaunchInput) (*batchv1.Job, error) {
|
|
if err := ValidateGPURequest(input.GPULimit); err != nil {
|
|
return nil, err
|
|
}
|
|
resources, err := resourceRequirements(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var sidecarResources corev1.ResourceRequirements
|
|
if input.InferenceSidecar != nil {
|
|
sidecar := input.InferenceSidecar
|
|
if strings.TrimSpace(sidecar.Name) == "" || strings.TrimSpace(sidecar.Image) == "" || strings.ContainsAny(sidecar.Image, " \t\r\n") || sidecar.Port <= 0 || sidecar.Port > 65535 || !strings.HasPrefix(strings.TrimSpace(sidecar.HealthPath), "/") {
|
|
return nil, fmt.Errorf("inference sidecar requires a name, image, valid port and absolute health path")
|
|
}
|
|
if err := ValidateGPURequest(sidecar.GPULimit); err != nil {
|
|
return nil, fmt.Errorf("inference sidecar: %w", err)
|
|
}
|
|
sidecarResources, err = resourceRequirements(LaunchInput{CPURequest: sidecar.CPURequest, CPULimit: sidecar.CPULimit, MemoryRequest: sidecar.MemoryRequest, MemoryLimit: sidecar.MemoryLimit, GPULimit: sidecar.GPULimit})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if hasPositiveGPU(sidecar.GPULimit) {
|
|
delete(resources.Requests, corev1.ResourceName("nvidia.com/gpu"))
|
|
delete(resources.Limits, corev1.ResourceName("nvidia.com/gpu"))
|
|
}
|
|
}
|
|
sharedMemorySize := strings.TrimSpace(input.SharedMemorySize)
|
|
if sharedMemorySize != "" {
|
|
quantity, quantityErr := resource.ParseQuantity(sharedMemorySize)
|
|
if quantityErr != nil || quantity.Value() <= 0 {
|
|
return nil, fmt.Errorf("shared memory size must be a positive Kubernetes resource quantity")
|
|
}
|
|
sharedMemorySize = quantity.String()
|
|
}
|
|
codeSubPath, err := cleanVolumeSubPath(input.CodeSubPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
codeMountPath := ""
|
|
if codeSubPath != "" {
|
|
codeMountPath, err = cleanMountPath(input.CodeMountPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
cleanPVCMounts := make([]PVCMount, 0, len(input.PVCMounts))
|
|
for _, item := range input.PVCMounts {
|
|
item.Name = strings.TrimSpace(item.Name)
|
|
item.PVCName = strings.TrimSpace(item.PVCName)
|
|
item.SubPath, err = cleanVolumeSubPath(item.SubPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("PVC mount %s: %w", item.Name, err)
|
|
}
|
|
item.MountPath, err = cleanMountPath(item.MountPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("PVC mount %s: %w", item.Name, err)
|
|
}
|
|
if item.Name == "" || item.PVCName == "" {
|
|
return nil, fmt.Errorf("PVC mount name and pvcName are required")
|
|
}
|
|
cleanPVCMounts = append(cleanPVCMounts, item)
|
|
}
|
|
jobName := launchJobName(input)
|
|
envVars := []corev1.EnvVar{
|
|
{Name: "RUN_ID", Value: input.RunID},
|
|
{Name: "WORKFLOW_ID", Value: input.WorkflowID},
|
|
{Name: "NODE_ID", Value: input.NodeID},
|
|
{Name: "NODE_LABEL", Value: input.NodeLabel},
|
|
{Name: "RUN_ATTEMPT", Value: strconv.Itoa(launchAttempt(input))},
|
|
}
|
|
envKeys := make([]string, 0, len(input.Env))
|
|
for key := range input.Env {
|
|
if _, reserved := launcherSystemEnvKeys[key]; reserved {
|
|
continue
|
|
}
|
|
envKeys = append(envKeys, key)
|
|
}
|
|
sort.Strings(envKeys)
|
|
for _, key := range envKeys {
|
|
if _, sensitive := input.SensitiveEnv[key]; sensitive {
|
|
continue
|
|
}
|
|
value := input.Env[key]
|
|
envVars = append(envVars, corev1.EnvVar{Name: key, Value: value})
|
|
}
|
|
sensitiveKeys := make([]string, 0, len(input.SensitiveEnv))
|
|
for key := range input.SensitiveEnv {
|
|
if _, reserved := launcherSystemEnvKeys[key]; reserved {
|
|
continue
|
|
}
|
|
sensitiveKeys = append(sensitiveKeys, key)
|
|
}
|
|
sort.Strings(sensitiveKeys)
|
|
sensitiveSecretName := launchSensitiveEnvSecretName(input)
|
|
for _, key := range sensitiveKeys {
|
|
envVars = append(envVars, corev1.EnvVar{
|
|
Name: key,
|
|
ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{Name: sensitiveSecretName},
|
|
Key: key,
|
|
}},
|
|
})
|
|
}
|
|
job := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: jobName,
|
|
Namespace: namespace,
|
|
Labels: map[string]string{
|
|
"app": "droboticflow",
|
|
"component": "operator-runner",
|
|
"runId": input.RunID,
|
|
"workflowId": input.WorkflowID,
|
|
"nodeId": input.NodeID,
|
|
"attempt": strconv.Itoa(launchAttempt(input)),
|
|
},
|
|
},
|
|
Spec: batchv1.JobSpec{
|
|
BackoffLimit: func() *int32 { value := defaultJobBackoffLimit; return &value }(),
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"app": "droboticflow",
|
|
"component": "operator-runner",
|
|
"runId": input.RunID,
|
|
"nodeId": input.NodeID,
|
|
"attempt": strconv.Itoa(launchAttempt(input)),
|
|
},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
SecurityContext: &corev1.PodSecurityContext{
|
|
RunAsNonRoot: boolPtr(true),
|
|
RunAsUser: int64Ptr(10001),
|
|
RunAsGroup: int64Ptr(10001),
|
|
FSGroup: int64Ptr(10001),
|
|
SeccompProfile: &corev1.SeccompProfile{
|
|
Type: corev1.SeccompProfileTypeRuntimeDefault,
|
|
},
|
|
},
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "operator-runner",
|
|
Image: input.Image,
|
|
ImagePullPolicy: corev1.PullAlways,
|
|
Command: input.Command,
|
|
Args: input.Args,
|
|
Env: envVars,
|
|
Resources: resources,
|
|
SecurityContext: &corev1.SecurityContext{
|
|
AllowPrivilegeEscalation: boolPtr(false),
|
|
Capabilities: &corev1.Capabilities{
|
|
Drop: []corev1.Capability{"ALL"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if input.InferenceSidecar != nil {
|
|
sidecar := input.InferenceSidecar
|
|
restartAlways := corev1.ContainerRestartPolicyAlways
|
|
sidecarEnv := make([]corev1.EnvVar, 0, len(sidecar.Env))
|
|
sidecarKeys := make([]string, 0, len(sidecar.Env))
|
|
for key := range sidecar.Env {
|
|
sidecarKeys = append(sidecarKeys, key)
|
|
}
|
|
sort.Strings(sidecarKeys)
|
|
for _, key := range sidecarKeys {
|
|
sidecarEnv = append(sidecarEnv, corev1.EnvVar{Name: key, Value: sidecar.Env[key]})
|
|
}
|
|
job.Spec.Template.Spec.InitContainers = []corev1.Container{{
|
|
Name: sidecar.Name, Image: sidecar.Image, ImagePullPolicy: corev1.PullAlways,
|
|
RestartPolicy: &restartAlways, Env: sidecarEnv, Resources: sidecarResources,
|
|
Ports: []corev1.ContainerPort{{Name: "inference-http", ContainerPort: sidecar.Port, Protocol: corev1.ProtocolTCP}},
|
|
StartupProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: sidecar.HealthPath, Port: intstr.FromInt32(sidecar.Port)}},
|
|
InitialDelaySeconds: 2, PeriodSeconds: 5, TimeoutSeconds: 3, FailureThreshold: 120,
|
|
},
|
|
SecurityContext: &corev1.SecurityContext{RunAsNonRoot: boolPtr(false), RunAsUser: int64Ptr(0), AllowPrivilegeEscalation: boolPtr(false)},
|
|
}}
|
|
}
|
|
if sharedMemorySize != "" {
|
|
quantity := resource.MustParse(sharedMemorySize)
|
|
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
|
|
Name: "droboticflow-shm",
|
|
VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{
|
|
Medium: corev1.StorageMediumMemory,
|
|
SizeLimit: &quantity,
|
|
}},
|
|
})
|
|
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
|
Name: "droboticflow-shm",
|
|
MountPath: "/dev/shm",
|
|
})
|
|
if len(job.Spec.Template.Spec.InitContainers) > 0 {
|
|
job.Spec.Template.Spec.InitContainers[0].VolumeMounts = append(job.Spec.Template.Spec.InitContainers[0].VolumeMounts, corev1.VolumeMount{Name: "droboticflow-shm", MountPath: "/dev/shm"})
|
|
}
|
|
}
|
|
if input.JobTTLSeconds > 0 {
|
|
job.Spec.TTLSecondsAfterFinished = func() *int32 {
|
|
value := input.JobTTLSeconds
|
|
return &value
|
|
}()
|
|
}
|
|
if input.TimeoutSeconds > 0 {
|
|
job.Spec.ActiveDeadlineSeconds = func() *int64 {
|
|
value := int64(input.TimeoutSeconds)
|
|
return &value
|
|
}()
|
|
}
|
|
if imagePullSecretName != "" {
|
|
job.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: imagePullSecretName}}
|
|
}
|
|
applyGPUScheduling(&job.Spec.Template.Spec, input)
|
|
if strings.TrimSpace(workspacePVCName) != "" {
|
|
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
|
|
Name: "workspace",
|
|
VolumeSource: corev1.VolumeSource{
|
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
|
ClaimName: workspacePVCName,
|
|
},
|
|
},
|
|
})
|
|
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
|
Name: "workspace",
|
|
MountPath: workspaceMountPath,
|
|
ReadOnly: input.CodeReadOnly,
|
|
})
|
|
if len(job.Spec.Template.Spec.InitContainers) > 0 {
|
|
job.Spec.Template.Spec.InitContainers[0].VolumeMounts = append(job.Spec.Template.Spec.InitContainers[0].VolumeMounts, corev1.VolumeMount{Name: "workspace", MountPath: workspaceMountPath})
|
|
}
|
|
if codeSubPath != "" {
|
|
runSubPath, err := cleanVolumeSubPath(pathClean("system/runs/" + input.RunID))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
|
Name: "workspace",
|
|
MountPath: strings.TrimRight(workspaceMountPath, "/") + "/system/runs/" + input.RunID,
|
|
SubPath: runSubPath,
|
|
})
|
|
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
|
Name: "workspace",
|
|
MountPath: codeMountPath,
|
|
SubPath: codeSubPath,
|
|
ReadOnly: input.CodeReadOnly,
|
|
})
|
|
}
|
|
}
|
|
volumeNames := map[string]string{workspacePVCName: "workspace"}
|
|
for index, item := range cleanPVCMounts {
|
|
volumeName, exists := volumeNames[item.PVCName]
|
|
if !exists {
|
|
volumeName = fmt.Sprintf("lightweight-pvc-%d", index)
|
|
volumeNames[item.PVCName] = volumeName
|
|
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
|
|
Name: volumeName,
|
|
VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: item.PVCName}},
|
|
})
|
|
}
|
|
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
|
Name: volumeName, MountPath: item.MountPath, SubPath: item.SubPath, ReadOnly: item.ReadOnly,
|
|
})
|
|
}
|
|
if input.RegistryAuth && imagePullSecretName != "" {
|
|
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
|
|
Name: "registry-auth",
|
|
VolumeSource: corev1.VolumeSource{
|
|
Secret: &corev1.SecretVolumeSource{
|
|
SecretName: imagePullSecretName,
|
|
Items: []corev1.KeyToPath{
|
|
{Key: ".dockerconfigjson", Path: "config.json"},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
|
Name: "registry-auth",
|
|
MountPath: "/kaniko/.docker",
|
|
})
|
|
}
|
|
return job, nil
|
|
}
|
|
|
|
func loadRestConfig() (*rest.Config, error) {
|
|
if config, err := rest.InClusterConfig(); err == nil {
|
|
return config, nil
|
|
}
|
|
kubeconfig := os.Getenv("KUBECONFIG")
|
|
if kubeconfig == "" {
|
|
home, err := os.UserHomeDir()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
kubeconfig = filepath.Join(home, ".kube", "config")
|
|
}
|
|
return clientcmd.BuildConfigFromFlags("", kubeconfig)
|
|
}
|
|
|
|
func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int) (*Launcher, error) {
|
|
trimmedPVCName := strings.TrimSpace(workspacePVCName)
|
|
if trimmedPVCName == "" {
|
|
return nil, fmt.Errorf("workspace pvc name is required")
|
|
}
|
|
trimmedMountPath := strings.TrimSpace(workspaceMountPath)
|
|
if trimmedMountPath == "" {
|
|
trimmedMountPath = "/workspace"
|
|
}
|
|
if !strings.HasPrefix(trimmedMountPath, "/") {
|
|
return nil, fmt.Errorf("workspace mount path must be absolute")
|
|
}
|
|
parsedGPUNodeSelector, err := parseNodeSelector(gpuNodeSelector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
parsedGPUTolerations, err := parseTolerations(gpuTolerationsJSON)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if jobTTLSeconds < 0 {
|
|
jobTTLSeconds = 0
|
|
}
|
|
sharedMemorySize := strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHARED_MEMORY_SIZE"))
|
|
if sharedMemorySize == "" {
|
|
sharedMemorySize = strings.TrimSpace(os.Getenv("DROBOTICFLOW_SHM_SIZE"))
|
|
}
|
|
if sharedMemorySize == "" {
|
|
sharedMemorySize = defaultSharedMemorySize
|
|
}
|
|
sharedMemoryQuantity, err := resource.ParseQuantity(sharedMemorySize)
|
|
if err != nil || sharedMemoryQuantity.Value() <= 0 {
|
|
return nil, fmt.Errorf("DROBOTICFLOW_SHARED_MEMORY_SIZE must be a positive Kubernetes resource quantity")
|
|
}
|
|
launcher := &Launcher{
|
|
namespace: namespace,
|
|
imagePullSecretName: imagePullSecretName,
|
|
workspacePVCName: trimmedPVCName,
|
|
workspaceMountPath: trimmedMountPath,
|
|
gpuNodeSelector: parsedGPUNodeSelector,
|
|
gpuTolerations: parsedGPUTolerations,
|
|
gpuRuntimeClass: strings.TrimSpace(gpuRuntimeClass),
|
|
jobTTLSeconds: int32(jobTTLSeconds),
|
|
sharedMemorySize: sharedMemoryQuantity.String(),
|
|
}
|
|
config, err := loadRestConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
clientset, err := kubernetes.NewForConfig(config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
launcher.clientset = clientset
|
|
launcher.restConfig = config
|
|
return launcher, nil
|
|
}
|
|
|
|
func waitDeadlineForLaunch(ctx context.Context, input LaunchInput, now time.Time) time.Time {
|
|
if deadline, ok := ctx.Deadline(); ok {
|
|
return deadline
|
|
}
|
|
if input.TimeoutSeconds > 0 {
|
|
return now.Add(time.Duration(input.TimeoutSeconds)*time.Second + 30*time.Second)
|
|
}
|
|
if input.LongRunning {
|
|
return now.Add(longRunningJobWaitTimeout)
|
|
}
|
|
return now.Add(defaultJobWaitTimeout)
|
|
}
|
|
|
|
func (launcher *Launcher) readJobOutput(ctx context.Context, jobName string) (string, error) {
|
|
var podName string
|
|
for attempt := 0; attempt < 30; attempt++ {
|
|
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{
|
|
LabelSelector: "job-name=" + jobName,
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(pods.Items) > 0 {
|
|
podName = pods.Items[0].Name
|
|
break
|
|
}
|
|
time.Sleep(time.Second)
|
|
}
|
|
if podName == "" {
|
|
return "", fmt.Errorf("no pod found for job %s", jobName)
|
|
}
|
|
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, &corev1.PodLogOptions{}).Stream(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer stream.Close()
|
|
output, err := io.ReadAll(stream)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(output), nil
|
|
}
|
|
|
|
func (launcher *Launcher) readJobOutputNow(ctx context.Context, jobName string) (string, error) {
|
|
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(pods.Items) == 0 {
|
|
return "", fmt.Errorf("no pod found for job %s", jobName)
|
|
}
|
|
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}).Stream(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer stream.Close()
|
|
output, err := io.ReadAll(stream)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(output), nil
|
|
}
|
|
|
|
// findJobPodName returns the name of the (first) pod backing jobName, waiting up
|
|
// to ~30s for it to exist. Returns "" if no pod appears in time.
|
|
func (launcher *Launcher) findJobPodName(ctx context.Context, jobName string) string {
|
|
for attempt := 0; attempt < 30; attempt++ {
|
|
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{
|
|
LabelSelector: "job-name=" + jobName,
|
|
})
|
|
if err == nil && len(pods.Items) > 0 {
|
|
return pods.Items[0].Name
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ""
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// podGPUSampleInterval controls how often the platform samples GPU usage inside a
|
|
// custom-container pod. Remote exec is heavier than operator-runner's in-process
|
|
// sampler, so the default is 10s (vs 1s). Override via
|
|
// DROBOTICFLOW_POD_GPU_SAMPLE_INTERVAL_SECONDS (clamped to [1, 300]).
|
|
func podGPUSampleInterval() time.Duration {
|
|
raw := strings.TrimSpace(os.Getenv("DROBOTICFLOW_POD_GPU_SAMPLE_INTERVAL_SECONDS"))
|
|
if raw == "" {
|
|
return 10 * time.Second
|
|
}
|
|
value, err := strconv.ParseFloat(raw, 64)
|
|
if err != nil || value < 1 {
|
|
return 10 * time.Second
|
|
}
|
|
if value > 300 {
|
|
value = 300
|
|
}
|
|
return time.Duration(value * float64(time.Second))
|
|
}
|
|
|
|
// parseGPUSampleOutput parses `nvidia-smi --query-gpu=utilization.gpu,memory.used,
|
|
// memory.total,name --format=csv,noheader,nounits` output into a single rolled-up
|
|
// GPUSample (max util across GPUs, summed memory, distinct names). Mirrors the
|
|
// rollup in backend/operator-runner/resource_monitor.py so platform-collected
|
|
// samples are directly comparable to operator-runner self-reported ones.
|
|
func parseGPUSampleOutput(output string) GPUSample {
|
|
type gpuLine struct {
|
|
util int
|
|
used int
|
|
total int
|
|
name string
|
|
}
|
|
var lines []gpuLine
|
|
for _, raw := range strings.Split(output, "\n") {
|
|
parts := strings.Split(strings.TrimSpace(raw), ",")
|
|
if len(parts) < 4 {
|
|
continue
|
|
}
|
|
util, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
|
|
used, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
|
|
total, err3 := strconv.Atoi(strings.TrimSpace(parts[2]))
|
|
if err1 != nil || err2 != nil || err3 != nil {
|
|
continue
|
|
}
|
|
lines = append(lines, gpuLine{util: util, used: used, total: total, name: strings.TrimSpace(parts[3])})
|
|
}
|
|
if len(lines) == 0 {
|
|
return GPUSample{}
|
|
}
|
|
maxUtil := 0
|
|
sumUsed := 0
|
|
sumTotal := 0
|
|
nameSet := make(map[string]struct{})
|
|
names := make([]string, 0)
|
|
for _, line := range lines {
|
|
if line.util > maxUtil {
|
|
maxUtil = line.util
|
|
}
|
|
sumUsed += line.used
|
|
sumTotal += line.total
|
|
if _, ok := nameSet[line.name]; !ok {
|
|
nameSet[line.name] = struct{}{}
|
|
names = append(names, line.name)
|
|
}
|
|
}
|
|
if maxUtil > 100 {
|
|
maxUtil = 100
|
|
}
|
|
if maxUtil < 0 {
|
|
maxUtil = 0
|
|
}
|
|
return GPUSample{
|
|
Timestamp: time.Now().UTC().Format("2006-01-02T15:04:05.000000Z"),
|
|
GPU: maxUtil,
|
|
GPUMemoryUsedMiB: sumUsed,
|
|
GPUMemoryTotalMiB: sumTotal,
|
|
GPUName: strings.Join(names, ", "),
|
|
GPUCount: len(lines),
|
|
}
|
|
}
|
|
|
|
// samplePodGPU execs nvidia-smi inside the pod's operator-runner container and
|
|
// returns one rolled-up GPUSample. Fails gracefully (returns error, zero sample)
|
|
// when the pod has no GPU, nvidia-smi is absent, restConfig is unavailable, or the
|
|
// worker lacks pods/exec RBAC — the caller simply skips that sample.
|
|
func (launcher *Launcher) samplePodGPU(ctx context.Context, podName string) (GPUSample, error) {
|
|
empty := GPUSample{}
|
|
if launcher.restConfig == nil {
|
|
return empty, fmt.Errorf("rest config unavailable")
|
|
}
|
|
request := launcher.clientset.CoreV1().RESTClient().Post().
|
|
Resource("pods").
|
|
Namespace(launcher.namespace).
|
|
Name(podName).
|
|
SubResource("exec").
|
|
VersionedParams(&corev1.PodExecOptions{
|
|
Container: "operator-runner",
|
|
Command: []string{
|
|
"nvidia-smi",
|
|
"--query-gpu=utilization.gpu,memory.used,memory.total,name",
|
|
"--format=csv,noheader,nounits",
|
|
},
|
|
Stdout: true,
|
|
Stderr: false,
|
|
}, scheme.ParameterCodec)
|
|
executor, err := remotecommand.NewSPDYExecutor(launcher.restConfig, http.MethodPost, request.URL())
|
|
if err != nil {
|
|
return empty, err
|
|
}
|
|
var stdout bytes.Buffer
|
|
if err := executor.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: &stdout}); err != nil {
|
|
return empty, err
|
|
}
|
|
return parseGPUSampleOutput(stdout.String()), nil
|
|
}
|
|
|
|
// findRunningNodePod returns the name of the first Running pod backing runID+nodeID,
|
|
// listed via the runId/nodeId labels every operator pod carries. Returns "" (no
|
|
// error) when no pod is Running yet — callers treat that as "not live right now".
|
|
func (launcher *Launcher) findRunningNodePod(ctx context.Context, runID string, nodeID string) (string, error) {
|
|
runID = strings.TrimSpace(runID)
|
|
nodeID = strings.TrimSpace(nodeID)
|
|
if runID == "" || nodeID == "" {
|
|
return "", fmt.Errorf("runID and nodeID are required")
|
|
}
|
|
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{
|
|
LabelSelector: "runId=" + runID + ",nodeId=" + nodeID,
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
for _, pod := range pods.Items {
|
|
if pod.Status.Phase == corev1.PodRunning {
|
|
return pod.Name, nil
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
// SamplePodGPUForRun samples GPU usage (nvidia-smi) inside the pod backing
|
|
// runID+nodeID. The worker uses it to surface live GPU metrics for custom-container
|
|
// operators DURING execution — those bypass operator-runner's in-process
|
|
// ResourceSampler, so otherwise the only GPU samples appear at node completion (when
|
|
// the launcher flushes its internal buffer). Fails gracefully (returns error) when
|
|
// no Running pod exists, the pod has no GPU, nvidia-smi is absent, or pods/exec
|
|
// RBAC is unavailable — the caller simply skips that sample.
|
|
func (launcher *Launcher) SamplePodGPUForRun(ctx context.Context, runID string, nodeID string) (GPUSample, error) {
|
|
empty := GPUSample{}
|
|
if launcher.restConfig == nil {
|
|
return empty, fmt.Errorf("rest config unavailable")
|
|
}
|
|
podName, err := launcher.findRunningNodePod(ctx, runID, nodeID)
|
|
if err != nil {
|
|
return empty, err
|
|
}
|
|
if podName == "" {
|
|
return empty, fmt.Errorf("no running pod for run %s node %s", runID, nodeID)
|
|
}
|
|
return launcher.samplePodGPU(ctx, podName)
|
|
}
|
|
|
|
// TailNodePodLogs returns the last tailLines of stdout from the operator-runner
|
|
// container of the pod backing runID+nodeID. The API uses it to surface live
|
|
// execution logs for a running node before the operator emits its structured
|
|
// result.json (which the worker persists only at node completion). Returns "" when
|
|
// no Running pod exists, so the caller falls back to the stored run logs.
|
|
func (launcher *Launcher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
|
|
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, false)
|
|
}
|
|
|
|
// TailNodePodLogsWithTimestamps keeps the Kubernetes source timestamp on each
|
|
// line so repeated API polls can deduplicate the same live log entry.
|
|
func (launcher *Launcher) TailNodePodLogsWithTimestamps(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
|
|
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, true)
|
|
}
|
|
|
|
func (launcher *Launcher) tailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64, timestamps bool) (string, error) {
|
|
if tailLines <= 0 {
|
|
tailLines = 300
|
|
}
|
|
podName, err := launcher.findRunningNodePod(ctx, runID, nodeID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if podName == "" {
|
|
return "", nil
|
|
}
|
|
tail := tailLines
|
|
options := &corev1.PodLogOptions{
|
|
Container: "operator-runner",
|
|
TailLines: &tail,
|
|
}
|
|
if timestamps {
|
|
options.Timestamps = true
|
|
}
|
|
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, options).Stream(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer stream.Close()
|
|
output, err := io.ReadAll(stream)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(output), nil
|
|
}
|
|
|
|
func appendDiagnosticLine(builder *strings.Builder, format string, args ...any) {
|
|
line := strings.TrimSpace(fmt.Sprintf(format, args...))
|
|
if line == "" {
|
|
return
|
|
}
|
|
if builder.Len() > 0 {
|
|
builder.WriteString("\n")
|
|
}
|
|
builder.WriteString(line)
|
|
}
|
|
|
|
func containerStateDiagnostic(name string, state corev1.ContainerState) string {
|
|
switch {
|
|
case state.Waiting != nil:
|
|
return fmt.Sprintf("container %s waiting reason=%s message=%s", name, state.Waiting.Reason, state.Waiting.Message)
|
|
case state.Terminated != nil:
|
|
return fmt.Sprintf("container %s terminated exitCode=%d reason=%s message=%s", name, state.Terminated.ExitCode, state.Terminated.Reason, state.Terminated.Message)
|
|
case state.Running != nil:
|
|
return fmt.Sprintf("container %s running startedAt=%s", name, state.Running.StartedAt.Time.Format(time.RFC3339))
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func (launcher *Launcher) readJobDiagnostics(ctx context.Context, job *batchv1.Job, logs string, logErr error) string {
|
|
var builder strings.Builder
|
|
if job != nil {
|
|
for _, condition := range job.Status.Conditions {
|
|
appendDiagnosticLine(&builder, "job condition type=%s status=%s reason=%s message=%s", condition.Type, condition.Status, condition.Reason, condition.Message)
|
|
}
|
|
}
|
|
jobName := ""
|
|
if job != nil {
|
|
jobName = job.Name
|
|
}
|
|
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{
|
|
LabelSelector: "job-name=" + jobName,
|
|
})
|
|
if err != nil {
|
|
appendDiagnosticLine(&builder, "pod inspection failed: %v", err)
|
|
} else if len(pods.Items) == 0 {
|
|
appendDiagnosticLine(&builder, "no pod found for job %s", jobName)
|
|
}
|
|
for _, pod := range pods.Items {
|
|
appendDiagnosticLine(&builder, "pod %s phase=%s reason=%s message=%s", pod.Name, pod.Status.Phase, pod.Status.Reason, pod.Status.Message)
|
|
for _, condition := range pod.Status.Conditions {
|
|
if condition.Status != corev1.ConditionTrue {
|
|
appendDiagnosticLine(&builder, "pod %s condition type=%s status=%s reason=%s message=%s", pod.Name, condition.Type, condition.Status, condition.Reason, condition.Message)
|
|
}
|
|
}
|
|
for _, status := range pod.Status.InitContainerStatuses {
|
|
if diagnostic := containerStateDiagnostic(status.Name, status.State); diagnostic != "" {
|
|
appendDiagnosticLine(&builder, "pod %s %s", pod.Name, diagnostic)
|
|
}
|
|
}
|
|
for _, status := range pod.Status.ContainerStatuses {
|
|
if diagnostic := containerStateDiagnostic(status.Name, status.State); diagnostic != "" {
|
|
appendDiagnosticLine(&builder, "pod %s %s", pod.Name, diagnostic)
|
|
}
|
|
}
|
|
events, eventErr := launcher.clientset.CoreV1().Events(launcher.namespace).List(ctx, metav1.ListOptions{
|
|
FieldSelector: "involvedObject.name=" + pod.Name,
|
|
})
|
|
if eventErr != nil {
|
|
appendDiagnosticLine(&builder, "pod %s event inspection failed: %v", pod.Name, eventErr)
|
|
continue
|
|
}
|
|
sort.Slice(events.Items, func(i int, j int) bool {
|
|
left := events.Items[i].LastTimestamp.Time
|
|
if left.IsZero() {
|
|
left = events.Items[i].EventTime.Time
|
|
}
|
|
right := events.Items[j].LastTimestamp.Time
|
|
if right.IsZero() {
|
|
right = events.Items[j].EventTime.Time
|
|
}
|
|
return left.Before(right)
|
|
})
|
|
for _, event := range events.Items {
|
|
appendDiagnosticLine(&builder, "pod %s event type=%s reason=%s message=%s", pod.Name, event.Type, event.Reason, event.Message)
|
|
}
|
|
}
|
|
if trimmed := strings.TrimSpace(logs); trimmed != "" {
|
|
appendDiagnosticLine(&builder, "pod logs:\n%s", trimmed)
|
|
} else if logErr != nil {
|
|
appendDiagnosticLine(&builder, "pod log read failed: %v", logErr)
|
|
}
|
|
return builder.String()
|
|
}
|
|
|
|
func (launcher *Launcher) TerminateJob(ctx context.Context, namespace string, jobName string) error {
|
|
targetNamespace := strings.TrimSpace(namespace)
|
|
if targetNamespace == "" {
|
|
targetNamespace = launcher.namespace
|
|
}
|
|
propagation := metav1.DeletePropagationBackground
|
|
err := launcher.clientset.BatchV1().Jobs(targetNamespace).Delete(ctx, jobName, metav1.DeleteOptions{
|
|
PropagationPolicy: &propagation,
|
|
})
|
|
if err != nil && strings.Contains(strings.ToLower(err.Error()), "not found") {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (launcher *Launcher) TerminateRun(ctx context.Context, namespace string, runID string) error {
|
|
if launcher.clientset == nil || strings.TrimSpace(runID) == "" {
|
|
return nil
|
|
}
|
|
targetNamespace := strings.TrimSpace(namespace)
|
|
if targetNamespace == "" {
|
|
targetNamespace = launcher.namespace
|
|
}
|
|
jobs, err := launcher.clientset.BatchV1().Jobs(targetNamespace).List(
|
|
ctx,
|
|
metav1.ListOptions{LabelSelector: "runId=" + strings.TrimSpace(runID)},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, job := range jobs.Items {
|
|
if err := launcher.TerminateJob(ctx, targetNamespace, job.Name); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
pods, err := launcher.clientset.CoreV1().Pods(targetNamespace).List(
|
|
ctx,
|
|
metav1.ListOptions{LabelSelector: "runId=" + strings.TrimSpace(runID)},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
propagation := metav1.DeletePropagationBackground
|
|
for _, pod := range pods.Items {
|
|
err := launcher.clientset.CoreV1().Pods(targetNamespace).Delete(ctx, pod.Name, metav1.DeleteOptions{PropagationPolicy: &propagation})
|
|
if err != nil && !apierrors.IsNotFound(err) {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// TerminateRunAndWait must finish before deleting a Run's writable workspace.
|
|
func (launcher *Launcher) TerminateRunAndWait(ctx context.Context, namespace, runID string) error {
|
|
if launcher.clientset == nil {
|
|
return fmt.Errorf("Kubernetes client unavailable")
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
|
|
defer cancel()
|
|
if namespace == "" {
|
|
namespace = launcher.namespace
|
|
}
|
|
for {
|
|
if err := launcher.TerminateRun(ctx, namespace, runID); err != nil {
|
|
return err
|
|
}
|
|
jobs, err := launcher.clientset.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId=" + runID})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pods, err := launcher.clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId=" + runID})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(jobs.Items) == 0 && len(pods.Items) == 0 {
|
|
return nil
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return fmt.Errorf("runtime termination not verified: %w", ctx.Err())
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
}
|
|
|
|
func orphanRuntimeCreatedAt(timestamp metav1.Time) string {
|
|
if timestamp.IsZero() {
|
|
return ""
|
|
}
|
|
return timestamp.UTC().Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func (launcher *Launcher) ReconcileOrphanedRuntimes(ctx context.Context, knownRunIDs map[string]struct{}, olderThan time.Time, cleanup bool) ([]OrphanRuntime, error) {
|
|
if launcher == nil || launcher.clientset == nil {
|
|
return nil, nil
|
|
}
|
|
result := make([]OrphanRuntime, 0)
|
|
jobs, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId"})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, job := range jobs.Items {
|
|
runID := strings.TrimSpace(job.Labels["runId"])
|
|
if runID == "" || !job.CreationTimestamp.Time.Before(olderThan) {
|
|
continue
|
|
}
|
|
if _, exists := knownRunIDs[runID]; exists {
|
|
continue
|
|
}
|
|
item := OrphanRuntime{
|
|
Kind: "Job",
|
|
Name: job.Name,
|
|
Namespace: job.Namespace,
|
|
RunID: runID,
|
|
CreatedAt: orphanRuntimeCreatedAt(job.CreationTimestamp),
|
|
}
|
|
if cleanup {
|
|
if err := launcher.TerminateJob(ctx, job.Namespace, job.Name); err != nil {
|
|
item.CleanupError = err.Error()
|
|
} else {
|
|
item.Cleaned = true
|
|
}
|
|
}
|
|
result = append(result, item)
|
|
}
|
|
|
|
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId"})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, pod := range pods.Items {
|
|
runID := strings.TrimSpace(pod.Labels["runId"])
|
|
if runID == "" || !pod.CreationTimestamp.Time.Before(olderThan) {
|
|
continue
|
|
}
|
|
if _, exists := knownRunIDs[runID]; exists {
|
|
continue
|
|
}
|
|
item := OrphanRuntime{
|
|
Kind: "Pod",
|
|
Name: pod.Name,
|
|
Namespace: pod.Namespace,
|
|
RunID: runID,
|
|
CreatedAt: orphanRuntimeCreatedAt(pod.CreationTimestamp),
|
|
}
|
|
if cleanup {
|
|
propagation := metav1.DeletePropagationBackground
|
|
err := launcher.clientset.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{PropagationPolicy: &propagation})
|
|
if err != nil && !apierrors.IsNotFound(err) {
|
|
item.CleanupError = err.Error()
|
|
} else {
|
|
item.Cleaned = true
|
|
}
|
|
}
|
|
result = append(result, item)
|
|
}
|
|
sort.Slice(result, func(i int, j int) bool {
|
|
if result[i].RunID == result[j].RunID {
|
|
if result[i].Kind == result[j].Kind {
|
|
return result[i].Name < result[j].Name
|
|
}
|
|
return result[i].Kind < result[j].Kind
|
|
}
|
|
return result[i].RunID < result[j].RunID
|
|
})
|
|
return result, nil
|
|
}
|
|
|
|
func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*LaunchResult, error) {
|
|
if err := ValidateGPURequest(input.GPULimit); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(input.GPUNodeSelector) == 0 {
|
|
input.GPUNodeSelector = cloneStringMap(launcher.gpuNodeSelector)
|
|
}
|
|
if len(input.GPUTolerations) == 0 {
|
|
input.GPUTolerations = cloneTolerations(launcher.gpuTolerations)
|
|
}
|
|
if strings.TrimSpace(input.GPURuntimeClass) == "" {
|
|
input.GPURuntimeClass = launcher.gpuRuntimeClass
|
|
}
|
|
if input.JobTTLSeconds == 0 {
|
|
input.JobTTLSeconds = launcher.jobTTLSeconds
|
|
}
|
|
if strings.TrimSpace(input.SharedMemorySize) == "" {
|
|
input.SharedMemorySize = launcher.sharedMemorySize
|
|
}
|
|
job, err := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, launcher.workspacePVCName, launcher.workspaceMountPath, input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := launcher.ensureGPUCapacity(ctx, input); err != nil {
|
|
return nil, err
|
|
}
|
|
created, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Create(ctx, job, metav1.CreateOptions{})
|
|
jobCreated := err == nil
|
|
if err != nil {
|
|
if !apierrors.IsAlreadyExists(err) {
|
|
return nil, err
|
|
}
|
|
created, err = launcher.clientset.BatchV1().Jobs(launcher.namespace).Get(ctx, job.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if len(input.SensitiveEnv) > 0 {
|
|
secretData := make(map[string]string, len(input.SensitiveEnv))
|
|
for key, value := range input.SensitiveEnv {
|
|
if _, reserved := launcherSystemEnvKeys[key]; reserved {
|
|
continue
|
|
}
|
|
secretData[key] = value
|
|
}
|
|
controller := true
|
|
blockOwnerDeletion := true
|
|
sensitiveSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: launchSensitiveEnvSecretName(input), Namespace: launcher.namespace,
|
|
Labels: map[string]string{"app": "droboticflow", "component": "operator-sensitive-env", "runId": input.RunID, "nodeId": input.NodeID},
|
|
OwnerReferences: []metav1.OwnerReference{{
|
|
APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", Name: created.Name, UID: created.UID,
|
|
Controller: &controller, BlockOwnerDeletion: &blockOwnerDeletion,
|
|
}},
|
|
},
|
|
Type: corev1.SecretTypeOpaque, StringData: secretData,
|
|
}
|
|
if _, secretErr := launcher.clientset.CoreV1().Secrets(launcher.namespace).Create(ctx, sensitiveSecret, metav1.CreateOptions{}); secretErr != nil && !apierrors.IsAlreadyExists(secretErr) {
|
|
if jobCreated {
|
|
propagation := metav1.DeletePropagationBackground
|
|
_ = launcher.clientset.BatchV1().Jobs(launcher.namespace).Delete(ctx, created.Name, metav1.DeleteOptions{PropagationPolicy: &propagation})
|
|
}
|
|
return nil, fmt.Errorf("create owner-referenced operator sensitive environment secret: %w", secretErr)
|
|
}
|
|
}
|
|
jobName := created.Name
|
|
reference := &JobReference{JobName: jobName, Namespace: launcher.namespace}
|
|
|
|
waitParentCtx := ctx
|
|
stopCancellationForwarder := func() {}
|
|
if input.LongRunning {
|
|
// Long-running training deliberately ignores ordinary workflow/node deadlines,
|
|
// but it must still detach promptly when the worker shuts down or loses its
|
|
// execution lease. The Kubernetes Job is left in place for the next worker to
|
|
// reconcile; only short-lived jobs are deleted on cancellation below.
|
|
detachedCtx, cancelDetached := context.WithCancel(context.Background())
|
|
forwarderDone := make(chan struct{})
|
|
go func() {
|
|
select {
|
|
case <-ctx.Done():
|
|
if errors.Is(ctx.Err(), context.Canceled) {
|
|
cancelDetached()
|
|
}
|
|
case <-forwarderDone:
|
|
}
|
|
}()
|
|
waitParentCtx = detachedCtx
|
|
stopCancellationForwarder = func() {
|
|
close(forwarderDone)
|
|
cancelDetached()
|
|
}
|
|
}
|
|
defer stopCancellationForwarder()
|
|
waitCtx, cancel := context.WithDeadline(waitParentCtx, waitDeadlineForLaunch(waitParentCtx, input, time.Now()))
|
|
defer cancel()
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
|
|
// Platform-side GPU sampler for custom-container operators. Builtin
|
|
// operator-runner pods self-report resourceSeries via main.py's
|
|
// ResourceSampler, so they are skipped (resolved Image == OperatorRunnerImage).
|
|
// Custom containers bypass main.py and would otherwise show GPU=0 on the
|
|
// resource monitor; the launcher samples nvidia-smi inside the pod instead.
|
|
samplerCtx, samplerCancel := context.WithCancel(context.Background())
|
|
defer samplerCancel()
|
|
var gpuSamples []GPUSample
|
|
var sampleMu sync.Mutex
|
|
samplerDone := make(chan struct{})
|
|
shouldSample := launcher.restConfig != nil &&
|
|
input.OperatorRunnerImage != "" &&
|
|
input.Image != input.OperatorRunnerImage
|
|
if shouldSample {
|
|
go func() {
|
|
defer close(samplerDone)
|
|
probeTicker := time.NewTicker(2 * time.Second)
|
|
defer probeTicker.Stop()
|
|
sampleTicker := time.NewTicker(podGPUSampleInterval())
|
|
defer sampleTicker.Stop()
|
|
podName := ""
|
|
running := false
|
|
for {
|
|
select {
|
|
case <-samplerCtx.Done():
|
|
return
|
|
case <-probeTicker.C:
|
|
if podName == "" {
|
|
podName = launcher.findJobPodName(samplerCtx, jobName)
|
|
if podName == "" {
|
|
continue
|
|
}
|
|
}
|
|
pod, err := launcher.clientset.CoreV1().Pods(launcher.namespace).Get(samplerCtx, podName, metav1.GetOptions{})
|
|
if err != nil {
|
|
continue
|
|
}
|
|
running = pod.Status.Phase == corev1.PodRunning
|
|
case <-sampleTicker.C:
|
|
if !running || podName == "" {
|
|
continue
|
|
}
|
|
sampleCallCtx, cancelCall := context.WithTimeout(samplerCtx, 5*time.Second)
|
|
sample, err := launcher.samplePodGPU(sampleCallCtx, podName)
|
|
cancelCall()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
sampleMu.Lock()
|
|
gpuSamples = append(gpuSamples, sample)
|
|
if len(gpuSamples) > 7200 {
|
|
gpuSamples = gpuSamples[len(gpuSamples)-7200:]
|
|
}
|
|
sampleMu.Unlock()
|
|
}
|
|
}
|
|
}()
|
|
} else {
|
|
close(samplerDone)
|
|
}
|
|
stopSampler := func() []GPUSample {
|
|
samplerCancel()
|
|
<-samplerDone
|
|
sampleMu.Lock()
|
|
defer sampleMu.Unlock()
|
|
return append([]GPUSample(nil), gpuSamples...)
|
|
}
|
|
|
|
checkStatus := func() (*LaunchResult, error) {
|
|
status, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Get(waitCtx, jobName, metav1.GetOptions{})
|
|
if err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
return nil, &JobNotFoundError{JobName: jobName}
|
|
}
|
|
return nil, err
|
|
}
|
|
if status.Status.Succeeded > 0 {
|
|
output, err := launcher.readJobOutput(waitCtx, jobName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &LaunchResult{Reference: reference, Output: output}, nil
|
|
}
|
|
if status.Status.Failed > 0 {
|
|
output, outputErr := launcher.readJobOutput(waitCtx, jobName)
|
|
diagnostics := strings.TrimSpace(launcher.readJobDiagnostics(waitCtx, status, output, outputErr))
|
|
if diagnostics == "" {
|
|
return nil, fmt.Errorf("kubernetes job %s failed", jobName)
|
|
}
|
|
return nil, fmt.Errorf("kubernetes job %s failed:\n%s", jobName, diagnostics)
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
for {
|
|
result, err := checkStatus()
|
|
if result != nil || err != nil {
|
|
if result != nil {
|
|
result.ResourceSeries = stopSampler()
|
|
} else {
|
|
stopSampler()
|
|
}
|
|
return result, err
|
|
}
|
|
select {
|
|
case <-waitCtx.Done():
|
|
stopSampler()
|
|
// A cancelled worker context is not proof that the business Job should
|
|
// be deleted. Worker rollouts and lease hand-offs cancel this waiter too;
|
|
// the surviving Job is intentionally left for reconciliation. Explicit
|
|
// user stops and terminal Run cleanup call TerminateRun separately.
|
|
if errors.Is(waitCtx.Err(), context.DeadlineExceeded) {
|
|
return nil, fmt.Errorf("timed out waiting for kubernetes job %s", jobName)
|
|
}
|
|
return nil, waitCtx.Err()
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
var terminalPendingReasons = map[string]struct{}{
|
|
"CreateContainerConfigError": {},
|
|
"CreateContainerError": {},
|
|
"ErrImagePull": {},
|
|
"ImagePullBackOff": {},
|
|
"InvalidImageName": {},
|
|
"Unschedulable": {},
|
|
}
|
|
|
|
func terminalPendingReason(reason string) bool {
|
|
_, ok := terminalPendingReasons[strings.TrimSpace(reason)]
|
|
return ok
|
|
}
|
|
|
|
func podBlockingState(pod corev1.Pod) (string, string) {
|
|
for _, condition := range pod.Status.Conditions {
|
|
if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && terminalPendingReason(condition.Reason) {
|
|
return condition.Reason, condition.Message
|
|
}
|
|
}
|
|
for _, status := range append(append([]corev1.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
|
|
if status.State.Waiting != nil && terminalPendingReason(status.State.Waiting.Reason) {
|
|
return status.State.Waiting.Reason, status.State.Waiting.Message
|
|
}
|
|
}
|
|
if terminalPendingReason(pod.Status.Reason) {
|
|
return pod.Status.Reason, pod.Status.Message
|
|
}
|
|
return "", ""
|
|
}
|
|
|
|
func (launcher *Launcher) inspectBlockingState(ctx context.Context, jobName string) (string, string, error) {
|
|
pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName})
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
for _, pod := range pods.Items {
|
|
if reason, message := podBlockingState(pod); reason != "" {
|
|
return reason, message, nil
|
|
}
|
|
}
|
|
return "", "", nil
|
|
}
|
|
|
|
func (launcher *Launcher) Inspect(ctx context.Context, runID string, nodeID string) (*JobInspection, error) {
|
|
selectorParts := []string{"runId=" + strings.TrimSpace(runID)}
|
|
if trimmedNodeID := strings.TrimSpace(nodeID); trimmedNodeID != "" {
|
|
selectorParts = append(selectorParts, "nodeId="+trimmedNodeID)
|
|
}
|
|
jobs, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).List(ctx, metav1.ListOptions{
|
|
LabelSelector: strings.Join(selectorParts, ","),
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(jobs.Items) == 0 {
|
|
return &JobInspection{State: JobStateMissing}, nil
|
|
}
|
|
sort.Slice(jobs.Items, func(i int, j int) bool {
|
|
return jobs.Items[i].CreationTimestamp.Time.Before(jobs.Items[j].CreationTimestamp.Time)
|
|
})
|
|
job := jobs.Items[len(jobs.Items)-1]
|
|
reference := &JobReference{JobName: job.Name, Namespace: launcher.namespace}
|
|
createdAt := orphanRuntimeCreatedAt(job.CreationTimestamp)
|
|
image := ""
|
|
for _, container := range job.Spec.Template.Spec.Containers {
|
|
if image == "" || container.Name == "operator-runner" {
|
|
image = container.Image
|
|
}
|
|
if container.Name == "operator-runner" {
|
|
break
|
|
}
|
|
}
|
|
switch {
|
|
case job.Status.Succeeded > 0:
|
|
output, err := launcher.readJobOutputNow(ctx, job.Name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &JobInspection{Reference: reference, Image: image, State: JobStateSucceeded, Output: output, CreatedAt: createdAt}, nil
|
|
case job.Status.Failed > 0:
|
|
output, outputErr := launcher.readJobOutputNow(ctx, job.Name)
|
|
diagnostics := launcher.readJobDiagnostics(ctx, &job, output, outputErr)
|
|
if strings.TrimSpace(diagnostics) != "" {
|
|
output = diagnostics
|
|
}
|
|
return &JobInspection{Reference: reference, Image: image, State: JobStateFailed, Output: output, CreatedAt: createdAt}, nil
|
|
}
|
|
reason, message, err := launcher.inspectBlockingState(ctx, job.Name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if reason != "" {
|
|
return &JobInspection{Reference: reference, Image: image, State: JobStatePending, CreatedAt: createdAt, BlockingReason: reason, BlockingMessage: message}, nil
|
|
}
|
|
if job.Status.Active > 0 {
|
|
return &JobInspection{Reference: reference, Image: image, State: JobStateActive, CreatedAt: createdAt}, nil
|
|
}
|
|
return &JobInspection{Reference: reference, Image: image, State: JobStatePending, CreatedAt: createdAt}, nil
|
|
}
|