548 lines
18 KiB
Go

package lightweight
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
)
var pvcNamePattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$`)
const (
DefaultMountPath = "/opt/droboticflow/lightweight/operator"
ContainerArgsConfiguredKey = "containerArgsConfigured"
ContainerCommandConfiguredKey = "containerCommandConfigured"
maxCodeManifestFiles = 5000
maxCodeManifestTotalBytes = int64(128 << 20)
maxCodeManifestFileBytes = int64(32 << 20)
maxCodeManifestDuration = 15 * time.Second
codeManifestHashWorkers = 8
)
var manifestCacheDirectoryNames = map[string]struct{}{
".git": {}, ".mypy_cache": {}, ".pytest_cache": {}, ".venv": {},
"__pycache__": {}, "node_modules": {}, "venv": {},
}
var manifestNonRuntimeDirectoryNames = map[string]struct{}{
".github": {}, "docs": {}, "examples": {}, "tests": {},
}
var manifestTopLevelRuntimeDirectoryNames = map[string]struct{}{
"arrow": {}, "artifacts": {}, "checkpoints": {}, "datasets": {},
"failed-checkpoints": {}, "logs": {}, "model": {}, "outputs": {},
"results": {}, "resume-from": {}, "tensorboard": {},
}
type Config struct {
Enabled bool
CodeSubPath string
MountPath string
CodeReadOnly bool
ContainerArgsConfigured bool
ContainerCommandConfigured bool
Mounts []Mount
}
type Mount struct {
Name string `json:"name" bson:"name"`
PVCName string `json:"pvcName" bson:"pvcName"`
SubPath string `json:"subPath,omitempty" bson:"subPath,omitempty"`
MountPath string `json:"mountPath" bson:"mountPath"`
// ReadOnly is opt-in; omitted PVC mounts are writable by default.
ReadOnly bool `json:"readOnly" bson:"readOnly"`
IDEEnabled bool `json:"ideEnabled" bson:"ideEnabled"`
}
type RuntimeConfig struct {
NodeID string
NodeLabel string
Image string
CodeSubPath string
MountPath string
Command []string
Args []string
ArgsConfigured bool
WorkspaceRoot string
Mounts []Mount
}
func NormalizeContract(contract map[string]any, commandConfigured bool, argsConfigured bool) map[string]any {
next := map[string]any{}
for key, value := range contract {
next[key] = value
}
raw, _ := next["lightweightOperator"].(map[string]any)
lightweight := map[string]any{}
for key, value := range raw {
lightweight[key] = value
}
lightweight["enabled"] = true
if commandConfigured {
lightweight[ContainerCommandConfiguredKey] = true
}
if argsConfigured {
lightweight[ContainerArgsConfiguredKey] = true
}
if codeSubPath := NormalizeCodeSubPath(stringMapValue(lightweight, "codeSubPath")); codeSubPath != "" {
lightweight["codeSubPath"] = codeSubPath
}
if mountPath := strings.TrimSpace(stringMapValue(lightweight, "mountPath")); mountPath != "" {
lightweight["mountPath"] = mountPath
}
codeReadOnly, _ := lightweight["codeReadOnly"].(bool)
lightweight["codeReadOnly"] = codeReadOnly
lightweight["mounts"] = normalizeMounts(lightweight["mounts"])
next["lightweightOperator"] = lightweight
return next
}
func ConfigFromContract(contract map[string]any) Config {
raw, ok := contract["lightweightOperator"].(map[string]any)
if !ok {
return Config{}
}
enabled, _ := raw["enabled"].(bool)
codeSubPath := NormalizeCodeSubPath(stringMapValue(raw, "codeSubPath"))
mountPath := strings.TrimSpace(stringMapValue(raw, "mountPath"))
codeReadOnly, _ := raw["codeReadOnly"].(bool)
argsConfigured, _ := raw[ContainerArgsConfiguredKey].(bool)
commandConfigured, _ := raw[ContainerCommandConfiguredKey].(bool)
return Config{
Enabled: enabled || codeSubPath != "" || mountPath != "",
CodeSubPath: codeSubPath,
MountPath: mountPath,
CodeReadOnly: codeReadOnly,
ContainerArgsConfigured: argsConfigured,
ContainerCommandConfigured: commandConfigured,
Mounts: normalizeMounts(raw["mounts"]),
}
}
func NormalizeCodeSubPath(value string) string {
return strings.Trim(strings.TrimSpace(value), "/")
}
func ValidateRuntimeConfig(config RuntimeConfig) error {
if strings.TrimSpace(config.Image) == "" {
return lightweightConfigError(config, "base image is required")
}
if strings.TrimSpace(config.CodeSubPath) == "" {
return lightweightConfigError(config, "codeSubPath is required")
}
if err := validateVolumeSubPath(config.CodeSubPath); err != nil {
return lightweightConfigError(config, "codeSubPath is invalid: "+err.Error())
}
mountPath := strings.TrimSpace(config.MountPath)
if mountPath == "" {
return lightweightConfigError(config, "mountPath is required")
}
if !strings.HasPrefix(mountPath, "/") {
return lightweightConfigError(config, "mountPath must be absolute")
}
workspaceRoot := cleanAbsPath(config.WorkspaceRoot, "/workspace")
if pathHasPrefix(mountPath, filepath.ToSlash(filepath.Join(workspaceRoot, "system", "runs"))) {
return lightweightConfigError(config, "mountPath must not overlap platform run output paths")
}
if len(normalizeStringSlice(config.Command)) == 0 {
return lightweightConfigError(config, "containerCommand is required")
}
if len(config.Args) == 0 && !config.ArgsConfigured {
return lightweightConfigError(config, "containerArgs must be explicitly configured; use [] when no args are needed")
}
if err := ValidateMounts(config.Mounts, mountPath, workspaceRoot); err != nil {
return lightweightConfigError(config, err.Error())
}
return nil
}
func ValidateMounts(mounts []Mount, codeMountPath string, workspaceRoot string) error {
names := map[string]struct{}{}
paths := []string{strings.TrimSpace(codeMountPath)}
for index, item := range mounts {
name := strings.TrimSpace(item.Name)
if name == "" {
return fmt.Errorf("mounts[%d].name is required", index)
}
if name == "code" {
return fmt.Errorf("mounts[%d].name is reserved: code", index)
}
if _, exists := names[name]; exists {
return fmt.Errorf("mounts[%d].name must be unique: %s", index, name)
}
names[name] = struct{}{}
pvcName := strings.TrimSpace(item.PVCName)
if pvcName == "" {
return fmt.Errorf("mounts[%d].pvcName is required", index)
}
if len(pvcName) > 253 || !pvcNamePattern.MatchString(pvcName) {
return fmt.Errorf("mounts[%d].pvcName must be a valid Kubernetes PVC name", index)
}
if item.SubPath != "" {
if err := validateVolumeSubPath(item.SubPath); err != nil {
return fmt.Errorf("mounts[%d].subPath is invalid: %w", index, err)
}
}
mountPath := strings.TrimSpace(item.MountPath)
if mountPath == "" || !strings.HasPrefix(mountPath, "/") {
return fmt.Errorf("mounts[%d].mountPath must be an absolute path", index)
}
if pathHasPrefix(mountPath, filepath.ToSlash(filepath.Join(workspaceRoot, "system", "runs"))) {
return fmt.Errorf("mounts[%d].mountPath must not overlap platform run output paths", index)
}
for _, existing := range paths {
if existing != "" && (pathHasPrefix(mountPath, existing) || pathHasPrefix(existing, mountPath)) {
return fmt.Errorf("mounts[%d].mountPath overlaps another mount: %s", index, mountPath)
}
}
paths = append(paths, mountPath)
}
return nil
}
func normalizeMounts(value any) []Mount {
if value == nil {
return []Mount{}
}
raw, err := json.Marshal(value)
if err != nil {
return []Mount{}
}
var decoded []Mount
if err := json.Unmarshal(raw, &decoded); err != nil {
return []Mount{}
}
result := make([]Mount, 0, len(decoded))
for _, mount := range decoded {
mount.Name = strings.TrimSpace(mount.Name)
mount.PVCName = strings.TrimSpace(mount.PVCName)
mount.SubPath = NormalizeCodeSubPath(mount.SubPath)
mount.MountPath = strings.TrimSpace(mount.MountPath)
result = append(result, mount)
}
return result
}
func GenerateWorkflowCodeManifests(workspaceRoot string, workflow domain.WorkflowRecord) (map[string]domain.LightweightOperatorCodeManifest, error) {
workspaceRoot = cleanAbsPath(workspaceRoot, "/workspace")
items := map[string]domain.LightweightOperatorCodeManifest{}
for _, node := range workflow.Nodes {
config := ConfigFromContract(node.Data.RuntimeContract)
if !config.Enabled {
continue
}
if err := ValidateRuntimeConfig(RuntimeConfig{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Image: node.Data.Image,
CodeSubPath: config.CodeSubPath,
MountPath: config.MountPath,
Command: node.Data.ContainerCommand,
Args: node.Data.ContainerArgs,
ArgsConfigured: config.ContainerArgsConfigured,
WorkspaceRoot: workspaceRoot,
}); err != nil {
return nil, err
}
manifest, err := GenerateCodeManifest(workspaceRoot, node, config)
if err != nil {
return nil, err
}
items[node.ID] = manifest
}
if len(items) == 0 {
return nil, nil
}
return items, nil
}
func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config Config) (domain.LightweightOperatorCodeManifest, error) {
codeSubPath := NormalizeCodeSubPath(config.CodeSubPath)
if err := validateVolumeSubPath(codeSubPath); err != nil {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s codeSubPath is invalid: %w", nodeLabel(node), err)
}
workspaceRoot = cleanAbsPath(workspaceRoot, "/workspace")
rootReal, err := filepath.EvalSymlinks(workspaceRoot)
if err != nil {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s workspace root is not readable: %w", nodeLabel(node), err)
}
codePath := filepath.Join(workspaceRoot, filepath.FromSlash(codeSubPath))
codeInfo, err := os.Lstat(codePath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory does not exist: %s", nodeLabel(node), codePath)
}
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory is not readable: %w", nodeLabel(node), err)
}
if codeInfo.Mode()&os.ModeSymlink != 0 {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory must not be a symlink: %s", nodeLabel(node), codePath)
}
if !codeInfo.IsDir() {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s codeSubPath must be a directory: %s", nodeLabel(node), codePath)
}
codeReal, err := filepath.EvalSymlinks(codePath)
if err != nil {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory is not readable: %w", nodeLabel(node), err)
}
if !pathInside(codeReal, rootReal) {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory escapes workspace root: %s", nodeLabel(node), codePath)
}
files := make([]domain.LightweightOperatorCodeFileManifest, 0)
excludedPaths := make([]string, 0)
totalBytes := int64(0)
startedAt := time.Now()
err = filepath.WalkDir(codePath, func(itemPath string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if itemPath == codePath {
return nil
}
rel, err := filepath.Rel(codePath, itemPath)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if entry.IsDir() && excludeCodeManifestDirectory(rel, entry.Name()) {
excludedPaths = append(excludedPaths, rel)
return filepath.SkipDir
}
if time.Since(startedAt) > maxCodeManifestDuration {
return fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration)
}
info, err := entry.Info()
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
target, err := filepath.EvalSymlinks(itemPath)
if err != nil {
return fmt.Errorf("symlink is not readable: %s: %w", itemPath, err)
}
if !pathInside(target, rootReal) {
return fmt.Errorf("symlink escapes workspace root: %s", itemPath)
}
return nil
}
if !info.Mode().IsRegular() {
return nil
}
if len(files) >= maxCodeManifestFiles {
return fmt.Errorf("code manifest exceeds %d source files; move generated data outside the code directory", maxCodeManifestFiles)
}
if info.Size() > maxCodeManifestFileBytes {
return fmt.Errorf("source file %s exceeds %d MiB; store models and generated data outside the code directory", rel, maxCodeManifestFileBytes>>20)
}
if totalBytes+info.Size() > maxCodeManifestTotalBytes {
return fmt.Errorf("code manifest exceeds %d MiB; move models and generated data outside the code directory", maxCodeManifestTotalBytes>>20)
}
totalBytes += info.Size()
files = append(files, domain.LightweightOperatorCodeFileManifest{
Path: rel,
SizeBytes: info.Size(),
Mode: info.Mode().String(),
ModTime: info.ModTime().UTC().Format(time.RFC3339Nano),
})
return nil
})
if err != nil {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory cannot be scanned: %w", nodeLabel(node), err)
}
if len(files) == 0 {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory has no regular files: %s", nodeLabel(node), codePath)
}
if err := hashCodeManifestFiles(codePath, files, startedAt); err != nil {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory cannot be scanned: %w", nodeLabel(node), err)
}
sort.Slice(files, func(left int, right int) bool {
return files[left].Path < files[right].Path
})
aggregate := sha256.New()
for _, file := range files {
_, _ = fmt.Fprintf(aggregate, "%s\x00%d\x00%s\x00", file.Path, file.SizeBytes, file.SHA256)
}
return domain.LightweightOperatorCodeManifest{
NodeID: node.ID,
NodeLabel: node.Data.Label,
CodeSubPath: codeSubPath,
MountPath: config.MountPath,
WorkspacePath: filepath.ToSlash(codePath),
// Code lives on the shared workspace PVC and is writable by default.
// Read-only behavior is opt-in through codeReadOnly.
ReadOnly: config.CodeReadOnly,
FileCount: len(files),
TotalBytes: totalBytes,
SHA256: hex.EncodeToString(aggregate.Sum(nil)),
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
Files: files,
ExcludedPaths: excludedPaths,
}, nil
}
func hashCodeManifestFiles(codePath string, files []domain.LightweightOperatorCodeFileManifest, startedAt time.Time) error {
workerCount := min(codeManifestHashWorkers, len(files))
jobs := make(chan int)
var workers sync.WaitGroup
var errorOnce sync.Once
var firstError error
for range workerCount {
workers.Add(1)
go func() {
defer workers.Done()
for index := range jobs {
if time.Since(startedAt) > maxCodeManifestDuration {
errorOnce.Do(func() {
firstError = fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration)
})
continue
}
sum, err := fileSHA256(filepath.Join(codePath, filepath.FromSlash(files[index].Path)))
if err != nil {
errorOnce.Do(func() { firstError = err })
continue
}
files[index].SHA256 = sum
}
}()
}
for index := range files {
jobs <- index
}
close(jobs)
workers.Wait()
return firstError
}
func excludeCodeManifestDirectory(relativePath string, name string) bool {
if _, excluded := manifestCacheDirectoryNames[name]; excluded {
return true
}
if _, excluded := manifestNonRuntimeDirectoryNames[name]; excluded {
return true
}
if strings.Contains(relativePath, "/") {
return false
}
_, excluded := manifestTopLevelRuntimeDirectoryNames[name]
return excluded
}
func MergeRunMetadata(run *domain.RunRecord, manifests map[string]domain.LightweightOperatorCodeManifest) {
if run == nil || len(manifests) == 0 {
return
}
if run.Metadata == nil {
run.Metadata = &domain.RunMetadata{}
}
run.Metadata.LightweightOperatorCodeManifests = manifests
}
func stringMapValue(items map[string]any, key string) string {
value, ok := items[key]
if !ok || value == nil {
return ""
}
return fmt.Sprint(value)
}
func normalizeStringSlice(items []string) []string {
normalized := make([]string, 0, len(items))
for _, item := range items {
value := strings.TrimSpace(item)
if value != "" {
normalized = append(normalized, value)
}
}
return normalized
}
func lightweightConfigError(config RuntimeConfig, message string) error {
label := strings.TrimSpace(config.NodeLabel)
if label == "" {
label = config.NodeID
}
if label == "" {
label = "unknown"
}
return fmt.Errorf("lightweight operator %s is not runnable: %s", label, message)
}
func validateVolumeSubPath(value string) error {
trimmed := NormalizeCodeSubPath(value)
if trimmed == "" {
return fmt.Errorf("value is required")
}
cleaned := filepath.ToSlash(filepath.Clean(trimmed))
if cleaned == "." || strings.HasPrefix(cleaned, "../") || cleaned == ".." || strings.Contains(cleaned, "/../") {
return fmt.Errorf("must stay within the workspace PVC")
}
if strings.HasPrefix(cleaned, "/") {
return fmt.Errorf("must be relative")
}
return nil
}
func cleanAbsPath(value string, fallback string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
trimmed = fallback
}
if !strings.HasPrefix(trimmed, "/") {
trimmed = fallback
}
return strings.TrimRight(filepath.ToSlash(filepath.Clean(trimmed)), "/")
}
func pathHasPrefix(value string, prefix string) bool {
value = cleanAbsPath(value, value)
prefix = cleanAbsPath(prefix, prefix)
return value == prefix || strings.HasPrefix(value, prefix+"/")
}
func pathInside(value string, root string) bool {
value = filepath.Clean(value)
root = filepath.Clean(root)
rel, err := filepath.Rel(root, value)
if err != nil {
return false
}
return rel == "." || (!strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != "..")
}
func fileSHA256(path string) (string, error) {
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
hash := sha256.New()
if _, err := io.Copy(hash, file); err != nil {
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func nodeLabel(node domain.WorkflowNode) string {
label := strings.TrimSpace(node.Data.Label)
if label != "" {
return label
}
if strings.TrimSpace(node.ID) != "" {
return node.ID
}
return "unknown"
}