diff --git a/internal/domain/types.go b/internal/domain/types.go index e3930a0a..5abdedf9 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -592,6 +592,7 @@ type LightweightOperatorCodeManifest struct { SHA256 string `json:"sha256" bson:"sha256"` GeneratedAt string `json:"generatedAt" bson:"generatedAt"` Files []LightweightOperatorCodeFileManifest `json:"files" bson:"files"` + ExcludedPaths []string `json:"excludedPaths,omitempty" bson:"excludedPaths,omitempty"` } type RunMetadata struct { diff --git a/internal/lightweight/lightweight.go b/internal/lightweight/lightweight.go index a161c2ce..cfcb8599 100644 --- a/internal/lightweight/lightweight.go +++ b/internal/lightweight/lightweight.go @@ -12,6 +12,7 @@ import ( "regexp" "sort" "strings" + "sync" "time" "droboticflow/internal/domain" @@ -23,8 +24,28 @@ const ( DefaultMountPath = "/opt/droboticflow/lightweight/operator" ContainerArgsConfiguredKey = "containerArgsConfigured" ContainerCommandConfiguredKey = "containerCommandConfigured" + maxCodeManifestFiles = 5000 + maxCodeManifestTotalBytes = int64(128 << 20) + maxCodeManifestFileBytes = int64(32 << 20) + maxCodeManifestDuration = 15 * time.Second + codeManifestHashWorkers = 8 ) +var manifestCacheDirectoryNames = map[string]struct{}{ + ".git": {}, ".mypy_cache": {}, ".pytest_cache": {}, ".venv": {}, + "__pycache__": {}, "node_modules": {}, "venv": {}, +} + +var manifestNonRuntimeDirectoryNames = map[string]struct{}{ + ".github": {}, "docs": {}, "examples": {}, "tests": {}, +} + +var manifestTopLevelRuntimeDirectoryNames = map[string]struct{}{ + "arrow": {}, "artifacts": {}, "checkpoints": {}, "datasets": {}, + "failed-checkpoints": {}, "logs": {}, "model": {}, "outputs": {}, + "results": {}, "resume-from": {}, "tensorboard": {}, +} + type Config struct { Enabled bool CodeSubPath string @@ -280,7 +301,9 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config } files := make([]domain.LightweightOperatorCodeFileManifest, 0) + excludedPaths := make([]string, 0) totalBytes := int64(0) + startedAt := time.Now() err = filepath.WalkDir(codePath, func(itemPath string, entry os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr @@ -288,6 +311,18 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config if itemPath == codePath { return nil } + rel, err := filepath.Rel(codePath, itemPath) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if entry.IsDir() && excludeCodeManifestDirectory(rel, entry.Name()) { + excludedPaths = append(excludedPaths, rel) + return filepath.SkipDir + } + if time.Since(startedAt) > maxCodeManifestDuration { + return fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration) + } info, err := entry.Info() if err != nil { return err @@ -305,14 +340,14 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config if !info.Mode().IsRegular() { return nil } - rel, err := filepath.Rel(codePath, itemPath) - if err != nil { - return err + if len(files) >= maxCodeManifestFiles { + return fmt.Errorf("code manifest exceeds %d source files; move generated data outside the code directory", maxCodeManifestFiles) } - rel = filepath.ToSlash(rel) - sum, err := fileSHA256(itemPath) - if err != nil { - return err + if info.Size() > maxCodeManifestFileBytes { + return fmt.Errorf("source file %s exceeds %d MiB; store models and generated data outside the code directory", rel, maxCodeManifestFileBytes>>20) + } + if totalBytes+info.Size() > maxCodeManifestTotalBytes { + return fmt.Errorf("code manifest exceeds %d MiB; move models and generated data outside the code directory", maxCodeManifestTotalBytes>>20) } totalBytes += info.Size() files = append(files, domain.LightweightOperatorCodeFileManifest{ @@ -320,7 +355,6 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config SizeBytes: info.Size(), Mode: info.Mode().String(), ModTime: info.ModTime().UTC().Format(time.RFC3339Nano), - SHA256: sum, }) return nil }) @@ -330,6 +364,9 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config if len(files) == 0 { return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory has no regular files: %s", nodeLabel(node), codePath) } + if err := hashCodeManifestFiles(codePath, files, startedAt); err != nil { + return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory cannot be scanned: %w", nodeLabel(node), err) + } sort.Slice(files, func(left int, right int) bool { return files[left].Path < files[right].Path }) @@ -345,15 +382,64 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config WorkspacePath: filepath.ToSlash(codePath), // Code lives on the shared workspace PVC and is writable by default. // Read-only behavior is opt-in through codeReadOnly. - ReadOnly: config.CodeReadOnly, - FileCount: len(files), - TotalBytes: totalBytes, - SHA256: hex.EncodeToString(aggregate.Sum(nil)), - GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), - Files: files, + ReadOnly: config.CodeReadOnly, + FileCount: len(files), + TotalBytes: totalBytes, + SHA256: hex.EncodeToString(aggregate.Sum(nil)), + GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), + Files: files, + ExcludedPaths: excludedPaths, }, nil } +func hashCodeManifestFiles(codePath string, files []domain.LightweightOperatorCodeFileManifest, startedAt time.Time) error { + workerCount := min(codeManifestHashWorkers, len(files)) + jobs := make(chan int) + var workers sync.WaitGroup + var errorOnce sync.Once + var firstError error + for range workerCount { + workers.Add(1) + go func() { + defer workers.Done() + for index := range jobs { + if time.Since(startedAt) > maxCodeManifestDuration { + errorOnce.Do(func() { + firstError = fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration) + }) + continue + } + sum, err := fileSHA256(filepath.Join(codePath, filepath.FromSlash(files[index].Path))) + if err != nil { + errorOnce.Do(func() { firstError = err }) + continue + } + files[index].SHA256 = sum + } + }() + } + for index := range files { + jobs <- index + } + close(jobs) + workers.Wait() + return firstError +} + +func excludeCodeManifestDirectory(relativePath string, name string) bool { + if _, excluded := manifestCacheDirectoryNames[name]; excluded { + return true + } + if _, excluded := manifestNonRuntimeDirectoryNames[name]; excluded { + return true + } + if strings.Contains(relativePath, "/") { + return false + } + _, excluded := manifestTopLevelRuntimeDirectoryNames[name] + return excluded +} + func MergeRunMetadata(run *domain.RunRecord, manifests map[string]domain.LightweightOperatorCodeManifest) { if run == nil || len(manifests) == 0 { return diff --git a/internal/lightweight/lightweight_test.go b/internal/lightweight/lightweight_test.go index 5dccaab6..7bc6a7ff 100644 --- a/internal/lightweight/lightweight_test.go +++ b/internal/lightweight/lightweight_test.go @@ -1,9 +1,13 @@ package lightweight import ( + "os" + "path/filepath" "strings" "testing" + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" ) @@ -72,3 +76,62 @@ func TestValidateMountsRejectsConflicts(t *testing.T) { }) } } + +func TestGenerateCodeManifestSkipsRuntimeAndCacheDirectories(t *testing.T) { + workspaceRoot := t.TempDir() + codeDir := filepath.Join(workspaceRoot, "lightweight-operators", "demo") + for _, directory := range []string{"checkpoints", "resume-from", "model", filepath.Join("pkg", "__pycache__"), filepath.Join("pkg", "tests")} { + if err := os.MkdirAll(filepath.Join(codeDir, directory), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", directory, err) + } + if err := os.WriteFile(filepath.Join(codeDir, directory, "ignored.bin"), []byte("runtime"), 0o644); err != nil { + t.Fatalf("write ignored file: %v", err) + } + } + if err := os.WriteFile(filepath.Join(codeDir, "operator.py"), []byte("print('ok')\n"), 0o644); err != nil { + t.Fatalf("write operator: %v", err) + } + + manifest, err := GenerateCodeManifest(workspaceRoot, domain.WorkflowNode{ + ID: "node-1", + Data: domain.WorkflowNodeData{Label: "demo"}, + }, Config{CodeSubPath: "lightweight-operators/demo", MountPath: "/opt/operator"}) + if err != nil { + t.Fatalf("generate manifest: %v", err) + } + if manifest.FileCount != 1 || len(manifest.Files) != 1 || manifest.Files[0].Path != "operator.py" { + t.Fatalf("unexpected included files: %#v", manifest.Files) + } + excluded := map[string]bool{} + for _, path := range manifest.ExcludedPaths { + excluded[path] = true + } + for _, expected := range []string{"checkpoints", "resume-from", "model", "pkg/__pycache__", "pkg/tests"} { + if !excluded[expected] { + t.Fatalf("expected excluded path %q in %#v", expected, manifest.ExcludedPaths) + } + } +} + +func TestGenerateCodeManifestRejectsOversizedSourceFileBeforeHashing(t *testing.T) { + workspaceRoot := t.TempDir() + codeDir := filepath.Join(workspaceRoot, "lightweight-operators", "demo") + if err := os.MkdirAll(codeDir, 0o755); err != nil { + t.Fatalf("mkdir code dir: %v", err) + } + largePath := filepath.Join(codeDir, "weights.bin") + if err := os.WriteFile(largePath, nil, 0o644); err != nil { + t.Fatalf("create large file: %v", err) + } + if err := os.Truncate(largePath, maxCodeManifestFileBytes+1); err != nil { + t.Fatalf("truncate large file: %v", err) + } + + _, err := GenerateCodeManifest(workspaceRoot, domain.WorkflowNode{ + ID: "node-1", + Data: domain.WorkflowNodeData{Label: "demo"}, + }, Config{CodeSubPath: "lightweight-operators/demo", MountPath: "/opt/operator"}) + if err == nil || !strings.Contains(err.Error(), "exceeds 32 MiB") { + t.Fatalf("expected bounded manifest error, got %v", err) + } +} diff --git a/src/lib/workflow-editor.js b/src/lib/workflow-editor.js index 15e56e9d..4059654e 100644 --- a/src/lib/workflow-editor.js +++ b/src/lib/workflow-editor.js @@ -43,7 +43,12 @@ export function getWorkflowRunActionState({ return { disabled: true, label: "模板不可运行", reason: "template", message: "" }; } if (isExecuting) { - return { disabled: true, label: "正在启动...", reason: "starting", message: "" }; + return { + disabled: true, + label: "正在创建 Run…", + reason: "starting", + message: "正在准备轻量算子代码清单并创建 Run,完成后会自动显示 Run 编号。", + }; } const lock = getWorkflowEditorLockState({ isReadOnly, activeRunStatus, isRunContextLoading }); if (lock.reason === "running") { diff --git a/src/pages/WorkflowEditor.tsx b/src/pages/WorkflowEditor.tsx index 33d7cd11..855b4b6d 100644 --- a/src/pages/WorkflowEditor.tsx +++ b/src/pages/WorkflowEditor.tsx @@ -3281,7 +3281,7 @@ export default function WorkflowEditor() { : editorLockState.reason === "checkingRun" ? "正在确认运行状态" : editorLockState.reason === "starting" - ? "正在保存并启动" + ? "正在创建 Run" : "只读模式"} )} diff --git a/tests/frontend/workflow-editor.test.mjs b/tests/frontend/workflow-editor.test.mjs index f9ec7ef8..c7e4532d 100644 --- a/tests/frontend/workflow-editor.test.mjs +++ b/tests/frontend/workflow-editor.test.mjs @@ -175,9 +175,18 @@ test("editor mutations lock during the save-and-start window", () => { isRunContextLoading: false, isExecuting: true, }); + const action = getWorkflowRunActionState({ + isTemplate: false, + isReadOnly: false, + activeRunStatus: "", + isRunContextLoading: false, + isExecuting: true, + }); assert.equal(lock.locked, true); assert.equal(lock.reason, "starting"); + assert.equal(action.label, "正在创建 Run…"); + assert.match(action.message, /代码清单/); }); test("node env rows include inherited workflow envs but keep node overrides first", () => {