🐛 Bound lightweight code manifests before run creation

This commit is contained in:
eust-w 2026-07-13 23:49:36 +08:00
parent 9b2b9def45
commit ce3a54c41d
6 changed files with 180 additions and 16 deletions

View File

@ -592,6 +592,7 @@ type LightweightOperatorCodeManifest struct {
SHA256 string `json:"sha256" bson:"sha256"`
GeneratedAt string `json:"generatedAt" bson:"generatedAt"`
Files []LightweightOperatorCodeFileManifest `json:"files" bson:"files"`
ExcludedPaths []string `json:"excludedPaths,omitempty" bson:"excludedPaths,omitempty"`
}
type RunMetadata struct {

View File

@ -12,6 +12,7 @@ import (
"regexp"
"sort"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
@ -23,8 +24,28 @@ const (
DefaultMountPath = "/opt/droboticflow/lightweight/operator"
ContainerArgsConfiguredKey = "containerArgsConfigured"
ContainerCommandConfiguredKey = "containerCommandConfigured"
maxCodeManifestFiles = 5000
maxCodeManifestTotalBytes = int64(128 << 20)
maxCodeManifestFileBytes = int64(32 << 20)
maxCodeManifestDuration = 15 * time.Second
codeManifestHashWorkers = 8
)
var manifestCacheDirectoryNames = map[string]struct{}{
".git": {}, ".mypy_cache": {}, ".pytest_cache": {}, ".venv": {},
"__pycache__": {}, "node_modules": {}, "venv": {},
}
var manifestNonRuntimeDirectoryNames = map[string]struct{}{
".github": {}, "docs": {}, "examples": {}, "tests": {},
}
var manifestTopLevelRuntimeDirectoryNames = map[string]struct{}{
"arrow": {}, "artifacts": {}, "checkpoints": {}, "datasets": {},
"failed-checkpoints": {}, "logs": {}, "model": {}, "outputs": {},
"results": {}, "resume-from": {}, "tensorboard": {},
}
type Config struct {
Enabled bool
CodeSubPath string
@ -280,7 +301,9 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
}
files := make([]domain.LightweightOperatorCodeFileManifest, 0)
excludedPaths := make([]string, 0)
totalBytes := int64(0)
startedAt := time.Now()
err = filepath.WalkDir(codePath, func(itemPath string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
@ -288,6 +311,18 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
if itemPath == codePath {
return nil
}
rel, err := filepath.Rel(codePath, itemPath)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if entry.IsDir() && excludeCodeManifestDirectory(rel, entry.Name()) {
excludedPaths = append(excludedPaths, rel)
return filepath.SkipDir
}
if time.Since(startedAt) > maxCodeManifestDuration {
return fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration)
}
info, err := entry.Info()
if err != nil {
return err
@ -305,14 +340,14 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
if !info.Mode().IsRegular() {
return nil
}
rel, err := filepath.Rel(codePath, itemPath)
if err != nil {
return err
if len(files) >= maxCodeManifestFiles {
return fmt.Errorf("code manifest exceeds %d source files; move generated data outside the code directory", maxCodeManifestFiles)
}
rel = filepath.ToSlash(rel)
sum, err := fileSHA256(itemPath)
if err != nil {
return err
if info.Size() > maxCodeManifestFileBytes {
return fmt.Errorf("source file %s exceeds %d MiB; store models and generated data outside the code directory", rel, maxCodeManifestFileBytes>>20)
}
if totalBytes+info.Size() > maxCodeManifestTotalBytes {
return fmt.Errorf("code manifest exceeds %d MiB; move models and generated data outside the code directory", maxCodeManifestTotalBytes>>20)
}
totalBytes += info.Size()
files = append(files, domain.LightweightOperatorCodeFileManifest{
@ -320,7 +355,6 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
SizeBytes: info.Size(),
Mode: info.Mode().String(),
ModTime: info.ModTime().UTC().Format(time.RFC3339Nano),
SHA256: sum,
})
return nil
})
@ -330,6 +364,9 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
if len(files) == 0 {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory has no regular files: %s", nodeLabel(node), codePath)
}
if err := hashCodeManifestFiles(codePath, files, startedAt); err != nil {
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory cannot be scanned: %w", nodeLabel(node), err)
}
sort.Slice(files, func(left int, right int) bool {
return files[left].Path < files[right].Path
})
@ -345,15 +382,64 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
WorkspacePath: filepath.ToSlash(codePath),
// Code lives on the shared workspace PVC and is writable by default.
// Read-only behavior is opt-in through codeReadOnly.
ReadOnly: config.CodeReadOnly,
FileCount: len(files),
TotalBytes: totalBytes,
SHA256: hex.EncodeToString(aggregate.Sum(nil)),
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
Files: files,
ReadOnly: config.CodeReadOnly,
FileCount: len(files),
TotalBytes: totalBytes,
SHA256: hex.EncodeToString(aggregate.Sum(nil)),
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
Files: files,
ExcludedPaths: excludedPaths,
}, nil
}
func hashCodeManifestFiles(codePath string, files []domain.LightweightOperatorCodeFileManifest, startedAt time.Time) error {
workerCount := min(codeManifestHashWorkers, len(files))
jobs := make(chan int)
var workers sync.WaitGroup
var errorOnce sync.Once
var firstError error
for range workerCount {
workers.Add(1)
go func() {
defer workers.Done()
for index := range jobs {
if time.Since(startedAt) > maxCodeManifestDuration {
errorOnce.Do(func() {
firstError = fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration)
})
continue
}
sum, err := fileSHA256(filepath.Join(codePath, filepath.FromSlash(files[index].Path)))
if err != nil {
errorOnce.Do(func() { firstError = err })
continue
}
files[index].SHA256 = sum
}
}()
}
for index := range files {
jobs <- index
}
close(jobs)
workers.Wait()
return firstError
}
func excludeCodeManifestDirectory(relativePath string, name string) bool {
if _, excluded := manifestCacheDirectoryNames[name]; excluded {
return true
}
if _, excluded := manifestNonRuntimeDirectoryNames[name]; excluded {
return true
}
if strings.Contains(relativePath, "/") {
return false
}
_, excluded := manifestTopLevelRuntimeDirectoryNames[name]
return excluded
}
func MergeRunMetadata(run *domain.RunRecord, manifests map[string]domain.LightweightOperatorCodeManifest) {
if run == nil || len(manifests) == 0 {
return

View File

@ -1,9 +1,13 @@
package lightweight
import (
"os"
"path/filepath"
"strings"
"testing"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
)
@ -72,3 +76,62 @@ func TestValidateMountsRejectsConflicts(t *testing.T) {
})
}
}
func TestGenerateCodeManifestSkipsRuntimeAndCacheDirectories(t *testing.T) {
workspaceRoot := t.TempDir()
codeDir := filepath.Join(workspaceRoot, "lightweight-operators", "demo")
for _, directory := range []string{"checkpoints", "resume-from", "model", filepath.Join("pkg", "__pycache__"), filepath.Join("pkg", "tests")} {
if err := os.MkdirAll(filepath.Join(codeDir, directory), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", directory, err)
}
if err := os.WriteFile(filepath.Join(codeDir, directory, "ignored.bin"), []byte("runtime"), 0o644); err != nil {
t.Fatalf("write ignored file: %v", err)
}
}
if err := os.WriteFile(filepath.Join(codeDir, "operator.py"), []byte("print('ok')\n"), 0o644); err != nil {
t.Fatalf("write operator: %v", err)
}
manifest, err := GenerateCodeManifest(workspaceRoot, domain.WorkflowNode{
ID: "node-1",
Data: domain.WorkflowNodeData{Label: "demo"},
}, Config{CodeSubPath: "lightweight-operators/demo", MountPath: "/opt/operator"})
if err != nil {
t.Fatalf("generate manifest: %v", err)
}
if manifest.FileCount != 1 || len(manifest.Files) != 1 || manifest.Files[0].Path != "operator.py" {
t.Fatalf("unexpected included files: %#v", manifest.Files)
}
excluded := map[string]bool{}
for _, path := range manifest.ExcludedPaths {
excluded[path] = true
}
for _, expected := range []string{"checkpoints", "resume-from", "model", "pkg/__pycache__", "pkg/tests"} {
if !excluded[expected] {
t.Fatalf("expected excluded path %q in %#v", expected, manifest.ExcludedPaths)
}
}
}
func TestGenerateCodeManifestRejectsOversizedSourceFileBeforeHashing(t *testing.T) {
workspaceRoot := t.TempDir()
codeDir := filepath.Join(workspaceRoot, "lightweight-operators", "demo")
if err := os.MkdirAll(codeDir, 0o755); err != nil {
t.Fatalf("mkdir code dir: %v", err)
}
largePath := filepath.Join(codeDir, "weights.bin")
if err := os.WriteFile(largePath, nil, 0o644); err != nil {
t.Fatalf("create large file: %v", err)
}
if err := os.Truncate(largePath, maxCodeManifestFileBytes+1); err != nil {
t.Fatalf("truncate large file: %v", err)
}
_, err := GenerateCodeManifest(workspaceRoot, domain.WorkflowNode{
ID: "node-1",
Data: domain.WorkflowNodeData{Label: "demo"},
}, Config{CodeSubPath: "lightweight-operators/demo", MountPath: "/opt/operator"})
if err == nil || !strings.Contains(err.Error(), "exceeds 32 MiB") {
t.Fatalf("expected bounded manifest error, got %v", err)
}
}

View File

@ -43,7 +43,12 @@ export function getWorkflowRunActionState({
return { disabled: true, label: "模板不可运行", reason: "template", message: "" };
}
if (isExecuting) {
return { disabled: true, label: "正在启动...", reason: "starting", message: "" };
return {
disabled: true,
label: "正在创建 Run…",
reason: "starting",
message: "正在准备轻量算子代码清单并创建 Run,完成后会自动显示 Run 编号。",
};
}
const lock = getWorkflowEditorLockState({ isReadOnly, activeRunStatus, isRunContextLoading });
if (lock.reason === "running") {

View File

@ -3281,7 +3281,7 @@ export default function WorkflowEditor() {
: editorLockState.reason === "checkingRun"
? "正在确认运行状态"
: editorLockState.reason === "starting"
? "正在保存并启动"
? "正在创建 Run"
: "只读模式"}
</span>
)}

View File

@ -175,9 +175,18 @@ test("editor mutations lock during the save-and-start window", () => {
isRunContextLoading: false,
isExecuting: true,
});
const action = getWorkflowRunActionState({
isTemplate: false,
isReadOnly: false,
activeRunStatus: "",
isRunContextLoading: false,
isExecuting: true,
});
assert.equal(lock.locked, true);
assert.equal(lock.reason, "starting");
assert.equal(action.label, "正在创建 Run…");
assert.match(action.message, /代码清单/);
});
test("node env rows include inherited workflow envs but keep node overrides first", () => {