🐛 Bound lightweight code manifests before run creation
This commit is contained in:
parent
9b2b9def45
commit
ce3a54c41d
@ -592,6 +592,7 @@ type LightweightOperatorCodeManifest struct {
|
||||
SHA256 string `json:"sha256" bson:"sha256"`
|
||||
GeneratedAt string `json:"generatedAt" bson:"generatedAt"`
|
||||
Files []LightweightOperatorCodeFileManifest `json:"files" bson:"files"`
|
||||
ExcludedPaths []string `json:"excludedPaths,omitempty" bson:"excludedPaths,omitempty"`
|
||||
}
|
||||
|
||||
type RunMetadata struct {
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
@ -23,8 +24,28 @@ const (
|
||||
DefaultMountPath = "/opt/droboticflow/lightweight/operator"
|
||||
ContainerArgsConfiguredKey = "containerArgsConfigured"
|
||||
ContainerCommandConfiguredKey = "containerCommandConfigured"
|
||||
maxCodeManifestFiles = 5000
|
||||
maxCodeManifestTotalBytes = int64(128 << 20)
|
||||
maxCodeManifestFileBytes = int64(32 << 20)
|
||||
maxCodeManifestDuration = 15 * time.Second
|
||||
codeManifestHashWorkers = 8
|
||||
)
|
||||
|
||||
var manifestCacheDirectoryNames = map[string]struct{}{
|
||||
".git": {}, ".mypy_cache": {}, ".pytest_cache": {}, ".venv": {},
|
||||
"__pycache__": {}, "node_modules": {}, "venv": {},
|
||||
}
|
||||
|
||||
var manifestNonRuntimeDirectoryNames = map[string]struct{}{
|
||||
".github": {}, "docs": {}, "examples": {}, "tests": {},
|
||||
}
|
||||
|
||||
var manifestTopLevelRuntimeDirectoryNames = map[string]struct{}{
|
||||
"arrow": {}, "artifacts": {}, "checkpoints": {}, "datasets": {},
|
||||
"failed-checkpoints": {}, "logs": {}, "model": {}, "outputs": {},
|
||||
"results": {}, "resume-from": {}, "tensorboard": {},
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Enabled bool
|
||||
CodeSubPath string
|
||||
@ -280,7 +301,9 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
|
||||
}
|
||||
|
||||
files := make([]domain.LightweightOperatorCodeFileManifest, 0)
|
||||
excludedPaths := make([]string, 0)
|
||||
totalBytes := int64(0)
|
||||
startedAt := time.Now()
|
||||
err = filepath.WalkDir(codePath, func(itemPath string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
@ -288,6 +311,18 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
|
||||
if itemPath == codePath {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(codePath, itemPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if entry.IsDir() && excludeCodeManifestDirectory(rel, entry.Name()) {
|
||||
excludedPaths = append(excludedPaths, rel)
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if time.Since(startedAt) > maxCodeManifestDuration {
|
||||
return fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration)
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
@ -305,14 +340,14 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(codePath, itemPath)
|
||||
if err != nil {
|
||||
return err
|
||||
if len(files) >= maxCodeManifestFiles {
|
||||
return fmt.Errorf("code manifest exceeds %d source files; move generated data outside the code directory", maxCodeManifestFiles)
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
sum, err := fileSHA256(itemPath)
|
||||
if err != nil {
|
||||
return err
|
||||
if info.Size() > maxCodeManifestFileBytes {
|
||||
return fmt.Errorf("source file %s exceeds %d MiB; store models and generated data outside the code directory", rel, maxCodeManifestFileBytes>>20)
|
||||
}
|
||||
if totalBytes+info.Size() > maxCodeManifestTotalBytes {
|
||||
return fmt.Errorf("code manifest exceeds %d MiB; move models and generated data outside the code directory", maxCodeManifestTotalBytes>>20)
|
||||
}
|
||||
totalBytes += info.Size()
|
||||
files = append(files, domain.LightweightOperatorCodeFileManifest{
|
||||
@ -320,7 +355,6 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
|
||||
SizeBytes: info.Size(),
|
||||
Mode: info.Mode().String(),
|
||||
ModTime: info.ModTime().UTC().Format(time.RFC3339Nano),
|
||||
SHA256: sum,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
@ -330,6 +364,9 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
|
||||
if len(files) == 0 {
|
||||
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory has no regular files: %s", nodeLabel(node), codePath)
|
||||
}
|
||||
if err := hashCodeManifestFiles(codePath, files, startedAt); err != nil {
|
||||
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory cannot be scanned: %w", nodeLabel(node), err)
|
||||
}
|
||||
sort.Slice(files, func(left int, right int) bool {
|
||||
return files[left].Path < files[right].Path
|
||||
})
|
||||
@ -345,15 +382,64 @@ func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config
|
||||
WorkspacePath: filepath.ToSlash(codePath),
|
||||
// Code lives on the shared workspace PVC and is writable by default.
|
||||
// Read-only behavior is opt-in through codeReadOnly.
|
||||
ReadOnly: config.CodeReadOnly,
|
||||
FileCount: len(files),
|
||||
TotalBytes: totalBytes,
|
||||
SHA256: hex.EncodeToString(aggregate.Sum(nil)),
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Files: files,
|
||||
ReadOnly: config.CodeReadOnly,
|
||||
FileCount: len(files),
|
||||
TotalBytes: totalBytes,
|
||||
SHA256: hex.EncodeToString(aggregate.Sum(nil)),
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Files: files,
|
||||
ExcludedPaths: excludedPaths,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func hashCodeManifestFiles(codePath string, files []domain.LightweightOperatorCodeFileManifest, startedAt time.Time) error {
|
||||
workerCount := min(codeManifestHashWorkers, len(files))
|
||||
jobs := make(chan int)
|
||||
var workers sync.WaitGroup
|
||||
var errorOnce sync.Once
|
||||
var firstError error
|
||||
for range workerCount {
|
||||
workers.Add(1)
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for index := range jobs {
|
||||
if time.Since(startedAt) > maxCodeManifestDuration {
|
||||
errorOnce.Do(func() {
|
||||
firstError = fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration)
|
||||
})
|
||||
continue
|
||||
}
|
||||
sum, err := fileSHA256(filepath.Join(codePath, filepath.FromSlash(files[index].Path)))
|
||||
if err != nil {
|
||||
errorOnce.Do(func() { firstError = err })
|
||||
continue
|
||||
}
|
||||
files[index].SHA256 = sum
|
||||
}
|
||||
}()
|
||||
}
|
||||
for index := range files {
|
||||
jobs <- index
|
||||
}
|
||||
close(jobs)
|
||||
workers.Wait()
|
||||
return firstError
|
||||
}
|
||||
|
||||
func excludeCodeManifestDirectory(relativePath string, name string) bool {
|
||||
if _, excluded := manifestCacheDirectoryNames[name]; excluded {
|
||||
return true
|
||||
}
|
||||
if _, excluded := manifestNonRuntimeDirectoryNames[name]; excluded {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(relativePath, "/") {
|
||||
return false
|
||||
}
|
||||
_, excluded := manifestTopLevelRuntimeDirectoryNames[name]
|
||||
return excluded
|
||||
}
|
||||
|
||||
func MergeRunMetadata(run *domain.RunRecord, manifests map[string]domain.LightweightOperatorCodeManifest) {
|
||||
if run == nil || len(manifests) == 0 {
|
||||
return
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
package lightweight
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
@ -72,3 +76,62 @@ func TestValidateMountsRejectsConflicts(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCodeManifestSkipsRuntimeAndCacheDirectories(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
codeDir := filepath.Join(workspaceRoot, "lightweight-operators", "demo")
|
||||
for _, directory := range []string{"checkpoints", "resume-from", "model", filepath.Join("pkg", "__pycache__"), filepath.Join("pkg", "tests")} {
|
||||
if err := os.MkdirAll(filepath.Join(codeDir, directory), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", directory, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(codeDir, directory, "ignored.bin"), []byte("runtime"), 0o644); err != nil {
|
||||
t.Fatalf("write ignored file: %v", err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(codeDir, "operator.py"), []byte("print('ok')\n"), 0o644); err != nil {
|
||||
t.Fatalf("write operator: %v", err)
|
||||
}
|
||||
|
||||
manifest, err := GenerateCodeManifest(workspaceRoot, domain.WorkflowNode{
|
||||
ID: "node-1",
|
||||
Data: domain.WorkflowNodeData{Label: "demo"},
|
||||
}, Config{CodeSubPath: "lightweight-operators/demo", MountPath: "/opt/operator"})
|
||||
if err != nil {
|
||||
t.Fatalf("generate manifest: %v", err)
|
||||
}
|
||||
if manifest.FileCount != 1 || len(manifest.Files) != 1 || manifest.Files[0].Path != "operator.py" {
|
||||
t.Fatalf("unexpected included files: %#v", manifest.Files)
|
||||
}
|
||||
excluded := map[string]bool{}
|
||||
for _, path := range manifest.ExcludedPaths {
|
||||
excluded[path] = true
|
||||
}
|
||||
for _, expected := range []string{"checkpoints", "resume-from", "model", "pkg/__pycache__", "pkg/tests"} {
|
||||
if !excluded[expected] {
|
||||
t.Fatalf("expected excluded path %q in %#v", expected, manifest.ExcludedPaths)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCodeManifestRejectsOversizedSourceFileBeforeHashing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
codeDir := filepath.Join(workspaceRoot, "lightweight-operators", "demo")
|
||||
if err := os.MkdirAll(codeDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir code dir: %v", err)
|
||||
}
|
||||
largePath := filepath.Join(codeDir, "weights.bin")
|
||||
if err := os.WriteFile(largePath, nil, 0o644); err != nil {
|
||||
t.Fatalf("create large file: %v", err)
|
||||
}
|
||||
if err := os.Truncate(largePath, maxCodeManifestFileBytes+1); err != nil {
|
||||
t.Fatalf("truncate large file: %v", err)
|
||||
}
|
||||
|
||||
_, err := GenerateCodeManifest(workspaceRoot, domain.WorkflowNode{
|
||||
ID: "node-1",
|
||||
Data: domain.WorkflowNodeData{Label: "demo"},
|
||||
}, Config{CodeSubPath: "lightweight-operators/demo", MountPath: "/opt/operator"})
|
||||
if err == nil || !strings.Contains(err.Error(), "exceeds 32 MiB") {
|
||||
t.Fatalf("expected bounded manifest error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -43,7 +43,12 @@ export function getWorkflowRunActionState({
|
||||
return { disabled: true, label: "模板不可运行", reason: "template", message: "" };
|
||||
}
|
||||
if (isExecuting) {
|
||||
return { disabled: true, label: "正在启动...", reason: "starting", message: "" };
|
||||
return {
|
||||
disabled: true,
|
||||
label: "正在创建 Run…",
|
||||
reason: "starting",
|
||||
message: "正在准备轻量算子代码清单并创建 Run,完成后会自动显示 Run 编号。",
|
||||
};
|
||||
}
|
||||
const lock = getWorkflowEditorLockState({ isReadOnly, activeRunStatus, isRunContextLoading });
|
||||
if (lock.reason === "running") {
|
||||
|
||||
@ -3281,7 +3281,7 @@ export default function WorkflowEditor() {
|
||||
: editorLockState.reason === "checkingRun"
|
||||
? "正在确认运行状态"
|
||||
: editorLockState.reason === "starting"
|
||||
? "正在保存并启动"
|
||||
? "正在创建 Run"
|
||||
: "只读模式"}
|
||||
</span>
|
||||
)}
|
||||
|
||||
@ -175,9 +175,18 @@ test("editor mutations lock during the save-and-start window", () => {
|
||||
isRunContextLoading: false,
|
||||
isExecuting: true,
|
||||
});
|
||||
const action = getWorkflowRunActionState({
|
||||
isTemplate: false,
|
||||
isReadOnly: false,
|
||||
activeRunStatus: "",
|
||||
isRunContextLoading: false,
|
||||
isExecuting: true,
|
||||
});
|
||||
|
||||
assert.equal(lock.locked, true);
|
||||
assert.equal(lock.reason, "starting");
|
||||
assert.equal(action.label, "正在创建 Run…");
|
||||
assert.match(action.message, /代码清单/);
|
||||
});
|
||||
|
||||
test("node env rows include inherited workflow envs but keep node overrides first", () => {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user