🐛 Enforce V4 delivery for legacy release runs
This commit is contained in:
parent
6d8b011c67
commit
f4e147e355
@ -59,3 +59,9 @@ status: accepted
|
||||
下游需要仅凭总 JSON 和原始数据定位、切分、按类归组。新 v4 使用任务原始输入的 BOS 选择冻结八类源文件及 bucket/prefix,主文件不再依赖平台 workspace。动作同时明确切片内秒数、原始 part 秒数与原始硬件时间戳;用原始左右 PTS 各自解析硬件区间,避免用左视频秒数误切另一相机。输入快照指纹不是单个 BOS 对象内容版本的保证。实际切割和分类打包仍由下游执行。
|
||||
|
||||
缺少可靠来源、分类标签或时间证据时,选择拒绝新 v4 交付,而非“尽量输出”后让客户承受丢分类或错切的风险。发布同时验证消费契约、来源归属和文件存在。已交付结果不原地改写;历史只按证据补齐并产生新批次。这个要求适用于尚未发布的 v4,不改变 v1–v3;旧本地 v4 候选须重新生成。实现和本地消费验证见 `docs/delivery-classification-v4.zh-CN.md`。
|
||||
|
||||
## 2026-11-08 修订:旧任务完成后的 V4 格式约束
|
||||
|
||||
同一组织、项目、工作流版本家族已启用已发布且可见的 V4 工作流时,尚未交付的旧任务也必须生成 V4。人工任务、来源 Run、已验收修订和原始输入仍属于原工作流,不重绑、不重跑标注。放行子 Run 单独冻结兼容的打包节点、启用参数及摘要镜像;执行、重启恢复与重试使用该快照。冻结的节点仅替换打包运行配置,不修改原工作流。缺少原始输入选择、冻结分类字典或已验收证据时,明确失败并恢复证据,禁止输出旧格式兜底。
|
||||
|
||||
Worker 在写入交付目标前检查实际格式;V4 客户目录只允许 `part.json`,审核历史仍为独立内部产物。已完整发布的旧包继续支持逐成员归属、SHA-256、大小和完成清单校验后的**只读状态恢复**;不续传未完成旧包、不接管冲突目录、不改写旧完成清单。历史转换仍产生独立候选/新批次,生产转换与正式发布分别验收。
|
||||
|
||||
@ -52,10 +52,16 @@
|
||||
|
||||
回退旧 API 不是无损写入保证:新版本保存分类后,旧版本重建草稿数组可能丢弃字典快照。回退前应暂停相关写操作或准备能保留新字段的兼容回退版本;只保留恢复历史不等于能自动恢复所有后续修改。已交付不可变文件不因此原地迁移或覆盖。
|
||||
|
||||
## 2026-10-22 启用决定
|
||||
## 2026-10-22 启用决定(在途任务范围已由 2026-11-08 修订替代)
|
||||
|
||||
用户随后确认启用 v4,范围仅为切换后新启动的工作流;在途任务、它们后续的审核/打包子运行和已交付文件继续原协议。此决定替代上节“继续保持关闭”的准备范围,不代表生产开关已经启用。实际发布仍需完成镜像、兼容性和目标核对,并审查具体部署及工作流切换计划。
|
||||
|
||||
用户明确确认当前无 tags 的启用词条以词条名称作为第一个独立 tags 标签。该初始化是一次有界字典变更:保留词条 ID、名称、用途、category、aliases 和既有历史,递增版本并记录新修订;已有非空 tags 不覆盖。历史任务中的冻结快照不因此重写,亦不反查新词条替代历史分类。后续 tags 仍由管理员维护。
|
||||
|
||||
新工作流使用新的定义版本及固定打包镜像。旧定义不原地改参数或镜像;切换必须保留已启动采集的原工作流归属,并阻止新监听器重新接入这些目录。新 v4 接入还需完整的原始文件选择,包含采集 meta;旧协议保留原有接入要求。准备与验收步骤见 [启用核对表](release/delivery-v4-activation.md)。
|
||||
|
||||
## 2026-11-08 旧任务交付修订
|
||||
|
||||
用户进一步要求不再新生成旧格式交付包。同一组织、项目和版本家族已启用 V4 时,旧任务后续放行也冻结兼容的 V4 打包配置;原工作流、人工任务归属和已验收内容不变。执行、重试与重启沿用冻结镜像及参数。无法证明原始来源、冻结字典、标签和时间证据时停在明确失败状态,先恢复证据,不回退旧格式。
|
||||
|
||||
已完整发布的旧文件仅允许校验完成清单、成员哈希和归属后只读恢复状态;缺失或冲突的旧上传不得继续补发。转换生成独立候选或新交付批次,不覆盖旧完成清单。线上替换、历史证据恢复和新批次正式发布需要分别验证;本地转换通过不代表全部历史包可正式交付。
|
||||
|
||||
@ -790,6 +790,7 @@ type LightweightOperatorCodeManifest struct {
|
||||
}
|
||||
|
||||
type RunMetadata struct {
|
||||
CaptureDeliveryPackage *CaptureDeliveryPackageRuntime `json:"captureDeliveryPackage,omitempty" bson:"captureDeliveryPackage,omitempty"`
|
||||
LightweightOperatorCodeManifests map[string]LightweightOperatorCodeManifest `json:"lightweightOperatorCodeManifests,omitempty" bson:"lightweightOperatorCodeManifests,omitempty"`
|
||||
TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"`
|
||||
AnnotationTaskIDsByNode map[string][]string `json:"annotationTaskIdsByNode,omitempty" bson:"annotationTaskIdsByNode,omitempty"`
|
||||
@ -797,6 +798,16 @@ type RunMetadata struct {
|
||||
HumanReleaseUniqueKey string `json:"humanReleaseUniqueKey,omitempty" bson:"humanReleaseUniqueKey,omitempty"`
|
||||
}
|
||||
|
||||
// A release child keeps its source workflow identity while freezing the newer
|
||||
// package runtime. It never changes the source task or the stored workflow.
|
||||
type CaptureDeliveryPackageRuntime struct {
|
||||
SourceRunID string `json:"sourceRunId" bson:"sourceRunId"`
|
||||
SourceWorkflowID string `json:"sourceWorkflowId" bson:"sourceWorkflowId"`
|
||||
PolicyWorkflowID string `json:"policyWorkflowId" bson:"policyWorkflowId"`
|
||||
PolicyVersion int `json:"policyVersion" bson:"policyVersion"`
|
||||
Node WorkflowNode `json:"node" bson:"node"`
|
||||
}
|
||||
|
||||
// RunFailure is the machine-readable counterpart to RunRecord.Error. Error is
|
||||
// intentionally retained for backwards compatibility and human-readable logs.
|
||||
type RunFailure struct {
|
||||
|
||||
@ -138,7 +138,19 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
publisher := &deliveryPublisher{}
|
||||
f.runtime.artifactPublisher = publisher
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy bytes"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil {
|
||||
t.Fatalf("old child cannot publish: %v", err)
|
||||
}
|
||||
f.activate()
|
||||
if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil {
|
||||
t.Fatalf("completed legacy recovery failed: %v", err)
|
||||
}
|
||||
for _, before := range []domain.CaptureRecord{active, done, launching} {
|
||||
after, _ := f.db.GetCaptureRecord(f.ctx, before.ID)
|
||||
if after == nil || after.WorkflowID != f.old.ID || after.RunID != before.RunID {
|
||||
@ -149,15 +161,6 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin
|
||||
if err != nil || lookup == nil || lookup.ID != active.ID {
|
||||
t.Fatalf("old child lost Capture: %+v %v", lookup, err)
|
||||
}
|
||||
publisher := &deliveryPublisher{}
|
||||
f.runtime.artifactPublisher = publisher
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy bytes"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil {
|
||||
t.Fatalf("old child cannot publish: %v", err)
|
||||
}
|
||||
child.Status = domain.RunStatusCompleted
|
||||
if err := f.db.UpsertRun(f.ctx, child); err != nil {
|
||||
t.Fatal(err)
|
||||
@ -180,8 +183,8 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin
|
||||
t.Fatalf("wrong new Run binding: %+v", newRun)
|
||||
}
|
||||
oldMode, err := f.runtime.classificationDeliveryEnabled(f.ctx, child.WorkflowID)
|
||||
if err != nil || oldMode {
|
||||
t.Fatalf("old child selected v4: %v %v", oldMode, err)
|
||||
if err != nil || !oldMode {
|
||||
t.Fatalf("old child did not select v4 policy: %v %v", oldMode, err)
|
||||
}
|
||||
newMode, err := f.runtime.classificationDeliveryEnabled(f.ctx, newRun.WorkflowID)
|
||||
if err != nil || !newMode {
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@ -88,6 +89,7 @@ type deliveryTarget struct {
|
||||
stageRoot string
|
||||
marker string
|
||||
publish func(key string, body []byte) error
|
||||
completed *domain.ExecutionArtifact
|
||||
}
|
||||
|
||||
// prepareDeliveryTarget performs every check that must hold before the first
|
||||
@ -96,6 +98,9 @@ type deliveryTarget struct {
|
||||
// ownership and the conflict rules of the shared destination prefix.
|
||||
func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (*deliveryTarget, error) {
|
||||
record, err := runtime.store.GetCaptureRecordByRunID(ctx, run.ID)
|
||||
if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil {
|
||||
record, err = runtime.frozenDeliveryCapture(ctx, run)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -139,6 +144,9 @@ func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.R
|
||||
if run.Trigger == domain.RunTriggerHumanRelease && run.ParentRunID != "" {
|
||||
ownerRunID = run.ParentRunID
|
||||
}
|
||||
if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil {
|
||||
ownerRunID = run.Metadata.CaptureDeliveryPackage.SourceRunID
|
||||
}
|
||||
owner := record.ID + ":" + ownerRunID
|
||||
// PutInternal is conditional-create. A failed invocation may leave staging
|
||||
// objects behind, so each retry needs its own private staging namespace.
|
||||
@ -166,6 +174,13 @@ func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.R
|
||||
return publisher.Delete(ctx, *connection, target.Bucket, stage)
|
||||
}
|
||||
if err := runtime.recheckV4DeliveryArtifact(ctx, run, artifact); err != nil {
|
||||
if errors.Is(err, errLegacyDeliveryRequiresV4) {
|
||||
completed, recoveryErr := completedLegacyDelivery(ctx, prepared, artifact)
|
||||
if recoveryErr == nil {
|
||||
prepared.completed = &completed
|
||||
return prepared, nil
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
known, err := publisher.Head(ctx, *connection, target.Bucket, prepared.marker)
|
||||
@ -196,6 +211,9 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
if prepared.completed != nil {
|
||||
return *prepared.completed, nil
|
||||
}
|
||||
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
|
||||
publishBytes := prepared.publish
|
||||
marker := prepared.marker
|
||||
@ -399,6 +417,9 @@ func (runtime *Runtime) publishCaptureDeliveryFile(ctx context.Context, run *dom
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
if prepared.completed != nil {
|
||||
return *prepared.completed, nil
|
||||
}
|
||||
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
|
||||
publishBytes := prepared.publish
|
||||
fileName := filepath.Base(routing.FileName)
|
||||
|
||||
@ -76,7 +76,11 @@ func (runtime *Runtime) classificationDeliveryEnabled(ctx context.Context, workf
|
||||
if workflow == nil {
|
||||
return false, nil
|
||||
}
|
||||
return workflowUsesV4CaptureDelivery(*workflow), nil
|
||||
if workflowUsesV4CaptureDelivery(*workflow) {
|
||||
return true, nil
|
||||
}
|
||||
policy, err := runtime.activeV4DeliveryPolicy(ctx, workflow)
|
||||
return policy != nil, err
|
||||
}
|
||||
|
||||
// Reuse normal internal-artifact persistence rather than exceeding the 64 KiB
|
||||
|
||||
@ -1094,6 +1094,18 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord,
|
||||
if workflowErr != nil {
|
||||
workflow = nil
|
||||
}
|
||||
if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil {
|
||||
if workflowErr != nil {
|
||||
return executionResult{}, nil, workflowErr
|
||||
}
|
||||
workflow, err = deliveryRuntimeWorkflow(workflow, run)
|
||||
if err != nil || workflow == nil {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("missing delivery workflow")
|
||||
}
|
||||
return executionResult{}, nil, err
|
||||
}
|
||||
}
|
||||
settings, err := runtime.store.GetSettings(ctx)
|
||||
if err != nil {
|
||||
return executionResult{}, nil, err
|
||||
@ -1152,7 +1164,10 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord,
|
||||
return executionResult{}, nil, fmt.Errorf("project deletion in progress")
|
||||
}
|
||||
}
|
||||
image, err := sourceIndexPackageImage(node, inputArtifacts, runtime.operatorRunnerImage)
|
||||
image, frozen, err := frozenDeliveryPackageImage(run, node)
|
||||
if err == nil && !frozen {
|
||||
image, err = sourceIndexPackageImage(node, inputArtifacts, runtime.operatorRunnerImage)
|
||||
}
|
||||
if err != nil {
|
||||
return executionResult{}, nil, err
|
||||
}
|
||||
|
||||
@ -157,6 +157,10 @@ func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch
|
||||
if gateExecution == nil {
|
||||
return fmt.Errorf("human release gate execution %s is missing", batch.GateNodeID)
|
||||
}
|
||||
frozenPackage, err := runtime.freezeLegacyDeliveryRuntime(ctx, workflow, parent, target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
artifacts, err := runtime.releasePartitionArtifacts(ctx, parent, *gateNode, *gateExecution, partition)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -189,6 +193,7 @@ func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch
|
||||
if child.Metadata == nil {
|
||||
child.Metadata = &domain.RunMetadata{}
|
||||
}
|
||||
child.Metadata.CaptureDeliveryPackage = frozenPackage
|
||||
child.Metadata.HumanReleasePartitionID = partition.ID
|
||||
child.Metadata.HumanReleaseUniqueKey = partition.UniqueKey
|
||||
child.UpdatedAt = nowISO()
|
||||
|
||||
144
internal/worker/legacy_delivery_recovery.go
Normal file
144
internal/worker/legacy_delivery_recovery.go
Normal file
@ -0,0 +1,144 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
// Recover only a byte-identical, fully completed immutable publication. This
|
||||
// path cannot resume an incomplete legacy upload after a V4 family cutover.
|
||||
func completedLegacyDelivery(ctx context.Context, target *deliveryTarget, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
|
||||
info, err := os.Stat(artifact.Path)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
paths := []string{artifact.Path}
|
||||
kind := "part-index"
|
||||
if info.IsDir() {
|
||||
kind = "directory"
|
||||
paths = nil
|
||||
err = filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, e error) error {
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("legacy delivery symlink")
|
||||
}
|
||||
if !entry.IsDir() {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
sort.Slice(paths, func(i, j int) bool {
|
||||
a, b := filepath.Base(paths[i]) == "packages-manifest.json", filepath.Base(paths[j]) == "packages-manifest.json"
|
||||
if a != b {
|
||||
return !a
|
||||
}
|
||||
return paths[i] < paths[j]
|
||||
})
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return artifact, fmt.Errorf("empty legacy delivery")
|
||||
}
|
||||
entries := make([]directoryManifestEntry, 0, len(paths))
|
||||
check := func(key, sha string, size int64) error {
|
||||
meta, e := target.publisher.Head(ctx, *target.connection, target.target.Bucket, key)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if !deliveryObjectMatches(meta, target.owner, sha, size) {
|
||||
return fmt.Errorf("legacy delivery incomplete or changed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, path := range paths {
|
||||
relative := filepath.Base(path)
|
||||
if info.IsDir() {
|
||||
relative, err = filepath.Rel(artifact.Path, path)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
relative = filepath.ToSlash(relative)
|
||||
} else {
|
||||
routing, e := partIndexPayloadPeek(path)
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
relative = filepath.Base(routing.FileName)
|
||||
}
|
||||
if relative == "manifest.json" || relative == ".delivery-owner.json" || strings.HasPrefix(relative, "../") {
|
||||
return artifact, fmt.Errorf("reserved legacy member")
|
||||
}
|
||||
f, e := os.Open(path)
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
hash := sha256.New()
|
||||
size, e := io.Copy(hash, f)
|
||||
closeErr := f.Close()
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
if closeErr != nil {
|
||||
return artifact, closeErr
|
||||
}
|
||||
entry := directoryManifestEntry{Path: relative, Key: target.target.Prefix + "/" + relative, SizeBytes: size, SHA256: hex.EncodeToString(hash.Sum(nil))}
|
||||
if e = check(entry.Key, entry.SHA256, size); e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
marker, _ := json.Marshal(map[string]string{"owner": target.owner})
|
||||
body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: kind, FileCount: len(entries), TotalBytes: sumManifestBytes(entries), Entries: entries})
|
||||
expected := map[string]bool{}
|
||||
for _, entry := range entries {
|
||||
expected[entry.Key] = true
|
||||
}
|
||||
manifestKey := target.target.Prefix + "/manifest.json"
|
||||
for key, value := range map[string][]byte{target.marker: marker, manifestKey: body} {
|
||||
hash := sha256.Sum256(value)
|
||||
if e := check(key, hex.EncodeToString(hash[:]), int64(len(value))); e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
expected[key] = true
|
||||
}
|
||||
keys, err := target.lister.ListPrefix(ctx, *target.connection, target.target.Bucket, target.target.Prefix+"/")
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
if len(keys) != len(expected) {
|
||||
return artifact, fmt.Errorf("legacy delivery member count differs")
|
||||
}
|
||||
for _, key := range keys {
|
||||
if !expected[key] {
|
||||
return artifact, fmt.Errorf("unexpected legacy delivery member")
|
||||
}
|
||||
}
|
||||
artifact.Path = "bos://" + target.target.Bucket + "/" + target.target.Prefix
|
||||
artifact.SizeBytes = sumManifestBytes(entries)
|
||||
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.target.ConnectionID, Bucket: target.target.Bucket, Prefix: target.target.Prefix, ManifestKey: manifestKey, Kind: domain.ArtifactRefKindDirectory}
|
||||
if !info.IsDir() {
|
||||
entry := entries[0]
|
||||
artifact.Path = "bos://" + target.target.Bucket + "/" + entry.Key
|
||||
ref.Prefix = ""
|
||||
ref.Key = entry.Key
|
||||
ref.SizeBytes = entry.SizeBytes
|
||||
ref.SHA256 = entry.SHA256
|
||||
ref.Kind = domain.ArtifactRefKindObject
|
||||
}
|
||||
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref}
|
||||
return artifact, nil
|
||||
}
|
||||
183
internal/worker/legacy_delivery_v4.go
Normal file
183
internal/worker/legacy_delivery_v4.go
Normal file
@ -0,0 +1,183 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
// The active released version is a delivery policy for its exact tenant,
|
||||
// project and definition family. It does not rebind any in-flight source Run.
|
||||
func (r *Runtime) activeV4DeliveryPolicy(ctx context.Context, source *domain.WorkflowRecord) (*domain.WorkflowRecord, error) {
|
||||
if source == nil || source.ProjectID == "" || source.OrganizationID == "" || source.CaptureAutomation == nil {
|
||||
return nil, nil
|
||||
}
|
||||
normalized := *source
|
||||
domain.NormalizeWorkflowDefinitionMetadata(&normalized)
|
||||
versions, err := r.store.ListWorkflowsByProject(ctx, source.ProjectID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var chosen *domain.WorkflowRecord
|
||||
for _, candidate := range versions {
|
||||
domain.NormalizeWorkflowDefinitionMetadata(&candidate)
|
||||
if candidate.OrganizationID != source.OrganizationID || candidate.ProjectID != source.ProjectID || candidate.DefinitionFamilyID != normalized.DefinitionFamilyID || candidate.DefinitionState != domain.WorkflowDefinitionStateReleased || candidate.Hidden || !workflowUsesV4CaptureDelivery(candidate) {
|
||||
continue
|
||||
}
|
||||
if chosen != nil {
|
||||
return nil, fmt.Errorf("同一工作流版本家族有多个启用的 V4 交付配置,请先确认")
|
||||
}
|
||||
copy := candidate
|
||||
chosen = ©
|
||||
}
|
||||
return chosen, nil
|
||||
}
|
||||
|
||||
func (r *Runtime) freezeLegacyDeliveryRuntime(ctx context.Context, source *domain.WorkflowRecord, parent *domain.RunRecord, targetID string) (*domain.CaptureDeliveryPackageRuntime, error) {
|
||||
if source == nil || parent == nil || workflowUsesV4CaptureDelivery(*source) || parent.Mode.Normalized() == domain.RunModeDebug {
|
||||
return nil, nil
|
||||
}
|
||||
policy, err := r.activeV4DeliveryPolicy(ctx, source)
|
||||
if err != nil || policy == nil {
|
||||
return nil, err
|
||||
}
|
||||
old := domain.GetNodeByID(source.Nodes, targetID)
|
||||
next := domain.GetNodeByID(policy.Nodes, targetID)
|
||||
if old == nil || next == nil || old.Data.OperatorID != "operator-builtin-package" || next.Data.OperatorID != old.Data.OperatorID {
|
||||
return nil, fmt.Errorf("旧任务缺少兼容的 V4 打包节点,请先核对交付配置")
|
||||
}
|
||||
image, err := sourceIndexPackageImage(*next, nil, r.operatorRunnerImage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected := strings.TrimSpace(next.Data.OperatorImageID)
|
||||
if expected == "" {
|
||||
expected = strings.TrimSpace(next.Data.Image)
|
||||
}
|
||||
if image != expected {
|
||||
return nil, fmt.Errorf("部署的 V4 打包镜像与已发布工作流不一致,请先核对")
|
||||
}
|
||||
var node domain.WorkflowNode
|
||||
body, err := json.Marshal(old)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = json.Unmarshal(body, &node); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node.Data.Image, node.Data.OperatorImageID = image, image
|
||||
node.Data.OperatorVersion = next.Data.OperatorVersion
|
||||
node.Data.PublicationEvidence = next.Data.PublicationEvidence
|
||||
node.Data.PublicationStatus = next.Data.PublicationStatus
|
||||
node.Data.GenerationSource = next.Data.GenerationSource
|
||||
node = applyNodeOverridesToNode(node, domain.NodeOverride{Hyperparams: []domain.HyperParam{
|
||||
{Key: "PIN_OPERATOR_IMAGE", Value: "true"}, {Key: "PART_INDEX_DELIVERY", Value: "true"}, {Key: "DELIVERY_JSON_V4", Value: "true"}, {Key: "DELIVERY_JSON_V3", Value: "false"}, {Key: "SOURCE_INDEX_DELIVERY", Value: "true"},
|
||||
}})
|
||||
audit := false
|
||||
for _, port := range node.Data.Outputs {
|
||||
if port.ID == "deliveryAudit" {
|
||||
audit = true
|
||||
}
|
||||
}
|
||||
if !audit {
|
||||
node.Data.Outputs = append(node.Data.Outputs, domain.PortDefinition{ID: "deliveryAudit", Name: "deliveryAudit", Type: domain.PortTypeJSON})
|
||||
}
|
||||
return &domain.CaptureDeliveryPackageRuntime{SourceRunID: parent.ID, SourceWorkflowID: source.ID, PolicyWorkflowID: policy.ID, PolicyVersion: policy.DefinitionVersion, Node: node}, nil
|
||||
}
|
||||
|
||||
func deliveryRuntimeWorkflow(workflow *domain.WorkflowRecord, run *domain.RunRecord) (*domain.WorkflowRecord, error) {
|
||||
if workflow == nil || run == nil || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil {
|
||||
return workflow, nil
|
||||
}
|
||||
frozen := run.Metadata.CaptureDeliveryPackage
|
||||
if workflow.ID != run.WorkflowID || workflow.ID != frozen.SourceWorkflowID || workflow.OrganizationID != run.OrganizationID || workflow.ProjectID != run.ProjectID || frozen.SourceRunID == "" || frozen.PolicyWorkflowID == "" || frozen.Node.Data.OperatorID != "operator-builtin-package" || !workflowUsesV4CaptureDelivery(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{frozen.Node}}) || !sourcePackageDigestImage.MatchString(frozen.Node.Data.Image) || frozen.Node.Data.Image != frozen.Node.Data.OperatorImageID {
|
||||
return nil, fmt.Errorf("V4 交付运行快照不完整或归属不符")
|
||||
}
|
||||
copy := *workflow
|
||||
copy.Nodes = append([]domain.WorkflowNode(nil), workflow.Nodes...)
|
||||
found := false
|
||||
for i, node := range copy.Nodes {
|
||||
if node.ID == frozen.Node.ID {
|
||||
if node.Data.OperatorID != "operator-builtin-package" {
|
||||
return nil, fmt.Errorf("V4 交付节点类型不符")
|
||||
}
|
||||
body, err := json.Marshal(frozen.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var replacement domain.WorkflowNode
|
||||
if err = json.Unmarshal(body, &replacement); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
copy.Nodes[i] = replacement
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("V4 交付节点不存在")
|
||||
}
|
||||
return ©, nil
|
||||
}
|
||||
|
||||
func frozenDeliveryPackageImage(run *domain.RunRecord, node domain.WorkflowNode) (string, bool, error) {
|
||||
if run == nil || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil || run.Metadata.CaptureDeliveryPackage.Node.ID != node.ID {
|
||||
return "", false, nil
|
||||
}
|
||||
frozen := run.Metadata.CaptureDeliveryPackage.Node
|
||||
if frozen.Data.OperatorID != "operator-builtin-package" || !workflowUsesV4CaptureDelivery(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{node}}) || !sourcePackageDigestImage.MatchString(frozen.Data.Image) || node.Data.Image != frozen.Data.Image {
|
||||
return "", true, fmt.Errorf("V4 打包运行快照已改变,请先核对")
|
||||
}
|
||||
return frozen.Data.Image, true, nil
|
||||
}
|
||||
|
||||
// A retry with an explicit immutable delivery snapshot shares the accepted
|
||||
// release attempt. Ordinary retry runs retain the Store's strict isolation.
|
||||
func (r *Runtime) frozenDeliveryCapture(ctx context.Context, run *domain.RunRecord) (*domain.CaptureRecord, error) {
|
||||
_, lineage, err := r.originalDeliveryRun(ctx, run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
capture, err := r.store.GetCaptureRecord(ctx, run.TaskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if capture == nil || capture.OrganizationID != run.OrganizationID || capture.ProjectID != run.ProjectID || capture.WorkflowID != run.WorkflowID || !lineage[capture.RunID] {
|
||||
return nil, fmt.Errorf("frozen delivery capture scope mismatch")
|
||||
}
|
||||
return capture, nil
|
||||
}
|
||||
|
||||
// Persist the newest retry association before it launches. A restart repeats
|
||||
// this safely; an unrelated or newer branch cannot be adopted or overwritten.
|
||||
func (r *Runtime) attachFrozenReleaseRetry(ctx context.Context, run *domain.RunRecord) error {
|
||||
if run.Trigger != domain.RunTriggerRetry || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil {
|
||||
return nil
|
||||
}
|
||||
_, lineage, err := r.originalDeliveryRun(ctx, run)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
partition, err := r.store.GetReleasePartitionByUniqueKey(ctx, run.OrganizationID, run.Metadata.HumanReleaseUniqueKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if partition == nil || partition.ID != run.Metadata.HumanReleasePartitionID || partition.UniqueKey != run.Metadata.HumanReleaseUniqueKey || !lineage[partition.ChildRunID] || partition.Status != domain.ReleasePartitionDispatched {
|
||||
return fmt.Errorf("V4 retry release partition scope mismatch")
|
||||
}
|
||||
batch, err := r.store.GetHumanBatch(ctx, partition.BatchID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if batch == nil || batch.OrganizationID != run.OrganizationID || batch.ProjectID != run.ProjectID || batch.WorkflowID != run.WorkflowID || batch.RunID != run.Metadata.CaptureDeliveryPackage.SourceRunID {
|
||||
return fmt.Errorf("V4 retry release batch scope mismatch")
|
||||
}
|
||||
if partition.ChildRunID == run.ID {
|
||||
return nil
|
||||
}
|
||||
partition.ChildRunID = run.ID
|
||||
partition.UpdatedAt = nowISO()
|
||||
return r.store.UpsertReleasePartition(ctx, *partition)
|
||||
}
|
||||
348
internal/worker/legacy_delivery_v4_test.go
Normal file
348
internal/worker/legacy_delivery_v4_test.go
Normal file
@ -0,0 +1,348 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/objectsync"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
func TestLegacyCaptureReleaseRequiresV4AfterFamilyCutover(t *testing.T) {
|
||||
f := newCaptureCutoverFixture(t, "legacy-source")
|
||||
record := f.seed("legacy-source", domain.CaptureRecordWaiting, "legacy-parent")
|
||||
parent := domain.RunRecord{ID: record.RunID, OrganizationID: record.OrganizationID, ProjectID: record.ProjectID, WorkflowID: f.old.ID, TaskID: record.ID, Status: domain.RunStatusWaitingForHuman}
|
||||
child := parent
|
||||
child.ID = "legacy-child"
|
||||
child.ParentRunID = parent.ID
|
||||
child.Trigger = domain.RunTriggerHumanRelease
|
||||
child.Status = domain.RunStatusRunning
|
||||
for _, run := range []domain.RunRecord{parent, child} {
|
||||
if err := f.db.UpsertRun(f.ctx, run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
f.activate()
|
||||
publisher := &deliveryPublisher{}
|
||||
f.runtime.artifactPublisher = publisher
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy payload"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root})
|
||||
if err == nil {
|
||||
t.Fatal("a newly completed legacy task published an old-format customer package after V4 cutover")
|
||||
}
|
||||
if len(publisher.objects) != 0 {
|
||||
t.Fatal("legacy rejection must precede owner markers and staging writes")
|
||||
}
|
||||
current, _ := f.db.GetCaptureRecord(f.ctx, record.ID)
|
||||
if current.WorkflowID != f.old.ID {
|
||||
t.Fatal("source ownership must not be rebound")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyCaptureReleaseCollectsV4AuditAfterFamilyCutover(t *testing.T) {
|
||||
f := newCaptureCutoverFixture(t, "legacy-audit")
|
||||
f.activate()
|
||||
enabled, err := f.runtime.classificationDeliveryEnabled(f.ctx, f.old.ID)
|
||||
if err != nil || !enabled {
|
||||
t.Fatalf("old task must collect the new customer format's internal audit after cutover: enabled=%v err=%v", enabled, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyCaptureRuntimeIsFrozenAndScoped(t *testing.T) {
|
||||
f := newCaptureCutoverFixture(t, "frozen")
|
||||
f.activate()
|
||||
parent := &domain.RunRecord{ID: "parent", OrganizationID: f.old.OrganizationID, ProjectID: f.old.ProjectID, WorkflowID: f.old.ID, TaskID: "capture"}
|
||||
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", f.next.Nodes[0].Data.OperatorImageID)
|
||||
frozen, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package")
|
||||
if err != nil || frozen == nil {
|
||||
t.Fatalf("freeze: %+v %v", frozen, err)
|
||||
}
|
||||
run := *parent
|
||||
run.ID = "child"
|
||||
run.ParentRunID = parent.ID
|
||||
run.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: frozen}
|
||||
effective, err := deliveryRuntimeWorkflow(&f.old, &run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !workflowUsesV4CaptureDelivery(*effective) || workflowUsesV4CaptureDelivery(f.old) || run.WorkflowID != f.old.ID {
|
||||
t.Fatal("execution must upgrade without mutating source ownership/configuration")
|
||||
}
|
||||
if len(effective.Nodes[0].Data.Outputs) != 1 || effective.Nodes[0].Data.Outputs[0].ID != "deliveryAudit" {
|
||||
t.Fatal("missing internal audit output")
|
||||
}
|
||||
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", "invalid-drift")
|
||||
image, handled, err := frozenDeliveryPackageImage(&run, effective.Nodes[0])
|
||||
if err != nil || !handled || image != frozen.Node.Data.Image {
|
||||
t.Fatalf("restart changed package image: %s %v", image, err)
|
||||
}
|
||||
run.OrganizationID = "foreign"
|
||||
if _, err = deliveryRuntimeWorkflow(&f.old, &run); err == nil {
|
||||
t.Fatal("foreign runtime accepted")
|
||||
}
|
||||
parent.Mode = domain.RunModeDebug
|
||||
if frozen, err = f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package"); err != nil || frozen != nil {
|
||||
t.Fatal("debug selection must be unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyCaptureRuntimeRejectsIncompatiblePublishedImage(t *testing.T) {
|
||||
f := newCaptureCutoverFixture(t, "image")
|
||||
f.activate()
|
||||
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", "registry.example/wrong@sha256:"+strings.Repeat("d", 64))
|
||||
if _, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, &domain.RunRecord{ID: "parent"}, "package"); err == nil {
|
||||
t.Fatal("unexpected operator image accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyCapturePolicyNeverCrossesFamilyOrTenant(t *testing.T) {
|
||||
for _, scope := range []string{"family", "organization", "project", "hidden"} {
|
||||
t.Run(scope, func(t *testing.T) {
|
||||
f := newCaptureCutoverFixture(t, "scope")
|
||||
f.activate()
|
||||
next := f.next
|
||||
next.Hidden = false
|
||||
switch scope {
|
||||
case "family":
|
||||
next.DefinitionFamilyID = "foreign"
|
||||
case "organization":
|
||||
next.OrganizationID = "foreign"
|
||||
case "project":
|
||||
next.ProjectID = "foreign"
|
||||
case "hidden":
|
||||
next.Hidden = true
|
||||
}
|
||||
if err := f.db.UpsertWorkflow(f.ctx, next); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policy, err := f.runtime.activeV4DeliveryPolicy(f.ctx, &f.old)
|
||||
if err != nil || policy != nil {
|
||||
t.Fatalf("crossed %s: %+v %v", scope, policy, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyCaptureRetryPreservesPackageAndAcceptedBaseline(t *testing.T) {
|
||||
f := newCaptureCutoverFixture(t, "retry")
|
||||
f.activate()
|
||||
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", f.next.Nodes[0].Data.OperatorImageID)
|
||||
parent := &domain.RunRecord{ID: "source", OrganizationID: f.old.OrganizationID, ProjectID: f.old.ProjectID, WorkflowID: f.old.ID, TaskID: "capture"}
|
||||
parent.Inputs = map[string]domain.ArtifactPayload{"sourceDirectory": {Mode: domain.ArtifactTransferModeRef, Type: domain.PortTypeFile, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: "cutover-storage", Bucket: "synthetic", Prefix: "incoming/retry", Kind: domain.ArtifactRefKindDirectory}}}
|
||||
if err := f.db.UpsertRun(f.ctx, *parent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frozen, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := *parent
|
||||
failed.ID = "failed"
|
||||
failed.ParentRunID = parent.ID
|
||||
failed.RestartFromNodeID = "package"
|
||||
failed.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: frozen, HumanReleasePartitionID: "partition", HumanReleaseUniqueKey: "key"}
|
||||
failed.NodeExecutions = []domain.NodeExecutionRecord{{NodeID: "gate", Status: domain.NodeExecutionStatusCompleted, Artifacts: []domain.ExecutionArtifact{{Name: "accepted", Type: domain.PortTypeJSON, Data: map[string]any{"accepted": true}}}}, {NodeID: "package", Status: domain.NodeExecutionStatusFailed}}
|
||||
if err := f.db.UpsertRun(f.ctx, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retry, err := f.runtime.enqueueRetryRun(f.ctx, &f.old, &failed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retry.Metadata == nil || retry.Metadata.CaptureDeliveryPackage == nil || retry.Metadata.CaptureDeliveryPackage.SourceRunID != parent.ID || retry.Metadata.HumanReleasePartitionID != "partition" || retry.RestartFromNodeID != "package" || len(retry.NodeExecutions) != 1 || retry.NodeExecutions[0].NodeID != "gate" {
|
||||
t.Fatalf("retry lost accepted delivery state: %+v", retry)
|
||||
}
|
||||
allowed := rerunAllowedNodeSet(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{{ID: "source"}, {ID: "gate"}, {ID: "package"}}, Edges: []domain.WorkflowEdge{{Source: "source", Target: "gate"}, {Source: "gate", Target: "package"}}}, retry)
|
||||
if _, ok := allowed["package"]; !ok {
|
||||
t.Fatal("retry lost package subtree")
|
||||
}
|
||||
if _, ok := allowed["gate"]; ok {
|
||||
t.Fatal("retry reruns accepted human gate")
|
||||
}
|
||||
retry.Metadata.CaptureDeliveryPackage.Node.Data.Hyperparams[0].Value = "changed"
|
||||
if frozen.Node.Data.Hyperparams[0].Value == "changed" {
|
||||
t.Fatal("retry aliased previous immutable selection")
|
||||
}
|
||||
}
|
||||
|
||||
// Every write method is counted; equal final objects alone would not prove a
|
||||
// completed legacy recovery is read-only.
|
||||
type readonlyLegacyPublisher struct {
|
||||
deliveryPublisher
|
||||
writes int
|
||||
}
|
||||
|
||||
func (p *readonlyLegacyPublisher) PutInternal(c context.Context, x domain.StorageConnectionRecord, b, k string, r io.Reader, n int64, t string, m map[string]string) (objectsync.ObjectMetadata, error) {
|
||||
p.writes++
|
||||
return p.deliveryPublisher.PutInternal(c, x, b, k, r, n, t, m)
|
||||
}
|
||||
func (p *readonlyLegacyPublisher) PublishConditional(c context.Context, x domain.StorageConnectionRecord, b, s, k string, n int64, e, a, z string, m map[string]string) (objectsync.ObjectMetadata, error) {
|
||||
p.writes++
|
||||
return p.deliveryPublisher.PublishConditional(c, x, b, s, k, n, e, a, z, m)
|
||||
}
|
||||
func (p *readonlyLegacyPublisher) Delete(c context.Context, x domain.StorageConnectionRecord, b, k string) error {
|
||||
p.writes++
|
||||
return p.deliveryPublisher.Delete(c, x, b, k)
|
||||
}
|
||||
|
||||
func TestLegacyCaptureCompletedRecoveryNeverWrites(t *testing.T) {
|
||||
for _, damage := range []string{"none", "missing", "modified", "extra", "owner", "file"} {
|
||||
t.Run(damage, func(t *testing.T) {
|
||||
f := newCaptureCutoverFixture(t, "completed")
|
||||
record := f.seed("completed", domain.CaptureRecordWaiting, "parent")
|
||||
run := domain.RunRecord{ID: "parent", OrganizationID: record.OrganizationID, ProjectID: record.ProjectID, WorkflowID: f.old.ID, TaskID: record.ID}
|
||||
if err := f.db.UpsertRun(f.ctx, run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publisher := &readonlyLegacyPublisher{}
|
||||
f.runtime.artifactPublisher = publisher
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "legacy.json")
|
||||
if err := os.WriteFile(path, []byte("old"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
artifact := domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}
|
||||
publish := f.runtime.publishCaptureDelivery
|
||||
if damage == "file" {
|
||||
path = filepath.Join(root, "part.json")
|
||||
if err := os.WriteFile(path, []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"part.json"}}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
artifact.Name = partIndexArtifactName
|
||||
artifact.Path = path
|
||||
publish = f.runtime.publishCaptureDeliveryFile
|
||||
}
|
||||
if _, err := publish(f.ctx, &run, "package", artifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.activate()
|
||||
publisher.writes = 0
|
||||
key := record.Delivery.Bucket + "/" + record.Delivery.Prefix + "/legacy.json"
|
||||
switch damage {
|
||||
case "missing":
|
||||
delete(publisher.objects, key)
|
||||
case "modified":
|
||||
publisher.metadata[key]["sha256"] = "changed"
|
||||
case "extra":
|
||||
publisher.objects[record.Delivery.Bucket+"/"+record.Delivery.Prefix+"/unexpected"] = []byte("x")
|
||||
case "owner":
|
||||
publisher.metadata[key]["delivery-owner"] = "foreign"
|
||||
}
|
||||
_, err := publish(f.ctx, &run, "package", artifact)
|
||||
if damage == "none" || damage == "file" {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatal("damaged legacy publication was adopted")
|
||||
}
|
||||
if publisher.writes != 0 {
|
||||
t.Fatalf("legacy recovery performed %d writes", publisher.writes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestV4CustomerDirectoryRejectsInternalOrMediaMembers(t *testing.T) {
|
||||
for _, name := range []string{"review-history.json", "left.mp4", "other.json"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, pub, run, _, artifact := originalWorkerFixture(t)
|
||||
if err := os.WriteFile(filepath.Join(artifact.Path, name), []byte("internal"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.publishCaptureDelivery(t.Context(), &run, "package", artifact); err == nil {
|
||||
t.Fatal("V4 leaked extra member")
|
||||
}
|
||||
if len(pub.puts) > 0 || len(pub.published) > 0 {
|
||||
t.Fatal("format guard ran after writes")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrozenV4RetryResolvesOnlyOriginalReleaseAncestry(t *testing.T) {
|
||||
for _, scenario := range []string{"valid", "foreign", "cycle", "missing"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
r, _, child, _, _ := originalWorkerFixture(t)
|
||||
retry := child
|
||||
retry.ID = "retry"
|
||||
retry.ParentRunID = child.ID
|
||||
retry.Trigger = domain.RunTriggerRetry
|
||||
retry.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: &domain.CaptureDeliveryPackageRuntime{SourceRunID: child.ParentRunID}}
|
||||
switch scenario {
|
||||
case "foreign":
|
||||
child.OrganizationID = "other"
|
||||
case "cycle":
|
||||
retry.Metadata.CaptureDeliveryPackage.SourceRunID = "unknown"
|
||||
child.ParentRunID = retry.ID
|
||||
case "missing":
|
||||
child.ParentRunID = "unknown"
|
||||
}
|
||||
if err := r.store.UpsertRun(t.Context(), child); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.store.UpsertRun(t.Context(), retry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := r.originalDeliverySelection(t.Context(), &retry)
|
||||
if (err == nil) != (scenario == "valid") {
|
||||
t.Fatalf("scenario=%s err=%v", scenario, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrozenReleaseRetryRestoresPartitionAndCaptureAssociation(t *testing.T) {
|
||||
r, _, child, _, _ := originalWorkerFixture(t)
|
||||
retry := child
|
||||
retry.ID = "retry"
|
||||
retry.ParentRunID = child.ID
|
||||
retry.Trigger = domain.RunTriggerRetry
|
||||
retry.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: &domain.CaptureDeliveryPackageRuntime{SourceRunID: child.ParentRunID}, HumanReleasePartitionID: "partition", HumanReleaseUniqueKey: "key"}
|
||||
if err := r.store.UpsertRun(t.Context(), retry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: child.OrganizationID, ProjectID: child.ProjectID, WorkflowID: child.WorkflowID, RunID: child.ParentRunID}
|
||||
if err := r.store.UpsertHumanBatch(t.Context(), batch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
partition := domain.ReleasePartitionRecord{ID: "partition", OrganizationID: child.OrganizationID, BatchID: batch.ID, ChildRunID: child.ID, UniqueKey: "key", Status: domain.ReleasePartitionDispatched}
|
||||
if err := r.store.UpsertReleasePartition(t.Context(), partition); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, err := r.store.GetReleasePartitionByUniqueKey(t.Context(), child.OrganizationID, "key")
|
||||
if err != nil || saved.ChildRunID != retry.ID {
|
||||
t.Fatalf("retry association: %+v %v", saved, err)
|
||||
}
|
||||
if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err != nil {
|
||||
t.Fatal("restart did not preserve association", err)
|
||||
}
|
||||
capture, err := r.frozenDeliveryCapture(t.Context(), &retry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
capture.RunID = retry.ID
|
||||
if err := r.store.UpsertCaptureRecord(t.Context(), *capture); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.originalDeliverySelection(t.Context(), &retry); err != nil {
|
||||
t.Fatal("reconciled retry lost original source", err)
|
||||
}
|
||||
saved.ChildRunID = "unrelated"
|
||||
if err := r.store.UpsertReleasePartition(t.Context(), *saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err == nil {
|
||||
t.Fatal("unrelated branch was overwritten")
|
||||
}
|
||||
}
|
||||
@ -3,6 +3,7 @@ package worker
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@ -14,28 +15,57 @@ import (
|
||||
"droboticflow/internal/workspacescope"
|
||||
)
|
||||
|
||||
// Release children carry accepted artifacts but no original Inputs. Resolve
|
||||
// only the persisted human-release edge; retries are independent selections.
|
||||
func (r *Runtime) originalDeliverySelection(ctx context.Context, run *domain.RunRecord) (*sourceidentity.Selection, error) {
|
||||
// Follow only validated persisted release ancestry, including retries carrying
|
||||
// the same frozen package selection. Never infer a source from current intake.
|
||||
func (r *Runtime) originalDeliveryRun(ctx context.Context, run *domain.RunRecord) (*domain.RunRecord, map[string]bool, error) {
|
||||
if run == nil {
|
||||
return nil, fmt.Errorf("missing original delivery run")
|
||||
return nil, nil, fmt.Errorf("missing original delivery run")
|
||||
}
|
||||
original := run
|
||||
if run.Trigger == domain.RunTriggerHumanRelease {
|
||||
lineage := map[string]bool{run.ID: true}
|
||||
if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil {
|
||||
wanted := run.Metadata.CaptureDeliveryPackage.SourceRunID
|
||||
for original.ID != wanted {
|
||||
if len(lineage) >= 32 || original.ParentRunID == "" {
|
||||
return nil, nil, fmt.Errorf("original delivery lineage incomplete")
|
||||
}
|
||||
parent, err := r.store.GetRun(ctx, original.ParentRunID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if parent == nil || lineage[parent.ID] || parent.OrganizationID != run.OrganizationID || parent.ProjectID != run.ProjectID || parent.WorkflowID != run.WorkflowID || parent.TaskID != run.TaskID {
|
||||
return nil, nil, fmt.Errorf("original delivery parent scope mismatch")
|
||||
}
|
||||
lineage[parent.ID] = true
|
||||
original = parent
|
||||
}
|
||||
} else if run.Trigger == domain.RunTriggerHumanRelease {
|
||||
parent, err := r.store.GetRun(ctx, run.ParentRunID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if parent == nil || parent.ID == run.ID || parent.OrganizationID != run.OrganizationID || parent.ProjectID != run.ProjectID || parent.WorkflowID != run.WorkflowID || parent.TaskID != run.TaskID {
|
||||
return nil, fmt.Errorf("original delivery parent scope mismatch")
|
||||
return nil, nil, fmt.Errorf("original delivery parent scope mismatch")
|
||||
}
|
||||
original = parent
|
||||
lineage[parent.ID] = true
|
||||
}
|
||||
capture, err := r.store.GetCaptureRecordByRunID(ctx, original.ID)
|
||||
return original, lineage, nil
|
||||
}
|
||||
|
||||
func (r *Runtime) originalDeliverySelection(ctx context.Context, run *domain.RunRecord) (*sourceidentity.Selection, error) {
|
||||
original, lineage, err := r.originalDeliveryRun(ctx, run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if capture == nil || capture.ID != original.TaskID || capture.RunID != original.ID && capture.RunID != run.ID {
|
||||
capture, err := r.store.GetCaptureRecordByRunID(ctx, original.ID)
|
||||
if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil {
|
||||
capture, err = r.frozenDeliveryCapture(ctx, run)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if capture == nil || capture.ID != original.TaskID || !lineage[capture.RunID] {
|
||||
return nil, fmt.Errorf("original delivery capture lineage mismatch")
|
||||
}
|
||||
return sourceidentity.FromRun(original, capture)
|
||||
@ -87,9 +117,25 @@ func (r *Runtime) preflightV4OriginalSource(ctx context.Context, run *domain.Run
|
||||
return r.recheckOriginalDeliverySelection(ctx, run, selection)
|
||||
}
|
||||
|
||||
// Detect the actual output schema, independent of mutable workflow switches.
|
||||
var errLegacyDeliveryRequiresV4 = errors.New("该工作流已启用 V4,禁止新发布旧格式交付包;请使用已验收数据重新生成 V4")
|
||||
|
||||
// Detect the actual output schema as well as the active format contract.
|
||||
// This guard runs before prepareDeliveryTarget writes even its owner marker.
|
||||
func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.RunRecord, artifact domain.ExecutionArtifact) error {
|
||||
required := run != nil && run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil
|
||||
if !required && run != nil {
|
||||
var err error
|
||||
required, err = r.classificationDeliveryEnabled(ctx, run.WorkflowID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
legacy := func() error {
|
||||
if required {
|
||||
return errLegacyDeliveryRequiresV4
|
||||
}
|
||||
return nil
|
||||
}
|
||||
name := artifact.Path
|
||||
info, err := os.Stat(name)
|
||||
if err != nil {
|
||||
@ -98,11 +144,11 @@ func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.Run
|
||||
if info.IsDir() {
|
||||
name = filepath.Join(name, "part.json")
|
||||
} else if filepath.Base(name) != "part.json" {
|
||||
return nil
|
||||
return legacy()
|
||||
}
|
||||
f, err := os.Open(name)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
return legacy()
|
||||
} // Legacy directory has no part.json.
|
||||
if err != nil {
|
||||
return err
|
||||
@ -133,7 +179,16 @@ func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.Run
|
||||
return fmt.Errorf("invalid delivery part JSON")
|
||||
}
|
||||
if part.Schema != deliveryindex.ClassificationPartSchema {
|
||||
return nil
|
||||
return legacy()
|
||||
}
|
||||
if info.IsDir() {
|
||||
members, e := os.ReadDir(artifact.Path)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if len(members) != 1 || members[0].Name() != "part.json" || !members[0].Type().IsRegular() {
|
||||
return fmt.Errorf("V4 客户交付目录必须只有 part.json")
|
||||
}
|
||||
}
|
||||
if len(raw) > 8<<20 {
|
||||
return fmt.Errorf("classification part exceeds 8 MiB")
|
||||
|
||||
@ -1400,6 +1400,10 @@ func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.Run
|
||||
Suggestion: "重新从现有发布版本创建工作流后再运行",
|
||||
})
|
||||
}
|
||||
workflow, err = deliveryRuntimeWorkflow(workflow, run)
|
||||
if err != nil {
|
||||
return runtime.failReconciledRun(ctx, run, err.Error(), domain.RunFailure{Code: "invalid_delivery_runtime", Stage: "orchestration", Retryable: false})
|
||||
}
|
||||
node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID)
|
||||
if node == nil {
|
||||
return runtime.failReconciledRun(ctx, run, fmt.Sprintf("node %s was deleted while run %s was active", run.CurrentNodeID, run.ID), domain.RunFailure{
|
||||
@ -2288,7 +2292,15 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, failedRun.Inputs, failedRun.TaskID)
|
||||
inputs := failedRun.Inputs
|
||||
if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil {
|
||||
source, _, e := runtime.originalDeliveryRun(ctx, failedRun)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
inputs = source.Inputs
|
||||
}
|
||||
retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, inputs, failedRun.TaskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -2296,6 +2308,30 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo
|
||||
retryRun.ParentRunID = failedRun.ID
|
||||
retryRun.AvailableAt = nextRunAvailableAt(workflow)
|
||||
lightweight.MergeRunMetadata(retryRun, manifests)
|
||||
if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil {
|
||||
// Preserve the immutable package selection and accepted release baseline.
|
||||
raw, e := json.Marshal(failedRun.Metadata.CaptureDeliveryPackage)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
var frozen domain.CaptureDeliveryPackageRuntime
|
||||
if e = json.Unmarshal(raw, &frozen); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if retryRun.Metadata == nil {
|
||||
retryRun.Metadata = &domain.RunMetadata{}
|
||||
}
|
||||
retryRun.Metadata.CaptureDeliveryPackage = &frozen
|
||||
retryRun.Metadata.HumanReleasePartitionID = failedRun.Metadata.HumanReleasePartitionID
|
||||
retryRun.Metadata.HumanReleaseUniqueKey = failedRun.Metadata.HumanReleaseUniqueKey
|
||||
retryRun.RestartFromNodeID = failedRun.RestartFromNodeID
|
||||
for _, execution := range failedRun.NodeExecutions {
|
||||
if execution.NodeID != frozen.Node.ID && execution.Status == domain.NodeExecutionStatusCompleted {
|
||||
applyExecution(retryRun, execution)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
retryRun.UpdatedAt = nowISO()
|
||||
if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil {
|
||||
return nil, err
|
||||
@ -2557,7 +2593,8 @@ func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord)
|
||||
strings.TrimSpace(run.DebugTarget.TargetNodeID): {},
|
||||
}
|
||||
}
|
||||
if (run.Trigger != domain.RunTriggerRerunSubtree && run.Trigger != domain.RunTriggerHumanRelease) || strings.TrimSpace(run.RestartFromNodeID) == "" {
|
||||
frozenDeliveryRetry := run.Trigger == domain.RunTriggerRetry && run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil
|
||||
if (run.Trigger != domain.RunTriggerRerunSubtree && run.Trigger != domain.RunTriggerHumanRelease && !frozenDeliveryRetry) || strings.TrimSpace(run.RestartFromNodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
visited := map[string]struct{}{}
|
||||
@ -2688,6 +2725,21 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
|
||||
claimed.UpdatedAt = nowISO()
|
||||
return claimed, runtime.store.UpsertRun(ctx, *claimed)
|
||||
}
|
||||
workflow, err = deliveryRuntimeWorkflow(workflow, claimed)
|
||||
if err != nil {
|
||||
claimed.Status = domain.RunStatusFailed
|
||||
claimed.EndedAt = nowISO()
|
||||
claimed.Error = err.Error()
|
||||
claimed.UpdatedAt = nowISO()
|
||||
return claimed, runtime.store.UpsertRun(ctx, *claimed)
|
||||
}
|
||||
if err = runtime.attachFrozenReleaseRetry(ctx, claimed); err != nil {
|
||||
claimed.Status = domain.RunStatusFailed
|
||||
claimed.EndedAt = nowISO()
|
||||
claimed.Error = err.Error()
|
||||
claimed.UpdatedAt = nowISO()
|
||||
return claimed, runtime.store.UpsertRun(ctx, *claimed)
|
||||
}
|
||||
if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 {
|
||||
currentRun := claimed
|
||||
message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user