forked from longtao.wu/droboticflow
432 lines
17 KiB
Go
432 lines
17 KiB
Go
package worker
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
|
||
"droboticflow/internal/domain"
|
||
"droboticflow/internal/objectsync"
|
||
"golang.org/x/sync/errgroup"
|
||
)
|
||
|
||
type deliveryPrefixLister interface {
|
||
ListPrefix(context.Context, domain.StorageConnectionRecord, string, string) ([]string, error)
|
||
}
|
||
|
||
// BOS exposes user-metadata through HTTP headers. The SDK canonicalizes header
|
||
// names (Sha256, Delivery-Owner), which must remain case-insensitive.
|
||
func deliveryMetadata(meta map[string]string, key string) string {
|
||
value, found := "", false
|
||
for name, candidate := range meta {
|
||
if strings.EqualFold(name, key) {
|
||
if found && candidate != value {
|
||
return ""
|
||
}
|
||
value, found = candidate, true
|
||
}
|
||
}
|
||
return value
|
||
}
|
||
|
||
func deliveryObjectMatches(meta objectsync.ObjectMetadata, owner, sha string, size int64) bool {
|
||
return meta.Exists && meta.SizeBytes == size && deliveryMetadata(meta.UserMeta, "delivery-owner") == owner && deliveryMetadata(meta.UserMeta, "sha256") == sha
|
||
}
|
||
|
||
// Every final object uses conditional create. A different delivery can never
|
||
// overwrite it, even when two workers race. Retries accept only identical bytes.
|
||
func publishDeliveryObject(ctx context.Context, p objectsync.Publisher, c domain.StorageConnectionRecord, bucket, stage, key, owner, sha string, size int64) error {
|
||
old, err := p.Head(ctx, c, bucket, key)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if old.Exists {
|
||
if deliveryObjectMatches(old, owner, sha, size) {
|
||
return nil
|
||
}
|
||
return fmt.Errorf("交付目录冲突,拒绝覆盖 %s", key)
|
||
}
|
||
src, err := p.Head(ctx, c, bucket, stage)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
_, err = p.PublishConditional(ctx, c, bucket, stage, key, size, src.ETag, "", "*", map[string]string{"delivery-owner": owner, "sha256": sha})
|
||
if err != nil {
|
||
// An ambiguous network response or an identical concurrent retry is safe.
|
||
got, e := p.Head(ctx, c, bucket, key)
|
||
if e == nil && deliveryObjectMatches(got, owner, sha, size) {
|
||
return nil
|
||
}
|
||
return err
|
||
}
|
||
got, err := p.Head(ctx, c, bucket, key)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !deliveryObjectMatches(got, owner, sha, size) {
|
||
return fmt.Errorf("交付文件校验失败 %s", key)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// deliveryTarget is the validated, owner-pinned destination shared by every
|
||
// final delivery publication mode.
|
||
type deliveryTarget struct {
|
||
record *domain.CaptureRecord
|
||
target *domain.CaptureDeliveryTarget
|
||
connection *domain.StorageConnectionRecord
|
||
publisher objectsync.Publisher
|
||
lister deliveryPrefixLister
|
||
owner string
|
||
stageRoot string
|
||
marker string
|
||
publish func(key string, body []byte) error
|
||
}
|
||
|
||
// prepareDeliveryTarget performs every check that must hold before the first
|
||
// byte of a delivery is written: frozen destination, prefix validity, overlap
|
||
// with other captures and with enabled intake listeners, delegated scope,
|
||
// ownership and the conflict rules of the shared destination prefix.
|
||
func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (*deliveryTarget, error) {
|
||
record, err := runtime.store.GetCaptureRecordByRunID(ctx, run.ID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if record == nil || record.Delivery == nil {
|
||
return nil, fmt.Errorf("该任务未配置交付目录,请明确配置后重新接入;不会回退到 runs 目录")
|
||
}
|
||
target := record.Delivery
|
||
if !domain.ValidDeliveryPrefix(target.Prefix) {
|
||
return nil, fmt.Errorf("无效交付目标目录")
|
||
}
|
||
otherCaptures, lookupErr := runtime.store.ListCaptureRecords(ctx, run.OrganizationID, "", 0)
|
||
if lookupErr != nil {
|
||
return nil, lookupErr
|
||
}
|
||
for _, other := range otherCaptures {
|
||
if other.ID != record.ID && other.Delivery != nil && other.Delivery.Bucket == target.Bucket && domain.StoragePrefixesOverlap(other.Delivery.Prefix, target.Prefix) {
|
||
return nil, fmt.Errorf("交付目录与已有任务目标重叠:%s;请先处理旧目录绑定,不会合并或覆盖", other.Delivery.Prefix)
|
||
}
|
||
}
|
||
listeners, e := runtime.store.ListEnabledCaptureAutomations(ctx, run.OrganizationID)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
for _, workflow := range listeners {
|
||
if c := workflow.CaptureAutomation; c != nil && c.Bucket == target.Bucket && domain.StoragePrefixesOverlap(c.Prefix, target.Prefix) {
|
||
return nil, fmt.Errorf("交付位置与已启用监听目录重叠,已停止发布")
|
||
}
|
||
}
|
||
connection, err := runtime.captureDeliveryConnection(ctx, run, record, artifact)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
publisher := runtime.outputArtifactPublisher()
|
||
lister, ok := publisher.(deliveryPrefixLister)
|
||
if !ok {
|
||
return nil, fmt.Errorf("交付存储不支持目录冲突检查")
|
||
}
|
||
ownerRunID := record.RunID
|
||
// Capture reconciliation may point RunID at its latest release child. The
|
||
// delivery owner must stay tied to the immutable accepted source Run.
|
||
if run.Trigger == domain.RunTriggerHumanRelease && run.ParentRunID != "" {
|
||
ownerRunID = run.ParentRunID
|
||
}
|
||
owner := record.ID + ":" + ownerRunID
|
||
// PutInternal is conditional-create. A failed invocation may leave staging
|
||
// objects behind, so each retry needs its own private staging namespace.
|
||
stageRoot := "system/runs/" + run.ID + "/delivery-staging/" + nodeID + "/" + domain.NewID("publish")
|
||
prepared := &deliveryTarget{
|
||
record: record,
|
||
target: target,
|
||
connection: connection,
|
||
publisher: publisher,
|
||
lister: lister,
|
||
owner: owner,
|
||
stageRoot: stageRoot,
|
||
marker: target.Prefix + "/.delivery-owner.json",
|
||
}
|
||
prepared.publish = func(key string, body []byte) error {
|
||
sum := sha256.Sum256(body)
|
||
sha := hex.EncodeToString(sum[:])
|
||
stage := stageRoot + "/" + sha
|
||
if _, e := publisher.PutInternal(ctx, *connection, target.Bucket, stage, bytes.NewReader(body), int64(len(body)), "application/json", nil); e != nil {
|
||
return e
|
||
}
|
||
if e := publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, int64(len(body))); e != nil {
|
||
return e
|
||
}
|
||
return publisher.Delete(ctx, *connection, target.Bucket, stage)
|
||
}
|
||
if err := runtime.recheckV4DeliveryArtifact(ctx, run, artifact); err != nil {
|
||
return nil, err
|
||
}
|
||
known, err := publisher.Head(ctx, *connection, target.Bucket, prepared.marker)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !known.Exists {
|
||
keys, e := prepared.lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
if len(keys) > 0 {
|
||
// Refuse to adopt an unowned destination. Part-index migration
|
||
// therefore requires the destination to carry its owner marker
|
||
// instead of quietly claiming an existing customer directory.
|
||
return nil, fmt.Errorf("目标目录已有文件,拒绝覆盖 %s", target.Prefix)
|
||
}
|
||
}
|
||
markerBody, _ := json.Marshal(map[string]string{"owner": owner})
|
||
if err = prepared.publish(prepared.marker, markerBody); err != nil {
|
||
return nil, err
|
||
}
|
||
return prepared, nil
|
||
}
|
||
|
||
func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
|
||
prepared, err := runtime.prepareDeliveryTarget(ctx, run, nodeID, artifact)
|
||
if err != nil {
|
||
return artifact, err
|
||
}
|
||
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
|
||
publishBytes := prepared.publish
|
||
marker := prepared.marker
|
||
paths := []string{}
|
||
err = filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, e error) error {
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if entry.Type()&os.ModeSymlink != 0 {
|
||
return fmt.Errorf("交付目录不允许符号链接")
|
||
}
|
||
if !entry.IsDir() {
|
||
paths = append(paths, path)
|
||
}
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
return artifact, err
|
||
}
|
||
sort.Slice(paths, func(i, j int) bool {
|
||
a, b := filepath.Base(paths[i]) == "packages-manifest.json", filepath.Base(paths[j]) == "packages-manifest.json"
|
||
if a != b {
|
||
return !a
|
||
}
|
||
return paths[i] < paths[j]
|
||
})
|
||
entries := make([]directoryManifestEntry, len(paths))
|
||
publishFile := func(ctx context.Context, i int, path string) error {
|
||
relative, e := filepath.Rel(artifact.Path, path)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
relative = filepath.ToSlash(relative)
|
||
if strings.HasPrefix(relative, "../") || relative == "manifest.json" || relative == ".delivery-owner.json" {
|
||
return fmt.Errorf("无效交付成员 %s", relative)
|
||
}
|
||
stage := prepared.stageRoot + "/files/" + relative
|
||
meta, sha, e := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, path)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
key := target.Prefix + "/" + relative
|
||
if e = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); e != nil {
|
||
return e
|
||
}
|
||
if e = publisher.Delete(ctx, *connection, target.Bucket, stage); e != nil {
|
||
return e
|
||
}
|
||
entries[i] = directoryManifestEntry{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}
|
||
return nil
|
||
}
|
||
parallel := 1
|
||
switch publisher.(type) {
|
||
case objectsync.BOSPublisher, *objectsync.BOSPublisher:
|
||
parallel = 4
|
||
}
|
||
group, gctx := errgroup.WithContext(ctx)
|
||
group.SetLimit(parallel)
|
||
for i, path := range paths {
|
||
if filepath.Base(path) == "packages-manifest.json" {
|
||
continue
|
||
}
|
||
group.Go(func() error { return publishFile(gctx, i, path) })
|
||
}
|
||
if e := group.Wait(); e != nil {
|
||
return artifact, e
|
||
}
|
||
// A visible package manifest must not precede its media files.
|
||
for i, path := range paths {
|
||
if filepath.Base(path) == "packages-manifest.json" {
|
||
if e := publishFile(ctx, i, path); e != nil {
|
||
return artifact, e
|
||
}
|
||
}
|
||
}
|
||
var total int64
|
||
for _, entry := range entries {
|
||
total += entry.SizeBytes
|
||
}
|
||
if len(entries) == 0 {
|
||
return artifact, fmt.Errorf("交付目录为空")
|
||
}
|
||
expected := map[string]bool{marker: true, target.Prefix + "/manifest.json": true}
|
||
for _, entry := range entries {
|
||
expected[entry.Key] = true
|
||
}
|
||
keys, e := prepared.lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
|
||
if e != nil {
|
||
return artifact, e
|
||
}
|
||
for _, key := range keys {
|
||
if !expected[key] {
|
||
return artifact, fmt.Errorf("目标目录包含非本次交付文件,拒绝完成 %s", key)
|
||
}
|
||
}
|
||
body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "directory", FileCount: len(entries), TotalBytes: total, Entries: entries})
|
||
manifestKey := target.Prefix + "/manifest.json"
|
||
if err = publishBytes(manifestKey, body); err != nil {
|
||
return artifact, err
|
||
}
|
||
artifact.Path = "bos://" + target.Bucket + "/" + target.Prefix
|
||
artifact.SizeBytes = total
|
||
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Prefix: target.Prefix, ManifestKey: manifestKey, Kind: domain.ArtifactRefKindDirectory}}
|
||
return artifact, nil
|
||
}
|
||
|
||
// partIndexSchemaVersion and partIndexArtifactName are the routing contract
|
||
// shared with the package operator. Both sides must move together.
|
||
const (
|
||
partIndexSchemaVersion = "droboticflow.delivery.part-index/v1"
|
||
partIndexArtifactName = "delivery.part-index"
|
||
partIndexFileName = "part.json"
|
||
)
|
||
|
||
// partIndexDelivery is the routing block a single-file delivery carries inside
|
||
// its own payload, so the worker never has to guess from a file name.
|
||
type partIndexDelivery struct {
|
||
Kind string `json:"kind"`
|
||
FileName string `json:"fileName"`
|
||
}
|
||
|
||
// partIndexPayloadPeek reads only the routing block. A real part index can
|
||
// embed megabytes of data, so the probe must read the whole document rather
|
||
// than a fixed prefix; the operator bounds the size through the workspace
|
||
// file, and the full content is verified by sha256 on upload.
|
||
const partIndexMaxPeekBytes = 256 << 20
|
||
|
||
func partIndexPayloadPeek(path string) (partIndexDelivery, error) {
|
||
info, err := os.Stat(path)
|
||
if err != nil {
|
||
return partIndexDelivery{}, err
|
||
}
|
||
if !info.Mode().IsRegular() {
|
||
return partIndexDelivery{}, fmt.Errorf("part index payload is not a regular file")
|
||
}
|
||
if info.Size() > partIndexMaxPeekBytes {
|
||
return partIndexDelivery{}, fmt.Errorf("part index payload exceeds %d bytes", partIndexMaxPeekBytes)
|
||
}
|
||
body, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return partIndexDelivery{}, err
|
||
}
|
||
var probe struct {
|
||
SchemaVersion string `json:"schemaVersion"`
|
||
Delivery *partIndexDelivery `json:"delivery"`
|
||
}
|
||
if err = json.Unmarshal(body, &probe); err != nil {
|
||
return partIndexDelivery{}, fmt.Errorf("part index payload is not readable JSON: %w", err)
|
||
}
|
||
if probe.Delivery == nil || probe.Delivery.Kind != "part_index" {
|
||
return partIndexDelivery{}, fmt.Errorf("part index payload is missing its delivery routing block")
|
||
}
|
||
if probe.SchemaVersion != partIndexSchemaVersion && probe.SchemaVersion != "droboticflow.delivery.part-index/v2" {
|
||
return partIndexDelivery{}, fmt.Errorf("unsupported part index schema %q", probe.SchemaVersion)
|
||
}
|
||
name := filepath.Base(strings.TrimSpace(probe.Delivery.FileName))
|
||
if name == "" || name == "." || name == ".." || name != probe.Delivery.FileName {
|
||
return partIndexDelivery{}, fmt.Errorf("part index payload declares an unsafe file name")
|
||
}
|
||
return *probe.Delivery, nil
|
||
}
|
||
|
||
// mergePartIndexIntoManifest builds the deterministic completion manifest body
|
||
// for a single-file part delivery. The body depends only on the delivered
|
||
// member, so publishBytes' conditional create already distinguishes the two
|
||
// legitimate existing cases (identical retry → accepted; anything else,
|
||
// including a legacy directory manifest or a foreign owner → rejected with a
|
||
// conflict). No in-place manifest mutation is performed: the worker publisher
|
||
// is write-only, and a completion marker that already describes a delivered
|
||
// directory tree must never be rewritten into a single-file manifest.
|
||
func mergePartIndexIntoManifest(_ context.Context, _ objectsync.Publisher, _ domain.StorageConnectionRecord, _ string, _ string, _ string, entries []directoryManifestEntry) ([]byte, error) {
|
||
return json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "part-index", FileCount: len(entries), TotalBytes: sumManifestBytes(entries), Entries: entries})
|
||
}
|
||
|
||
func sumManifestBytes(entries []directoryManifestEntry) int64 {
|
||
total := int64(0)
|
||
for _, entry := range entries {
|
||
total += entry.SizeBytes
|
||
}
|
||
return total
|
||
}
|
||
|
||
// publishCaptureDeliveryFile places a single file into the frozen part
|
||
// destination. It shares every protection of the directory publication — one
|
||
// owner marker per destination, conditional create, sha256/size verification,
|
||
// idempotent retry and "never overwrite another delivery" — but writes one
|
||
// member plus the shared completion manifest instead of a tree.
|
||
func (runtime *Runtime) publishCaptureDeliveryFile(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
|
||
info, err := os.Lstat(artifact.Path)
|
||
if err != nil {
|
||
return artifact, err
|
||
}
|
||
if !info.Mode().IsRegular() {
|
||
return artifact, fmt.Errorf("part index delivery requires a regular file")
|
||
}
|
||
routing, err := partIndexPayloadPeek(artifact.Path)
|
||
if err != nil {
|
||
return artifact, err
|
||
}
|
||
prepared, err := runtime.prepareDeliveryTarget(ctx, run, nodeID, artifact)
|
||
if err != nil {
|
||
return artifact, err
|
||
}
|
||
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
|
||
publishBytes := prepared.publish
|
||
fileName := filepath.Base(routing.FileName)
|
||
relative := fileName
|
||
stage := prepared.stageRoot + "/files/" + relative
|
||
meta, sha, err := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, artifact.Path)
|
||
if err != nil {
|
||
return artifact, err
|
||
}
|
||
key := target.Prefix + "/" + relative
|
||
if err = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); err != nil {
|
||
return artifact, err
|
||
}
|
||
if err = publisher.Delete(ctx, *connection, target.Bucket, stage); err != nil {
|
||
return artifact, err
|
||
}
|
||
entries := []directoryManifestEntry{{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}}
|
||
manifestKey := target.Prefix + "/manifest.json"
|
||
body, err := mergePartIndexIntoManifest(ctx, publisher, *connection, target.Bucket, manifestKey, owner, entries)
|
||
if err != nil {
|
||
return artifact, err
|
||
}
|
||
if err = publishBytes(manifestKey, body); err != nil {
|
||
return artifact, err
|
||
}
|
||
artifact.Path = "bos://" + target.Bucket + "/" + key
|
||
artifact.SizeBytes = meta.SizeBytes
|
||
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Key: key, ManifestKey: manifestKey, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject}}
|
||
return artifact, nil
|
||
}
|