droboticflow/internal/worker/capture_delivery.go

432 lines
17 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package worker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"golang.org/x/sync/errgroup"
)
type deliveryPrefixLister interface {
ListPrefix(context.Context, domain.StorageConnectionRecord, string, string) ([]string, error)
}
// BOS exposes user-metadata through HTTP headers. The SDK canonicalizes header
// names (Sha256, Delivery-Owner), which must remain case-insensitive.
func deliveryMetadata(meta map[string]string, key string) string {
value, found := "", false
for name, candidate := range meta {
if strings.EqualFold(name, key) {
if found && candidate != value {
return ""
}
value, found = candidate, true
}
}
return value
}
func deliveryObjectMatches(meta objectsync.ObjectMetadata, owner, sha string, size int64) bool {
return meta.Exists && meta.SizeBytes == size && deliveryMetadata(meta.UserMeta, "delivery-owner") == owner && deliveryMetadata(meta.UserMeta, "sha256") == sha
}
// Every final object uses conditional create. A different delivery can never
// overwrite it, even when two workers race. Retries accept only identical bytes.
func publishDeliveryObject(ctx context.Context, p objectsync.Publisher, c domain.StorageConnectionRecord, bucket, stage, key, owner, sha string, size int64) error {
old, err := p.Head(ctx, c, bucket, key)
if err != nil {
return err
}
if old.Exists {
if deliveryObjectMatches(old, owner, sha, size) {
return nil
}
return fmt.Errorf("交付目录冲突,拒绝覆盖 %s", key)
}
src, err := p.Head(ctx, c, bucket, stage)
if err != nil {
return err
}
_, err = p.PublishConditional(ctx, c, bucket, stage, key, size, src.ETag, "", "*", map[string]string{"delivery-owner": owner, "sha256": sha})
if err != nil {
// An ambiguous network response or an identical concurrent retry is safe.
got, e := p.Head(ctx, c, bucket, key)
if e == nil && deliveryObjectMatches(got, owner, sha, size) {
return nil
}
return err
}
got, err := p.Head(ctx, c, bucket, key)
if err != nil {
return err
}
if !deliveryObjectMatches(got, owner, sha, size) {
return fmt.Errorf("交付文件校验失败 %s", key)
}
return nil
}
// deliveryTarget is the validated, owner-pinned destination shared by every
// final delivery publication mode.
type deliveryTarget struct {
record *domain.CaptureRecord
target *domain.CaptureDeliveryTarget
connection *domain.StorageConnectionRecord
publisher objectsync.Publisher
lister deliveryPrefixLister
owner string
stageRoot string
marker string
publish func(key string, body []byte) error
}
// prepareDeliveryTarget performs every check that must hold before the first
// byte of a delivery is written: frozen destination, prefix validity, overlap
// with other captures and with enabled intake listeners, delegated scope,
// ownership and the conflict rules of the shared destination prefix.
func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (*deliveryTarget, error) {
record, err := runtime.store.GetCaptureRecordByRunID(ctx, run.ID)
if err != nil {
return nil, err
}
if record == nil || record.Delivery == nil {
return nil, fmt.Errorf("该任务未配置交付目录,请明确配置后重新接入;不会回退到 runs 目录")
}
target := record.Delivery
if !domain.ValidDeliveryPrefix(target.Prefix) {
return nil, fmt.Errorf("无效交付目标目录")
}
otherCaptures, lookupErr := runtime.store.ListCaptureRecords(ctx, run.OrganizationID, "", 0)
if lookupErr != nil {
return nil, lookupErr
}
for _, other := range otherCaptures {
if other.ID != record.ID && other.Delivery != nil && other.Delivery.Bucket == target.Bucket && domain.StoragePrefixesOverlap(other.Delivery.Prefix, target.Prefix) {
return nil, fmt.Errorf("交付目录与已有任务目标重叠:%s;请先处理旧目录绑定,不会合并或覆盖", other.Delivery.Prefix)
}
}
listeners, e := runtime.store.ListEnabledCaptureAutomations(ctx, run.OrganizationID)
if e != nil {
return nil, e
}
for _, workflow := range listeners {
if c := workflow.CaptureAutomation; c != nil && c.Bucket == target.Bucket && domain.StoragePrefixesOverlap(c.Prefix, target.Prefix) {
return nil, fmt.Errorf("交付位置与已启用监听目录重叠,已停止发布")
}
}
connection, err := runtime.captureDeliveryConnection(ctx, run, record, artifact)
if err != nil {
return nil, err
}
publisher := runtime.outputArtifactPublisher()
lister, ok := publisher.(deliveryPrefixLister)
if !ok {
return nil, fmt.Errorf("交付存储不支持目录冲突检查")
}
ownerRunID := record.RunID
// Capture reconciliation may point RunID at its latest release child. The
// delivery owner must stay tied to the immutable accepted source Run.
if run.Trigger == domain.RunTriggerHumanRelease && run.ParentRunID != "" {
ownerRunID = run.ParentRunID
}
owner := record.ID + ":" + ownerRunID
// PutInternal is conditional-create. A failed invocation may leave staging
// objects behind, so each retry needs its own private staging namespace.
stageRoot := "system/runs/" + run.ID + "/delivery-staging/" + nodeID + "/" + domain.NewID("publish")
prepared := &deliveryTarget{
record: record,
target: target,
connection: connection,
publisher: publisher,
lister: lister,
owner: owner,
stageRoot: stageRoot,
marker: target.Prefix + "/.delivery-owner.json",
}
prepared.publish = func(key string, body []byte) error {
sum := sha256.Sum256(body)
sha := hex.EncodeToString(sum[:])
stage := stageRoot + "/" + sha
if _, e := publisher.PutInternal(ctx, *connection, target.Bucket, stage, bytes.NewReader(body), int64(len(body)), "application/json", nil); e != nil {
return e
}
if e := publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, int64(len(body))); e != nil {
return e
}
return publisher.Delete(ctx, *connection, target.Bucket, stage)
}
if err := runtime.recheckV4DeliveryArtifact(ctx, run, artifact); err != nil {
return nil, err
}
known, err := publisher.Head(ctx, *connection, target.Bucket, prepared.marker)
if err != nil {
return nil, err
}
if !known.Exists {
keys, e := prepared.lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
if e != nil {
return nil, e
}
if len(keys) > 0 {
// Refuse to adopt an unowned destination. Part-index migration
// therefore requires the destination to carry its owner marker
// instead of quietly claiming an existing customer directory.
return nil, fmt.Errorf("目标目录已有文件,拒绝覆盖 %s", target.Prefix)
}
}
markerBody, _ := json.Marshal(map[string]string{"owner": owner})
if err = prepared.publish(prepared.marker, markerBody); err != nil {
return nil, err
}
return prepared, nil
}
func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
prepared, err := runtime.prepareDeliveryTarget(ctx, run, nodeID, artifact)
if err != nil {
return artifact, err
}
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
publishBytes := prepared.publish
marker := prepared.marker
paths := []string{}
err = filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, e error) error {
if e != nil {
return e
}
if entry.Type()&os.ModeSymlink != 0 {
return fmt.Errorf("交付目录不允许符号链接")
}
if !entry.IsDir() {
paths = append(paths, path)
}
return nil
})
if err != nil {
return artifact, err
}
sort.Slice(paths, func(i, j int) bool {
a, b := filepath.Base(paths[i]) == "packages-manifest.json", filepath.Base(paths[j]) == "packages-manifest.json"
if a != b {
return !a
}
return paths[i] < paths[j]
})
entries := make([]directoryManifestEntry, len(paths))
publishFile := func(ctx context.Context, i int, path string) error {
relative, e := filepath.Rel(artifact.Path, path)
if e != nil {
return e
}
relative = filepath.ToSlash(relative)
if strings.HasPrefix(relative, "../") || relative == "manifest.json" || relative == ".delivery-owner.json" {
return fmt.Errorf("无效交付成员 %s", relative)
}
stage := prepared.stageRoot + "/files/" + relative
meta, sha, e := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, path)
if e != nil {
return e
}
key := target.Prefix + "/" + relative
if e = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); e != nil {
return e
}
if e = publisher.Delete(ctx, *connection, target.Bucket, stage); e != nil {
return e
}
entries[i] = directoryManifestEntry{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}
return nil
}
parallel := 1
switch publisher.(type) {
case objectsync.BOSPublisher, *objectsync.BOSPublisher:
parallel = 4
}
group, gctx := errgroup.WithContext(ctx)
group.SetLimit(parallel)
for i, path := range paths {
if filepath.Base(path) == "packages-manifest.json" {
continue
}
group.Go(func() error { return publishFile(gctx, i, path) })
}
if e := group.Wait(); e != nil {
return artifact, e
}
// A visible package manifest must not precede its media files.
for i, path := range paths {
if filepath.Base(path) == "packages-manifest.json" {
if e := publishFile(ctx, i, path); e != nil {
return artifact, e
}
}
}
var total int64
for _, entry := range entries {
total += entry.SizeBytes
}
if len(entries) == 0 {
return artifact, fmt.Errorf("交付目录为空")
}
expected := map[string]bool{marker: true, target.Prefix + "/manifest.json": true}
for _, entry := range entries {
expected[entry.Key] = true
}
keys, e := prepared.lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
if e != nil {
return artifact, e
}
for _, key := range keys {
if !expected[key] {
return artifact, fmt.Errorf("目标目录包含非本次交付文件,拒绝完成 %s", key)
}
}
body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "directory", FileCount: len(entries), TotalBytes: total, Entries: entries})
manifestKey := target.Prefix + "/manifest.json"
if err = publishBytes(manifestKey, body); err != nil {
return artifact, err
}
artifact.Path = "bos://" + target.Bucket + "/" + target.Prefix
artifact.SizeBytes = total
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Prefix: target.Prefix, ManifestKey: manifestKey, Kind: domain.ArtifactRefKindDirectory}}
return artifact, nil
}
// partIndexSchemaVersion and partIndexArtifactName are the routing contract
// shared with the package operator. Both sides must move together.
const (
partIndexSchemaVersion = "droboticflow.delivery.part-index/v1"
partIndexArtifactName = "delivery.part-index"
partIndexFileName = "part.json"
)
// partIndexDelivery is the routing block a single-file delivery carries inside
// its own payload, so the worker never has to guess from a file name.
type partIndexDelivery struct {
Kind string `json:"kind"`
FileName string `json:"fileName"`
}
// partIndexPayloadPeek reads only the routing block. A real part index can
// embed megabytes of data, so the probe must read the whole document rather
// than a fixed prefix; the operator bounds the size through the workspace
// file, and the full content is verified by sha256 on upload.
const partIndexMaxPeekBytes = 256 << 20
func partIndexPayloadPeek(path string) (partIndexDelivery, error) {
info, err := os.Stat(path)
if err != nil {
return partIndexDelivery{}, err
}
if !info.Mode().IsRegular() {
return partIndexDelivery{}, fmt.Errorf("part index payload is not a regular file")
}
if info.Size() > partIndexMaxPeekBytes {
return partIndexDelivery{}, fmt.Errorf("part index payload exceeds %d bytes", partIndexMaxPeekBytes)
}
body, err := os.ReadFile(path)
if err != nil {
return partIndexDelivery{}, err
}
var probe struct {
SchemaVersion string `json:"schemaVersion"`
Delivery *partIndexDelivery `json:"delivery"`
}
if err = json.Unmarshal(body, &probe); err != nil {
return partIndexDelivery{}, fmt.Errorf("part index payload is not readable JSON: %w", err)
}
if probe.Delivery == nil || probe.Delivery.Kind != "part_index" {
return partIndexDelivery{}, fmt.Errorf("part index payload is missing its delivery routing block")
}
if probe.SchemaVersion != partIndexSchemaVersion && probe.SchemaVersion != "droboticflow.delivery.part-index/v2" {
return partIndexDelivery{}, fmt.Errorf("unsupported part index schema %q", probe.SchemaVersion)
}
name := filepath.Base(strings.TrimSpace(probe.Delivery.FileName))
if name == "" || name == "." || name == ".." || name != probe.Delivery.FileName {
return partIndexDelivery{}, fmt.Errorf("part index payload declares an unsafe file name")
}
return *probe.Delivery, nil
}
// mergePartIndexIntoManifest builds the deterministic completion manifest body
// for a single-file part delivery. The body depends only on the delivered
// member, so publishBytes' conditional create already distinguishes the two
// legitimate existing cases (identical retry → accepted; anything else,
// including a legacy directory manifest or a foreign owner → rejected with a
// conflict). No in-place manifest mutation is performed: the worker publisher
// is write-only, and a completion marker that already describes a delivered
// directory tree must never be rewritten into a single-file manifest.
func mergePartIndexIntoManifest(_ context.Context, _ objectsync.Publisher, _ domain.StorageConnectionRecord, _ string, _ string, _ string, entries []directoryManifestEntry) ([]byte, error) {
return json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "part-index", FileCount: len(entries), TotalBytes: sumManifestBytes(entries), Entries: entries})
}
func sumManifestBytes(entries []directoryManifestEntry) int64 {
total := int64(0)
for _, entry := range entries {
total += entry.SizeBytes
}
return total
}
// publishCaptureDeliveryFile places a single file into the frozen part
// destination. It shares every protection of the directory publication — one
// owner marker per destination, conditional create, sha256/size verification,
// idempotent retry and "never overwrite another delivery" — but writes one
// member plus the shared completion manifest instead of a tree.
func (runtime *Runtime) publishCaptureDeliveryFile(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
info, err := os.Lstat(artifact.Path)
if err != nil {
return artifact, err
}
if !info.Mode().IsRegular() {
return artifact, fmt.Errorf("part index delivery requires a regular file")
}
routing, err := partIndexPayloadPeek(artifact.Path)
if err != nil {
return artifact, err
}
prepared, err := runtime.prepareDeliveryTarget(ctx, run, nodeID, artifact)
if err != nil {
return artifact, err
}
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
publishBytes := prepared.publish
fileName := filepath.Base(routing.FileName)
relative := fileName
stage := prepared.stageRoot + "/files/" + relative
meta, sha, err := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, artifact.Path)
if err != nil {
return artifact, err
}
key := target.Prefix + "/" + relative
if err = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); err != nil {
return artifact, err
}
if err = publisher.Delete(ctx, *connection, target.Bucket, stage); err != nil {
return artifact, err
}
entries := []directoryManifestEntry{{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}}
manifestKey := target.Prefix + "/manifest.json"
body, err := mergePartIndexIntoManifest(ctx, publisher, *connection, target.Bucket, manifestKey, owner, entries)
if err != nil {
return artifact, err
}
if err = publishBytes(manifestKey, body); err != nil {
return artifact, err
}
artifact.Path = "bos://" + target.Bucket + "/" + key
artifact.SizeBytes = meta.SizeBytes
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Key: key, ManifestKey: manifestKey, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject}}
return artifact, nil
}