droboticflow/internal/worker/artifact_storage.go

564 lines
21 KiB
Go

package worker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"mime"
"os"
"path/filepath"
"sort"
"strings"
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"github.com/baidubce/bce-sdk-go/services/bos"
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
)
const defaultArtifactPrefixTemplate = "artifacts/{organizationId}/{projectId}/{workflowId}/{runId}/{nodeId}"
type directoryManifestEntry struct {
Path string `json:"path"`
Key string `json:"key"`
SizeBytes int64 `json:"sizeBytes"`
SHA256 string `json:"sha256,omitempty"`
}
type directoryManifest struct {
SchemaVersion int `json:"schemaVersion"`
Kind string `json:"kind"`
Source map[string]string `json:"source,omitempty"`
FileCount int `json:"fileCount"`
TotalBytes int64 `json:"totalBytes"`
Entries []directoryManifestEntry `json:"entries"`
}
func compactArtifactData(data map[string]any) map[string]any {
if len(data) == 0 {
return nil
}
compact := map[string]any{}
// WiLoR reports its count inside result. Keep a scalar projection when
// removing the full frame stream so downstream task summaries remain exact.
if data["schemaVersion"] == "droboticflow.wilor.hand_pose_result/v1" {
if result, ok := data["result"].(map[string]any); ok && result["frameCount"] != nil {
compact["frameCount"] = result["frameCount"]
} else if frames, ok := data["frames"].([]any); ok {
compact["frameCount"] = len(frames)
}
}
for key, value := range data {
if key == "outcome" || key == "deliverySelection" || key == "acceptedMaterializations" {
compact[key] = value
continue
}
// These fields are required by downstream port-aware adapters (notably
// virtual Qwen clips and human-release packaging) but do not themselves
// contain a storage locator. Response sanitisation still removes any
// nested source/ref/path values before they reach the browser.
switch key {
case "preview", "reportSummary", "dataRef", "count", "schemaVersion", "datasetType", "kind", "clipId", "clipName", "sourceIsFullVideo", "startSecond", "endSecond", "sourceDurationSeconds", "behavior", "objectChange", "rawPTS", "annotation", "collectionId", "cameraSide", "role", "transformChain", "privacyTransform", "rectification":
compact[key] = value
continue
}
if strings.HasSuffix(key, "Count") {
compact[key] = value
}
}
if len(compact) == 0 {
return nil
}
return compact
}
// structuredJSONRequiresReference keeps the transport contract semantic rather
// than size-based. A small manifest carrying a path, BOS reference, dataset or
// collection is still external data and must never become a payload.inline.
func structuredJSONRequiresReference(artifact domain.ExecutionArtifact, value any) bool {
name := strings.ToLower(strings.TrimSpace(artifact.Name))
if strings.Contains(name, "manifest") || strings.Contains(name, "dataset") {
return true
}
if data := artifact.Data; len(data) > 0 {
if _, ok := data["dataRef"]; ok {
return true
}
if structuredValueHasExternalReference(data) {
return true
}
}
return structuredValueHasExternalReference(value)
}
func structuredValueHasExternalReference(value any) bool {
switch typed := value.(type) {
case map[string]any:
for key, child := range typed {
normalized := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(key), "_", ""), "-", ""))
switch normalized {
case "dataref", "storagepath", "previewurl", "manifestkey", "connectionid", "bucket", "prefix", "relativepath", "clippath", "filepath", "payloadpath", "partitiondir", "ref":
return true
case "entries", "collections", "datasets", "datarefs", "directory", "files":
return true
}
if structuredValueHasExternalReference(child) {
return true
}
}
case []any:
for _, child := range typed {
if structuredValueHasExternalReference(child) {
return true
}
}
}
return false
}
func (runtime *Runtime) outputArtifactPublisher() objectsync.Publisher {
if runtime.artifactPublisher != nil {
return runtime.artifactPublisher
}
return objectsync.BOSPublisher{GlobalEndpoint: runtime.globalBOSEndpoint}
}
func normalizeArtifactKey(value string) string {
parts := strings.FieldsFunc(strings.TrimSpace(value), func(r rune) bool {
return r == '\\' || r == '/'
})
cleaned := make([]string, 0, len(parts))
for _, part := range parts {
part = artifactSlug(part)
if part != "" && part != "." && part != ".." {
cleaned = append(cleaned, part)
}
}
return strings.Join(cleaned, "/")
}
func artifactSlug(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "artifact"
}
var builder strings.Builder
for _, r := range value {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '-':
builder.WriteRune(r)
default:
builder.WriteRune('-')
}
}
return strings.Trim(builder.String(), "-")
}
func artifactPrefix(policy domain.ArtifactStoragePolicy, run *domain.RunRecord, nodeID string) string {
template := strings.Trim(strings.TrimSpace(policy.PrefixTemplate), "/")
if template == "" {
template = defaultArtifactPrefixTemplate
}
replacements := map[string]string{
"{organizationId}": run.OrganizationID,
"{projectId}": run.ProjectID,
"{workflowId}": run.WorkflowID,
"{runId}": run.ID,
"{nodeId}": nodeID,
}
for token, value := range replacements {
template = strings.ReplaceAll(template, token, artifactSlug(value))
}
return normalizeArtifactKey(template)
}
func (runtime *Runtime) artifactStoragePolicy(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.ArtifactStoragePolicy, error) {
if workflow != nil && workflow.ArtifactStorage != nil && strings.TrimSpace(workflow.ArtifactStorage.ConnectionID) != "" {
copy := *workflow.ArtifactStorage
return &copy, nil
}
organizationID := ""
if workflow != nil {
organizationID = workflow.OrganizationID
}
settings, err := runtime.store.GetArtifactStorageSettings(ctx, organizationID)
if err != nil {
return nil, err
}
if strings.TrimSpace(settings.ConnectionID) == "" || strings.TrimSpace(settings.Bucket) == "" {
return nil, nil
}
return &domain.ArtifactStoragePolicy{
ConnectionID: settings.ConnectionID,
Bucket: settings.Bucket,
PrefixTemplate: settings.PrefixTemplate,
}, nil
}
func (runtime *Runtime) artifactStorageConnection(ctx context.Context, connectionID string, organizationID string) (*domain.StorageConnectionRecord, error) {
connection, err := runtime.store.GetStorageConnection(ctx, strings.TrimSpace(connectionID))
if err != nil {
return nil, err
}
if connection == nil {
return nil, fmt.Errorf("artifact storage connection %s not found", connectionID)
}
if strings.TrimSpace(connection.OrganizationID) != "" && strings.TrimSpace(organizationID) != "" && connection.OrganizationID != organizationID {
return nil, fmt.Errorf("artifact storage connection belongs to another organization")
}
copy := *connection
if strings.TrimSpace(copy.Endpoint) == "" {
copy.Endpoint = runtime.globalBOSEndpoint
}
return &copy, nil
}
func fileSHA256(path string) (string, error) {
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
hash := sha256.New()
if _, err := io.Copy(hash, file); err != nil {
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func putArtifactFile(ctx context.Context, publisher objectsync.Publisher, connection domain.StorageConnectionRecord, bucket, key, path string) (objectsync.ObjectMetadata, string, error) {
info, err := os.Stat(path)
if err != nil {
return objectsync.ObjectMetadata{}, "", err
}
sha, err := fileSHA256(path)
if err != nil {
return objectsync.ObjectMetadata{}, "", err
}
file, err := os.Open(path)
if err != nil {
return objectsync.ObjectMetadata{}, "", err
}
defer file.Close()
contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(path)))
if contentType == "" {
contentType = "application/octet-stream"
}
meta, err := publisher.PutInternal(ctx, connection, bucket, key, file, info.Size(), contentType, map[string]string{"sha256": sha})
return meta, sha, err
}
func (runtime *Runtime) externalizeDirectory(ctx context.Context, publisher objectsync.Publisher, connection domain.StorageConnectionRecord, policy domain.ArtifactStoragePolicy, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, map[string]domain.ArtifactReference, error) {
basePrefix := artifactPrefix(policy, run, nodeID) + "/" + artifactSlug(artifact.ID)
entries := make([]directoryManifestEntry, 0)
sharedRefs := map[string]domain.ArtifactReference{}
var totalBytes int64
err := filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
relative, err := filepath.Rel(artifact.Path, path)
if err != nil {
return err
}
key := basePrefix + "/files/" + normalizeArtifactKey(filepath.ToSlash(relative))
meta, sha, err := putArtifactFile(ctx, publisher, connection, policy.Bucket, key, path)
if err != nil {
return err
}
entries = append(entries, directoryManifestEntry{Path: filepath.ToSlash(relative), Key: key, SizeBytes: meta.SizeBytes, SHA256: sha})
sharedRefs[filepath.Clean(path)] = domain.ArtifactReference{
Provider: connection.Provider, ConnectionID: policy.ConnectionID, Bucket: policy.Bucket, Key: key,
VersionID: meta.VersionID, ETag: meta.ETag, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject,
}
totalBytes += meta.SizeBytes
return nil
})
if err != nil {
return artifact, nil, err
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path })
manifest := directoryManifest{SchemaVersion: 1, Kind: "directory", FileCount: len(entries), TotalBytes: totalBytes, Entries: entries}
payload, err := json.Marshal(manifest)
if err != nil {
return artifact, nil, err
}
manifestKey := basePrefix + "/manifest.json"
meta, err := publisher.PutInternal(ctx, connection, policy.Bucket, manifestKey, bytes.NewReader(payload), int64(len(payload)), "application/json", nil)
if err != nil {
return artifact, nil, err
}
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: policy.ConnectionID, Bucket: policy.Bucket, Prefix: basePrefix + "/files", ManifestKey: manifestKey, VersionID: meta.VersionID, ETag: meta.ETag, SizeBytes: totalBytes, Kind: domain.ArtifactRefKindDirectory}
artifact.Path = "bos://" + policy.Bucket + "/" + ref.Prefix
artifact.SizeBytes = totalBytes
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref}
artifact.Data = compactArtifactData(artifact.Data)
return artifact, sharedRefs, nil
}
func (runtime *Runtime) normalizeOutputArtifacts(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord, nodeID string, artifacts []domain.ExecutionArtifact) ([]domain.ExecutionArtifact, error) {
policy, err := runtime.artifactStoragePolicy(ctx, workflow)
if err != nil {
return nil, err
}
var connection *domain.StorageConnectionRecord
var publisher objectsync.Publisher
if policy != nil {
connection, err = runtime.artifactStorageConnection(ctx, policy.ConnectionID, run.OrganizationID)
if err != nil {
return nil, err
}
publisher = runtime.outputArtifactPublisher()
}
result := make([]domain.ExecutionArtifact, len(artifacts))
// A package directory and its visible MP4/JSON members are often returned
// together. Externalise the directory once, then bind member artifacts to
// the exact already-uploaded object refs instead of copying every byte a
// second time.
sharedDirectoryFileRefs := map[string]domain.ArtifactReference{}
order := make([]int, 0, len(artifacts))
for index, artifact := range artifacts {
if domain.IsInternalDeliveryAudit(artifact) {
order = append(order, index)
}
}
for index, artifact := range artifacts {
if !domain.IsInternalDeliveryAudit(artifact) {
order = append(order, index)
}
}
for _, index := range order {
artifact := artifacts[index]
result[index] = artifact
if artifact.Payload != nil && artifact.Payload.Mode == domain.ArtifactTransferModeRef {
if err := domain.ValidateArtifactPayload(*artifact.Payload, artifact.Type); err != nil {
return nil, fmt.Errorf("artifact_ref_invalid: %w", err)
}
ref := artifact.Payload.Ref
if ref.Kind == domain.ArtifactRefKindDirectory && strings.TrimSpace(ref.ManifestKey) == "" {
return nil, fmt.Errorf("manifest_incomplete: directory artifact ref requires manifestKey")
}
refConnection, err := runtime.artifactStorageConnection(ctx, ref.ConnectionID, run.OrganizationID)
if err != nil {
return nil, fmt.Errorf("artifact_ref_invalid: %w", err)
}
checkKey := ref.Key
if ref.Kind == domain.ArtifactRefKindDirectory {
checkKey = ref.ManifestKey
}
meta, err := runtime.outputArtifactPublisher().Head(ctx, *refConnection, ref.Bucket, checkKey)
if err != nil || !meta.Exists {
if err == nil {
err = fmt.Errorf("object does not exist")
}
return nil, fmt.Errorf("artifact_ref_invalid: %w", err)
}
continue
}
info, statErr := os.Stat(artifact.Path)
// A part-index delivery is one file, not a tree. It runs before the
// generic inline/upload paths so the frozen capture destination and the
// shared conflict rules always win over the run's artifact policy.
if statErr == nil && info.Mode().IsRegular() && artifact.Name == partIndexFileName {
if _, e := partIndexPayloadPeek(artifact.Path); e == nil {
delivered, e := runtime.publishCaptureDeliveryFile(ctx, run, nodeID, artifact)
if e != nil {
return nil, e
}
result[index] = delivered
continue
}
}
if statErr == nil && info.Mode().IsRegular() && artifact.Type == domain.PortTypeJSON && info.Size() <= domain.MaxInlinePayloadBytes {
payloadBytes, readErr := os.ReadFile(artifact.Path)
if readErr != nil {
return nil, readErr
}
var inline any
if json.Unmarshal(payloadBytes, &inline) == nil && !structuredJSONRequiresReference(artifact, inline) {
result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: inline}
continue
}
}
if statErr != nil && strings.HasPrefix(strings.ToLower(strings.TrimSpace(artifact.Path)), "bos://") {
location := strings.TrimPrefix(strings.TrimSpace(artifact.Path), "bos://")
parts := strings.SplitN(location, "/", 2)
if len(parts) == 2 && strings.TrimSpace(parts[0]) != "" && strings.TrimSpace(parts[1]) != "" {
connectionID := ""
if workflow != nil {
if node := domain.GetNodeByID(workflow.Nodes, nodeID); node != nil {
connectionID = strings.TrimSpace(node.Data.ConnectionID)
}
}
if connectionID == "" && policy != nil {
connectionID = policy.ConnectionID
}
if connectionID != "" {
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: connectionID, Bucket: parts[0], Key: parts[1], SizeBytes: artifact.SizeBytes, Kind: domain.ArtifactRefKindObject}
result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref}
result[index].Data = compactArtifactData(artifact.Data)
continue
}
}
}
if statErr == nil && info.IsDir() && artifact.Name == "delivery.directory.index" {
delivered, e := runtime.publishCaptureDelivery(ctx, run, nodeID, artifact)
if e != nil {
return nil, e
}
result[index] = delivered
continue
}
if policy == nil {
continue // Draft/legacy workflows retain workspace-path compatibility.
}
if statErr != nil {
if workflow != nil && workflow.DefinitionState == domain.WorkflowDefinitionStateReleased {
return nil, fmt.Errorf("artifact_externalization_failed: artifact path %s is not accessible to the worker and no durable ref was provided", artifact.Path)
}
continue
}
if info.Mode().IsRegular() {
if shared, exists := sharedDirectoryFileRefs[filepath.Clean(artifact.Path)]; exists {
ref := shared
result[index].Path = "bos://" + ref.Bucket + "/" + ref.Key
result[index].SizeBytes = ref.SizeBytes
result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &ref}
result[index].Data = compactArtifactData(artifact.Data)
continue
}
}
if info.IsDir() {
externalized, refs, externalizeErr := runtime.externalizeDirectory(ctx, publisher, *connection, *policy, run, nodeID, artifact)
if externalizeErr != nil {
return nil, fmt.Errorf("artifact_externalization_failed: %w", externalizeErr)
}
result[index] = externalized
for path, ref := range refs {
sharedDirectoryFileRefs[path] = ref
}
continue
}
key := artifactPrefix(*policy, run, nodeID) + "/" + artifactSlug(artifact.ID) + "/" + artifactSlug(filepath.Base(artifact.Path))
meta, sha, uploadErr := putArtifactFile(ctx, publisher, *connection, policy.Bucket, key, artifact.Path)
if uploadErr != nil {
return nil, fmt.Errorf("artifact_externalization_failed: %w", uploadErr)
}
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: policy.ConnectionID, Bucket: policy.Bucket, Key: key, VersionID: meta.VersionID, ETag: meta.ETag, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject}
result[index].Path = "bos://" + policy.Bucket + "/" + key
result[index].SizeBytes = meta.SizeBytes
result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref}
result[index].Data = compactArtifactData(artifact.Data)
}
if run != nil && run.Mode == domain.RunModeDebug {
result = tagDebugArtifacts(run, result)
}
return result, nil
}
func (runtime *Runtime) resolveDirectoryRunInputs(ctx context.Context, run *domain.RunRecord) error {
if run == nil {
return nil
}
changed := false
resolvedByInputID := map[string]*domain.ArtifactReference{}
for index := range run.ResolvedInputBindings {
binding := &run.ResolvedInputBindings[index]
if binding.Payload.Mode != domain.ArtifactTransferModeRef || binding.Payload.Ref == nil {
continue
}
if resolved, ok := resolvedByInputID[binding.WorkflowInputID]; ok {
refCopy := *resolved
binding.Payload.Ref = &refCopy
changed = true
continue
}
ref := binding.Payload.Ref
connection, err := runtime.artifactStorageConnection(ctx, ref.ConnectionID, run.OrganizationID)
if err != nil {
return fmt.Errorf("input_resolution_failed: %w", err)
}
publisher := objectsync.BOSPublisher{GlobalEndpoint: runtime.globalBOSEndpoint}
if ref.Kind == domain.ArtifactRefKindObject || strings.TrimSpace(ref.ManifestKey) != "" {
checkKey := ref.Key
if ref.Kind == domain.ArtifactRefKindDirectory {
checkKey = ref.ManifestKey
}
meta, headErr := publisher.Head(ctx, *connection, ref.Bucket, checkKey)
if headErr != nil || !meta.Exists {
if headErr == nil {
headErr = fmt.Errorf("object does not exist")
}
return fmt.Errorf("input_resolution_failed: %w", headErr)
}
refCopy := *ref
resolvedByInputID[binding.WorkflowInputID] = &refCopy
continue
}
client, err := bos.NewClient(connection.AccessKeyID, connection.SecretAccessKey, connection.Endpoint)
if err != nil {
return fmt.Errorf("input_resolution_failed: %w", err)
}
requestPrefix := strings.Trim(strings.TrimSpace(ref.Prefix), "/") + "/"
entries := make([]directoryManifestEntry, 0)
marker := ""
for {
page, listErr := client.ListObjectsWithContext(ctx, ref.Bucket, &bosapi.ListObjectsArgs{Prefix: requestPrefix, Marker: marker, MaxKeys: 1000})
if listErr != nil {
return fmt.Errorf("input_resolution_failed: %w", listErr)
}
if page == nil {
break
}
for _, object := range page.Contents {
key := strings.TrimSpace(object.Key)
if key == "" || strings.Contains(key, "/_droboticflow/manifests/") {
continue
}
entries = append(entries, directoryManifestEntry{Path: strings.TrimPrefix(key, requestPrefix), Key: key, SizeBytes: int64(object.Size)})
}
if !page.IsTruncated || strings.TrimSpace(page.NextMarker) == "" || page.NextMarker == marker {
break
}
marker = page.NextMarker
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path })
var totalBytes int64
for _, entry := range entries {
totalBytes += entry.SizeBytes
}
manifest := directoryManifest{SchemaVersion: 1, Kind: "directory", Source: map[string]string{"bucket": ref.Bucket, "prefix": ref.Prefix}, FileCount: len(entries), TotalBytes: totalBytes, Entries: entries}
payload, _ := json.Marshal(manifest)
manifestKey := strings.Trim(requestPrefix, "/") + "/_droboticflow/manifests/" + artifactSlug(run.ID) + ".json"
meta, putErr := publisher.PutInternal(ctx, *connection, ref.Bucket, manifestKey, bytes.NewReader(payload), int64(len(payload)), "application/json", nil)
if putErr != nil {
return fmt.Errorf("input_resolution_failed: %w", putErr)
}
ref.ManifestKey = manifestKey
ref.SizeBytes = totalBytes
ref.ETag = meta.ETag
ref.VersionID = meta.VersionID
binding.Payload.Ref = ref
refCopy := *ref
resolvedByInputID[binding.WorkflowInputID] = &refCopy
if input, ok := run.Inputs[binding.WorkflowInputID]; ok && input.Ref != nil {
input.Ref = ref
run.Inputs[binding.WorkflowInputID] = input
}
changed = true
}
if changed {
run.UpdatedAt = nowISO()
return runtime.store.UpsertRun(ctx, *run)
}
return nil
}