forked from longtao.wu/droboticflow
564 lines
21 KiB
Go
564 lines
21 KiB
Go
package worker
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/objectsync"
|
|
|
|
"github.com/baidubce/bce-sdk-go/services/bos"
|
|
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
|
|
)
|
|
|
|
const defaultArtifactPrefixTemplate = "artifacts/{organizationId}/{projectId}/{workflowId}/{runId}/{nodeId}"
|
|
|
|
type directoryManifestEntry struct {
|
|
Path string `json:"path"`
|
|
Key string `json:"key"`
|
|
SizeBytes int64 `json:"sizeBytes"`
|
|
SHA256 string `json:"sha256,omitempty"`
|
|
}
|
|
|
|
type directoryManifest struct {
|
|
SchemaVersion int `json:"schemaVersion"`
|
|
Kind string `json:"kind"`
|
|
Source map[string]string `json:"source,omitempty"`
|
|
FileCount int `json:"fileCount"`
|
|
TotalBytes int64 `json:"totalBytes"`
|
|
Entries []directoryManifestEntry `json:"entries"`
|
|
}
|
|
|
|
func compactArtifactData(data map[string]any) map[string]any {
|
|
if len(data) == 0 {
|
|
return nil
|
|
}
|
|
compact := map[string]any{}
|
|
// WiLoR reports its count inside result. Keep a scalar projection when
|
|
// removing the full frame stream so downstream task summaries remain exact.
|
|
if data["schemaVersion"] == "droboticflow.wilor.hand_pose_result/v1" {
|
|
if result, ok := data["result"].(map[string]any); ok && result["frameCount"] != nil {
|
|
compact["frameCount"] = result["frameCount"]
|
|
} else if frames, ok := data["frames"].([]any); ok {
|
|
compact["frameCount"] = len(frames)
|
|
}
|
|
}
|
|
for key, value := range data {
|
|
if key == "outcome" || key == "deliverySelection" || key == "acceptedMaterializations" {
|
|
compact[key] = value
|
|
continue
|
|
}
|
|
// These fields are required by downstream port-aware adapters (notably
|
|
// virtual Qwen clips and human-release packaging) but do not themselves
|
|
// contain a storage locator. Response sanitisation still removes any
|
|
// nested source/ref/path values before they reach the browser.
|
|
switch key {
|
|
case "preview", "reportSummary", "dataRef", "count", "schemaVersion", "datasetType", "kind", "clipId", "clipName", "sourceIsFullVideo", "startSecond", "endSecond", "sourceDurationSeconds", "behavior", "objectChange", "rawPTS", "annotation", "collectionId", "cameraSide", "role", "transformChain", "privacyTransform", "rectification":
|
|
compact[key] = value
|
|
continue
|
|
}
|
|
if strings.HasSuffix(key, "Count") {
|
|
compact[key] = value
|
|
}
|
|
}
|
|
if len(compact) == 0 {
|
|
return nil
|
|
}
|
|
return compact
|
|
}
|
|
|
|
// structuredJSONRequiresReference keeps the transport contract semantic rather
|
|
// than size-based. A small manifest carrying a path, BOS reference, dataset or
|
|
// collection is still external data and must never become a payload.inline.
|
|
func structuredJSONRequiresReference(artifact domain.ExecutionArtifact, value any) bool {
|
|
name := strings.ToLower(strings.TrimSpace(artifact.Name))
|
|
if strings.Contains(name, "manifest") || strings.Contains(name, "dataset") {
|
|
return true
|
|
}
|
|
if data := artifact.Data; len(data) > 0 {
|
|
if _, ok := data["dataRef"]; ok {
|
|
return true
|
|
}
|
|
if structuredValueHasExternalReference(data) {
|
|
return true
|
|
}
|
|
}
|
|
return structuredValueHasExternalReference(value)
|
|
}
|
|
|
|
func structuredValueHasExternalReference(value any) bool {
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
for key, child := range typed {
|
|
normalized := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(key), "_", ""), "-", ""))
|
|
switch normalized {
|
|
case "dataref", "storagepath", "previewurl", "manifestkey", "connectionid", "bucket", "prefix", "relativepath", "clippath", "filepath", "payloadpath", "partitiondir", "ref":
|
|
return true
|
|
case "entries", "collections", "datasets", "datarefs", "directory", "files":
|
|
return true
|
|
}
|
|
if structuredValueHasExternalReference(child) {
|
|
return true
|
|
}
|
|
}
|
|
case []any:
|
|
for _, child := range typed {
|
|
if structuredValueHasExternalReference(child) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (runtime *Runtime) outputArtifactPublisher() objectsync.Publisher {
|
|
if runtime.artifactPublisher != nil {
|
|
return runtime.artifactPublisher
|
|
}
|
|
return objectsync.BOSPublisher{GlobalEndpoint: runtime.globalBOSEndpoint}
|
|
}
|
|
|
|
func normalizeArtifactKey(value string) string {
|
|
parts := strings.FieldsFunc(strings.TrimSpace(value), func(r rune) bool {
|
|
return r == '\\' || r == '/'
|
|
})
|
|
cleaned := make([]string, 0, len(parts))
|
|
for _, part := range parts {
|
|
part = artifactSlug(part)
|
|
if part != "" && part != "." && part != ".." {
|
|
cleaned = append(cleaned, part)
|
|
}
|
|
}
|
|
return strings.Join(cleaned, "/")
|
|
}
|
|
|
|
func artifactSlug(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return "artifact"
|
|
}
|
|
var builder strings.Builder
|
|
for _, r := range value {
|
|
switch {
|
|
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '-':
|
|
builder.WriteRune(r)
|
|
default:
|
|
builder.WriteRune('-')
|
|
}
|
|
}
|
|
return strings.Trim(builder.String(), "-")
|
|
}
|
|
|
|
func artifactPrefix(policy domain.ArtifactStoragePolicy, run *domain.RunRecord, nodeID string) string {
|
|
template := strings.Trim(strings.TrimSpace(policy.PrefixTemplate), "/")
|
|
if template == "" {
|
|
template = defaultArtifactPrefixTemplate
|
|
}
|
|
replacements := map[string]string{
|
|
"{organizationId}": run.OrganizationID,
|
|
"{projectId}": run.ProjectID,
|
|
"{workflowId}": run.WorkflowID,
|
|
"{runId}": run.ID,
|
|
"{nodeId}": nodeID,
|
|
}
|
|
for token, value := range replacements {
|
|
template = strings.ReplaceAll(template, token, artifactSlug(value))
|
|
}
|
|
return normalizeArtifactKey(template)
|
|
}
|
|
|
|
func (runtime *Runtime) artifactStoragePolicy(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.ArtifactStoragePolicy, error) {
|
|
if workflow != nil && workflow.ArtifactStorage != nil && strings.TrimSpace(workflow.ArtifactStorage.ConnectionID) != "" {
|
|
copy := *workflow.ArtifactStorage
|
|
return ©, nil
|
|
}
|
|
organizationID := ""
|
|
if workflow != nil {
|
|
organizationID = workflow.OrganizationID
|
|
}
|
|
settings, err := runtime.store.GetArtifactStorageSettings(ctx, organizationID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if strings.TrimSpace(settings.ConnectionID) == "" || strings.TrimSpace(settings.Bucket) == "" {
|
|
return nil, nil
|
|
}
|
|
return &domain.ArtifactStoragePolicy{
|
|
ConnectionID: settings.ConnectionID,
|
|
Bucket: settings.Bucket,
|
|
PrefixTemplate: settings.PrefixTemplate,
|
|
}, nil
|
|
}
|
|
|
|
func (runtime *Runtime) artifactStorageConnection(ctx context.Context, connectionID string, organizationID string) (*domain.StorageConnectionRecord, error) {
|
|
connection, err := runtime.store.GetStorageConnection(ctx, strings.TrimSpace(connectionID))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if connection == nil {
|
|
return nil, fmt.Errorf("artifact storage connection %s not found", connectionID)
|
|
}
|
|
if strings.TrimSpace(connection.OrganizationID) != "" && strings.TrimSpace(organizationID) != "" && connection.OrganizationID != organizationID {
|
|
return nil, fmt.Errorf("artifact storage connection belongs to another organization")
|
|
}
|
|
copy := *connection
|
|
if strings.TrimSpace(copy.Endpoint) == "" {
|
|
copy.Endpoint = runtime.globalBOSEndpoint
|
|
}
|
|
return ©, nil
|
|
}
|
|
|
|
func fileSHA256(path string) (string, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file.Close()
|
|
hash := sha256.New()
|
|
if _, err := io.Copy(hash, file); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
|
}
|
|
|
|
func putArtifactFile(ctx context.Context, publisher objectsync.Publisher, connection domain.StorageConnectionRecord, bucket, key, path string) (objectsync.ObjectMetadata, string, error) {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return objectsync.ObjectMetadata{}, "", err
|
|
}
|
|
sha, err := fileSHA256(path)
|
|
if err != nil {
|
|
return objectsync.ObjectMetadata{}, "", err
|
|
}
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return objectsync.ObjectMetadata{}, "", err
|
|
}
|
|
defer file.Close()
|
|
contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(path)))
|
|
if contentType == "" {
|
|
contentType = "application/octet-stream"
|
|
}
|
|
meta, err := publisher.PutInternal(ctx, connection, bucket, key, file, info.Size(), contentType, map[string]string{"sha256": sha})
|
|
return meta, sha, err
|
|
}
|
|
|
|
func (runtime *Runtime) externalizeDirectory(ctx context.Context, publisher objectsync.Publisher, connection domain.StorageConnectionRecord, policy domain.ArtifactStoragePolicy, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, map[string]domain.ArtifactReference, error) {
|
|
basePrefix := artifactPrefix(policy, run, nodeID) + "/" + artifactSlug(artifact.ID)
|
|
entries := make([]directoryManifestEntry, 0)
|
|
sharedRefs := map[string]domain.ArtifactReference{}
|
|
var totalBytes int64
|
|
err := filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if entry.IsDir() {
|
|
return nil
|
|
}
|
|
relative, err := filepath.Rel(artifact.Path, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
key := basePrefix + "/files/" + normalizeArtifactKey(filepath.ToSlash(relative))
|
|
meta, sha, err := putArtifactFile(ctx, publisher, connection, policy.Bucket, key, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entries = append(entries, directoryManifestEntry{Path: filepath.ToSlash(relative), Key: key, SizeBytes: meta.SizeBytes, SHA256: sha})
|
|
sharedRefs[filepath.Clean(path)] = domain.ArtifactReference{
|
|
Provider: connection.Provider, ConnectionID: policy.ConnectionID, Bucket: policy.Bucket, Key: key,
|
|
VersionID: meta.VersionID, ETag: meta.ETag, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject,
|
|
}
|
|
totalBytes += meta.SizeBytes
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return artifact, nil, err
|
|
}
|
|
sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path })
|
|
manifest := directoryManifest{SchemaVersion: 1, Kind: "directory", FileCount: len(entries), TotalBytes: totalBytes, Entries: entries}
|
|
payload, err := json.Marshal(manifest)
|
|
if err != nil {
|
|
return artifact, nil, err
|
|
}
|
|
manifestKey := basePrefix + "/manifest.json"
|
|
meta, err := publisher.PutInternal(ctx, connection, policy.Bucket, manifestKey, bytes.NewReader(payload), int64(len(payload)), "application/json", nil)
|
|
if err != nil {
|
|
return artifact, nil, err
|
|
}
|
|
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: policy.ConnectionID, Bucket: policy.Bucket, Prefix: basePrefix + "/files", ManifestKey: manifestKey, VersionID: meta.VersionID, ETag: meta.ETag, SizeBytes: totalBytes, Kind: domain.ArtifactRefKindDirectory}
|
|
artifact.Path = "bos://" + policy.Bucket + "/" + ref.Prefix
|
|
artifact.SizeBytes = totalBytes
|
|
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref}
|
|
artifact.Data = compactArtifactData(artifact.Data)
|
|
return artifact, sharedRefs, nil
|
|
}
|
|
|
|
func (runtime *Runtime) normalizeOutputArtifacts(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord, nodeID string, artifacts []domain.ExecutionArtifact) ([]domain.ExecutionArtifact, error) {
|
|
policy, err := runtime.artifactStoragePolicy(ctx, workflow)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var connection *domain.StorageConnectionRecord
|
|
var publisher objectsync.Publisher
|
|
if policy != nil {
|
|
connection, err = runtime.artifactStorageConnection(ctx, policy.ConnectionID, run.OrganizationID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
publisher = runtime.outputArtifactPublisher()
|
|
}
|
|
result := make([]domain.ExecutionArtifact, len(artifacts))
|
|
// A package directory and its visible MP4/JSON members are often returned
|
|
// together. Externalise the directory once, then bind member artifacts to
|
|
// the exact already-uploaded object refs instead of copying every byte a
|
|
// second time.
|
|
sharedDirectoryFileRefs := map[string]domain.ArtifactReference{}
|
|
order := make([]int, 0, len(artifacts))
|
|
for index, artifact := range artifacts {
|
|
if domain.IsInternalDeliveryAudit(artifact) {
|
|
order = append(order, index)
|
|
}
|
|
}
|
|
for index, artifact := range artifacts {
|
|
if !domain.IsInternalDeliveryAudit(artifact) {
|
|
order = append(order, index)
|
|
}
|
|
}
|
|
for _, index := range order {
|
|
artifact := artifacts[index]
|
|
result[index] = artifact
|
|
if artifact.Payload != nil && artifact.Payload.Mode == domain.ArtifactTransferModeRef {
|
|
if err := domain.ValidateArtifactPayload(*artifact.Payload, artifact.Type); err != nil {
|
|
return nil, fmt.Errorf("artifact_ref_invalid: %w", err)
|
|
}
|
|
ref := artifact.Payload.Ref
|
|
if ref.Kind == domain.ArtifactRefKindDirectory && strings.TrimSpace(ref.ManifestKey) == "" {
|
|
return nil, fmt.Errorf("manifest_incomplete: directory artifact ref requires manifestKey")
|
|
}
|
|
refConnection, err := runtime.artifactStorageConnection(ctx, ref.ConnectionID, run.OrganizationID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("artifact_ref_invalid: %w", err)
|
|
}
|
|
checkKey := ref.Key
|
|
if ref.Kind == domain.ArtifactRefKindDirectory {
|
|
checkKey = ref.ManifestKey
|
|
}
|
|
meta, err := runtime.outputArtifactPublisher().Head(ctx, *refConnection, ref.Bucket, checkKey)
|
|
if err != nil || !meta.Exists {
|
|
if err == nil {
|
|
err = fmt.Errorf("object does not exist")
|
|
}
|
|
return nil, fmt.Errorf("artifact_ref_invalid: %w", err)
|
|
}
|
|
continue
|
|
}
|
|
info, statErr := os.Stat(artifact.Path)
|
|
// A part-index delivery is one file, not a tree. It runs before the
|
|
// generic inline/upload paths so the frozen capture destination and the
|
|
// shared conflict rules always win over the run's artifact policy.
|
|
if statErr == nil && info.Mode().IsRegular() && artifact.Name == partIndexFileName {
|
|
if _, e := partIndexPayloadPeek(artifact.Path); e == nil {
|
|
delivered, e := runtime.publishCaptureDeliveryFile(ctx, run, nodeID, artifact)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
result[index] = delivered
|
|
continue
|
|
}
|
|
}
|
|
if statErr == nil && info.Mode().IsRegular() && artifact.Type == domain.PortTypeJSON && info.Size() <= domain.MaxInlinePayloadBytes {
|
|
payloadBytes, readErr := os.ReadFile(artifact.Path)
|
|
if readErr != nil {
|
|
return nil, readErr
|
|
}
|
|
var inline any
|
|
if json.Unmarshal(payloadBytes, &inline) == nil && !structuredJSONRequiresReference(artifact, inline) {
|
|
result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: inline}
|
|
continue
|
|
}
|
|
}
|
|
if statErr != nil && strings.HasPrefix(strings.ToLower(strings.TrimSpace(artifact.Path)), "bos://") {
|
|
location := strings.TrimPrefix(strings.TrimSpace(artifact.Path), "bos://")
|
|
parts := strings.SplitN(location, "/", 2)
|
|
if len(parts) == 2 && strings.TrimSpace(parts[0]) != "" && strings.TrimSpace(parts[1]) != "" {
|
|
connectionID := ""
|
|
if workflow != nil {
|
|
if node := domain.GetNodeByID(workflow.Nodes, nodeID); node != nil {
|
|
connectionID = strings.TrimSpace(node.Data.ConnectionID)
|
|
}
|
|
}
|
|
if connectionID == "" && policy != nil {
|
|
connectionID = policy.ConnectionID
|
|
}
|
|
if connectionID != "" {
|
|
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: connectionID, Bucket: parts[0], Key: parts[1], SizeBytes: artifact.SizeBytes, Kind: domain.ArtifactRefKindObject}
|
|
result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref}
|
|
result[index].Data = compactArtifactData(artifact.Data)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
if statErr == nil && info.IsDir() && artifact.Name == "delivery.directory.index" {
|
|
delivered, e := runtime.publishCaptureDelivery(ctx, run, nodeID, artifact)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
result[index] = delivered
|
|
continue
|
|
}
|
|
if policy == nil {
|
|
continue // Draft/legacy workflows retain workspace-path compatibility.
|
|
}
|
|
if statErr != nil {
|
|
if workflow != nil && workflow.DefinitionState == domain.WorkflowDefinitionStateReleased {
|
|
return nil, fmt.Errorf("artifact_externalization_failed: artifact path %s is not accessible to the worker and no durable ref was provided", artifact.Path)
|
|
}
|
|
continue
|
|
}
|
|
if info.Mode().IsRegular() {
|
|
if shared, exists := sharedDirectoryFileRefs[filepath.Clean(artifact.Path)]; exists {
|
|
ref := shared
|
|
result[index].Path = "bos://" + ref.Bucket + "/" + ref.Key
|
|
result[index].SizeBytes = ref.SizeBytes
|
|
result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &ref}
|
|
result[index].Data = compactArtifactData(artifact.Data)
|
|
continue
|
|
}
|
|
}
|
|
if info.IsDir() {
|
|
externalized, refs, externalizeErr := runtime.externalizeDirectory(ctx, publisher, *connection, *policy, run, nodeID, artifact)
|
|
if externalizeErr != nil {
|
|
return nil, fmt.Errorf("artifact_externalization_failed: %w", externalizeErr)
|
|
}
|
|
result[index] = externalized
|
|
for path, ref := range refs {
|
|
sharedDirectoryFileRefs[path] = ref
|
|
}
|
|
continue
|
|
}
|
|
key := artifactPrefix(*policy, run, nodeID) + "/" + artifactSlug(artifact.ID) + "/" + artifactSlug(filepath.Base(artifact.Path))
|
|
meta, sha, uploadErr := putArtifactFile(ctx, publisher, *connection, policy.Bucket, key, artifact.Path)
|
|
if uploadErr != nil {
|
|
return nil, fmt.Errorf("artifact_externalization_failed: %w", uploadErr)
|
|
}
|
|
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: policy.ConnectionID, Bucket: policy.Bucket, Key: key, VersionID: meta.VersionID, ETag: meta.ETag, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject}
|
|
result[index].Path = "bos://" + policy.Bucket + "/" + key
|
|
result[index].SizeBytes = meta.SizeBytes
|
|
result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref}
|
|
result[index].Data = compactArtifactData(artifact.Data)
|
|
}
|
|
if run != nil && run.Mode == domain.RunModeDebug {
|
|
result = tagDebugArtifacts(run, result)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveDirectoryRunInputs(ctx context.Context, run *domain.RunRecord) error {
|
|
if run == nil {
|
|
return nil
|
|
}
|
|
changed := false
|
|
resolvedByInputID := map[string]*domain.ArtifactReference{}
|
|
for index := range run.ResolvedInputBindings {
|
|
binding := &run.ResolvedInputBindings[index]
|
|
if binding.Payload.Mode != domain.ArtifactTransferModeRef || binding.Payload.Ref == nil {
|
|
continue
|
|
}
|
|
if resolved, ok := resolvedByInputID[binding.WorkflowInputID]; ok {
|
|
refCopy := *resolved
|
|
binding.Payload.Ref = &refCopy
|
|
changed = true
|
|
continue
|
|
}
|
|
ref := binding.Payload.Ref
|
|
connection, err := runtime.artifactStorageConnection(ctx, ref.ConnectionID, run.OrganizationID)
|
|
if err != nil {
|
|
return fmt.Errorf("input_resolution_failed: %w", err)
|
|
}
|
|
publisher := objectsync.BOSPublisher{GlobalEndpoint: runtime.globalBOSEndpoint}
|
|
if ref.Kind == domain.ArtifactRefKindObject || strings.TrimSpace(ref.ManifestKey) != "" {
|
|
checkKey := ref.Key
|
|
if ref.Kind == domain.ArtifactRefKindDirectory {
|
|
checkKey = ref.ManifestKey
|
|
}
|
|
meta, headErr := publisher.Head(ctx, *connection, ref.Bucket, checkKey)
|
|
if headErr != nil || !meta.Exists {
|
|
if headErr == nil {
|
|
headErr = fmt.Errorf("object does not exist")
|
|
}
|
|
return fmt.Errorf("input_resolution_failed: %w", headErr)
|
|
}
|
|
refCopy := *ref
|
|
resolvedByInputID[binding.WorkflowInputID] = &refCopy
|
|
continue
|
|
}
|
|
client, err := bos.NewClient(connection.AccessKeyID, connection.SecretAccessKey, connection.Endpoint)
|
|
if err != nil {
|
|
return fmt.Errorf("input_resolution_failed: %w", err)
|
|
}
|
|
requestPrefix := strings.Trim(strings.TrimSpace(ref.Prefix), "/") + "/"
|
|
entries := make([]directoryManifestEntry, 0)
|
|
marker := ""
|
|
for {
|
|
page, listErr := client.ListObjectsWithContext(ctx, ref.Bucket, &bosapi.ListObjectsArgs{Prefix: requestPrefix, Marker: marker, MaxKeys: 1000})
|
|
if listErr != nil {
|
|
return fmt.Errorf("input_resolution_failed: %w", listErr)
|
|
}
|
|
if page == nil {
|
|
break
|
|
}
|
|
for _, object := range page.Contents {
|
|
key := strings.TrimSpace(object.Key)
|
|
if key == "" || strings.Contains(key, "/_droboticflow/manifests/") {
|
|
continue
|
|
}
|
|
entries = append(entries, directoryManifestEntry{Path: strings.TrimPrefix(key, requestPrefix), Key: key, SizeBytes: int64(object.Size)})
|
|
}
|
|
if !page.IsTruncated || strings.TrimSpace(page.NextMarker) == "" || page.NextMarker == marker {
|
|
break
|
|
}
|
|
marker = page.NextMarker
|
|
}
|
|
sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path })
|
|
var totalBytes int64
|
|
for _, entry := range entries {
|
|
totalBytes += entry.SizeBytes
|
|
}
|
|
manifest := directoryManifest{SchemaVersion: 1, Kind: "directory", Source: map[string]string{"bucket": ref.Bucket, "prefix": ref.Prefix}, FileCount: len(entries), TotalBytes: totalBytes, Entries: entries}
|
|
payload, _ := json.Marshal(manifest)
|
|
manifestKey := strings.Trim(requestPrefix, "/") + "/_droboticflow/manifests/" + artifactSlug(run.ID) + ".json"
|
|
meta, putErr := publisher.PutInternal(ctx, *connection, ref.Bucket, manifestKey, bytes.NewReader(payload), int64(len(payload)), "application/json", nil)
|
|
if putErr != nil {
|
|
return fmt.Errorf("input_resolution_failed: %w", putErr)
|
|
}
|
|
ref.ManifestKey = manifestKey
|
|
ref.SizeBytes = totalBytes
|
|
ref.ETag = meta.ETag
|
|
ref.VersionID = meta.VersionID
|
|
binding.Payload.Ref = ref
|
|
refCopy := *ref
|
|
resolvedByInputID[binding.WorkflowInputID] = &refCopy
|
|
if input, ok := run.Inputs[binding.WorkflowInputID]; ok && input.Ref != nil {
|
|
input.Ref = ref
|
|
run.Inputs[binding.WorkflowInputID] = input
|
|
}
|
|
changed = true
|
|
}
|
|
if changed {
|
|
run.UpdatedAt = nowISO()
|
|
return runtime.store.UpsertRun(ctx, *run)
|
|
}
|
|
return nil
|
|
}
|