package api import ( "context" "droboticflow/internal/domain" "droboticflow/internal/objectsync" "fmt" "golang.org/x/sync/errgroup" "os" "path/filepath" "strings" ) func annotationObjectSHA(meta objectsync.ObjectMetadata) string { for k, v := range meta.UserMeta { if strings.EqualFold(k, "sha256") { return v } } return "" } // Caller has already fenced the current generation and validated this exact // older revision path. Virtual BOS directories must not use rmdir/unlinkat. func (s *server) removeAnnotationGeneration(ctx context.Context, org, path string) error { c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, path) if e != nil { return e } if c == nil { return os.RemoveAll(path) } if !strings.HasPrefix(key, "system/runs/") || !strings.Contains(key, "/human-materializations/") || !strings.HasPrefix(filepath.Base(key), "revision-") { return fmt.Errorf("unsafe annotation cleanup prefix") } p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint} prefix := strings.TrimSuffix(key, "/") + "/" keys, e := p.ListPrefix(ctx, *c, bucket, prefix) if e != nil { return e } group, gctx := errgroup.WithContext(ctx) group.SetLimit(4) for _, objectKey := range keys { if !strings.HasPrefix(objectKey, prefix) { return fmt.Errorf("foreign object in annotation cleanup") } group.Go(func() error { return p.Delete(gctx, *c, bucket, objectKey) }) } return group.Wait() } func (s *server) annotationMediaCopyReady(ctx context.Context, org, path string, file map[string]any) error { c, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, path, false) if e != nil { return e } if c == nil { return nil } if stringValue(file["etag"]) == "" { return fmt.Errorf("legacy media needs initial verification") } meta, e := (objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}).Head(ctx, *c, bucket, key) if e != nil { return e } if !meta.Exists || meta.ETag != stringValue(file["etag"]) || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != stringValue(file["sha256"]) { return fmt.Errorf("cached media version changed") } return nil } func (s *server) annotationWorkspaceObject(ctx context.Context, org, path string) (*domain.StorageConnectionRecord, string, string, error) { return s.annotationWorkspaceObjectAccess(ctx, org, path, true) } func (s *server) publishAnnotationFile(ctx context.Context, org, source, target, sha string) (objectsync.ObjectMetadata, error) { if e := s.assertAnnotationLease(ctx); e != nil { return objectsync.ObjectMetadata{}, e } c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, target) if e != nil { return objectsync.ObjectMetadata{}, e } if c == nil { e := copyAnnotationFile(ctx, source, target, sha) return objectsync.ObjectMetadata{}, e } p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint} info, e := os.Stat(source) if e != nil { return objectsync.ObjectMetadata{}, e } meta, e := p.Head(ctx, *c, bucket, key) if e != nil { return meta, e } if meta.Exists { if meta.SizeBytes == info.Size() && annotationObjectSHA(meta) == sha { return meta, nil } return meta, fmt.Errorf("annotation output conflict: %s", filepath.Base(target)) } f, e := os.Open(source) if e != nil { return meta, e } defer f.Close() meta, e = p.PutInternal(ctx, *c, bucket, key, contextFileReader{ctx, f}, info.Size(), "application/octet-stream", map[string]string{"sha256": sha}) if e != nil { actual, he := p.Head(ctx, *c, bucket, key) if he == nil && actual.Exists && actual.SizeBytes == info.Size() && annotationObjectSHA(actual) == sha { return actual, nil } } return meta, e } // The bytes stay in BOS. Source ETag and digest metadata fence the copy, while // the fresh generation keeps independent objects for existing retention rules. func (s *server) copyAnnotationObject(ctx context.Context, org, source, target string, file map[string]any) (objectsync.ObjectMetadata, error) { sha := stringValue(file["sha256"]) if sha == "" { return objectsync.ObjectMetadata{}, fmt.Errorf("missing verified media digest") } c, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, source, false) if e != nil { return objectsync.ObjectMetadata{}, e } if c == nil { e := copyAnnotationFile(ctx, source, target, sha) return objectsync.ObjectMetadata{}, e } toConnection, toBucket, toKey, e := s.annotationWorkspaceObject(ctx, org, target) if e != nil { return objectsync.ObjectMetadata{}, e } if toConnection == nil || c.ID != toConnection.ID || bucket != toBucket { return objectsync.ObjectMetadata{}, fmt.Errorf("cross-workspace media copy") } p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint} meta, e := p.Head(ctx, *c, bucket, key) if e != nil { return meta, e } // Old generations without authoritative object metadata take the verified // normal materialization path once, rather than trusting an unpinned copy. if !meta.Exists || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != sha || stringValue(file["etag"]) == "" || meta.ETag != stringValue(file["etag"]) { return meta, fmt.Errorf("cached media version unavailable") } old, e := p.Head(ctx, *c, bucket, toKey) if e != nil { return old, e } if old.Exists { if old.SizeBytes == meta.SizeBytes && annotationObjectSHA(old) == sha { return old, nil } return old, fmt.Errorf("annotation media destination conflict") } if e = s.assertAnnotationLease(ctx); e != nil { return old, e } return p.PublishConditional(ctx, *c, bucket, key, toKey, meta.SizeBytes, meta.ETag, "", "*", map[string]string{"sha256": sha}) } func (s *server) publishAnnotationDirectory(ctx context.Context, org, source, target string, manifest map[string]any) (string, int64, error) { // Publish all media/sidecars before the generation manifest. Limit network // concurrency independently of encoder concurrency. group, gctx := errgroup.WithContext(ctx) group.SetLimit(4) items := listValue(manifest["items"]) manifest["items"] = items for i, raw := range items { item := mapValue(raw) items[i] = item dir := stringValue(item["directory"]) files := mapValue(item["files"]) item["files"] = files for kind, rawFile := range files { file := mapValue(rawFile) files[kind] = file name := stringValue(file["name"]) if name == "" || filepath.Base(name) != name || filepath.IsAbs(dir) || strings.Contains(dir, "..") { return "", 0, fmt.Errorf("invalid annotation member path") } group.Go(func() error { meta, e := s.publishAnnotationFile(gctx, org, filepath.Join(source, dir, name), filepath.Join(target, dir, name), stringValue(file["sha256"])) if e == nil && meta.ETag != "" { file["etag"] = meta.ETag } return e }) } } if e := group.Wait(); e != nil { return "", 0, e } path := filepath.Join(source, "revision-manifest.json") sha, size, e := writeJSONFile(path, manifest) if e != nil { return "", 0, e } _, e = s.publishAnnotationFile(ctx, org, path, filepath.Join(target, "revision-manifest.json"), sha) return sha, size, e }