droboticflow/internal/api/annotation_storage.go
2026-09-27 05:32:22 -07:00

207 lines
7.0 KiB
Go

package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"fmt"
"golang.org/x/sync/errgroup"
"os"
"path/filepath"
"strings"
)
func annotationObjectSHA(meta objectsync.ObjectMetadata) string {
for k, v := range meta.UserMeta {
if strings.EqualFold(k, "sha256") {
return v
}
}
return ""
}
// Caller has already fenced the current generation and validated this exact
// older revision path. Virtual BOS directories must not use rmdir/unlinkat.
func (s *server) removeAnnotationGeneration(ctx context.Context, org, path string) error {
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, path)
if e != nil {
return e
}
if c == nil {
return os.RemoveAll(path)
}
if !strings.HasPrefix(key, "system/runs/") || !strings.Contains(key, "/human-materializations/") || !strings.HasPrefix(filepath.Base(key), "revision-") {
return fmt.Errorf("unsafe annotation cleanup prefix")
}
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
prefix := strings.TrimSuffix(key, "/") + "/"
keys, e := p.ListPrefix(ctx, *c, bucket, prefix)
if e != nil {
return e
}
group, gctx := errgroup.WithContext(ctx)
group.SetLimit(4)
for _, objectKey := range keys {
if !strings.HasPrefix(objectKey, prefix) {
return fmt.Errorf("foreign object in annotation cleanup")
}
group.Go(func() error { return p.Delete(gctx, *c, bucket, objectKey) })
}
return group.Wait()
}
func (s *server) annotationMediaCopyReady(ctx context.Context, org, path string, file map[string]any) error {
c, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, path, false)
if e != nil {
return e
}
if c == nil {
return nil
}
if stringValue(file["etag"]) == "" {
return fmt.Errorf("legacy media needs initial verification")
}
meta, e := (objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}).Head(ctx, *c, bucket, key)
if e != nil {
return e
}
if !meta.Exists || meta.ETag != stringValue(file["etag"]) || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != stringValue(file["sha256"]) {
return fmt.Errorf("cached media version changed")
}
return nil
}
func (s *server) annotationWorkspaceObject(ctx context.Context, org, path string) (*domain.StorageConnectionRecord, string, string, error) {
return s.annotationWorkspaceObjectAccess(ctx, org, path, true)
}
func (s *server) publishAnnotationFile(ctx context.Context, org, source, target, sha string) (objectsync.ObjectMetadata, error) {
if e := s.assertAnnotationLease(ctx); e != nil {
return objectsync.ObjectMetadata{}, e
}
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, target)
if e != nil {
return objectsync.ObjectMetadata{}, e
}
if c == nil {
e := copyAnnotationFile(ctx, source, target, sha)
return objectsync.ObjectMetadata{}, e
}
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
info, e := os.Stat(source)
if e != nil {
return objectsync.ObjectMetadata{}, e
}
meta, e := p.Head(ctx, *c, bucket, key)
if e != nil {
return meta, e
}
if meta.Exists {
if meta.SizeBytes == info.Size() && annotationObjectSHA(meta) == sha {
return meta, nil
}
return meta, fmt.Errorf("annotation output conflict: %s", filepath.Base(target))
}
f, e := os.Open(source)
if e != nil {
return meta, e
}
defer f.Close()
meta, e = p.PutInternal(ctx, *c, bucket, key, contextFileReader{ctx, f}, info.Size(), "application/octet-stream", map[string]string{"sha256": sha})
if e != nil {
actual, he := p.Head(ctx, *c, bucket, key)
if he == nil && actual.Exists && actual.SizeBytes == info.Size() && annotationObjectSHA(actual) == sha {
return actual, nil
}
}
return meta, e
}
// The bytes stay in BOS. Source ETag and digest metadata fence the copy, while
// the fresh generation keeps independent objects for existing retention rules.
func (s *server) copyAnnotationObject(ctx context.Context, org, source, target string, file map[string]any) (objectsync.ObjectMetadata, error) {
sha := stringValue(file["sha256"])
if sha == "" {
return objectsync.ObjectMetadata{}, fmt.Errorf("missing verified media digest")
}
c, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, source, false)
if e != nil {
return objectsync.ObjectMetadata{}, e
}
if c == nil {
e := copyAnnotationFile(ctx, source, target, sha)
return objectsync.ObjectMetadata{}, e
}
toConnection, toBucket, toKey, e := s.annotationWorkspaceObject(ctx, org, target)
if e != nil {
return objectsync.ObjectMetadata{}, e
}
if toConnection == nil || c.ID != toConnection.ID || bucket != toBucket {
return objectsync.ObjectMetadata{}, fmt.Errorf("cross-workspace media copy")
}
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
meta, e := p.Head(ctx, *c, bucket, key)
if e != nil {
return meta, e
}
// Old generations without authoritative object metadata take the verified
// normal materialization path once, rather than trusting an unpinned copy.
if !meta.Exists || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != sha || stringValue(file["etag"]) == "" || meta.ETag != stringValue(file["etag"]) {
return meta, fmt.Errorf("cached media version unavailable")
}
old, e := p.Head(ctx, *c, bucket, toKey)
if e != nil {
return old, e
}
if old.Exists {
if old.SizeBytes == meta.SizeBytes && annotationObjectSHA(old) == sha {
return old, nil
}
return old, fmt.Errorf("annotation media destination conflict")
}
if e = s.assertAnnotationLease(ctx); e != nil {
return old, e
}
return p.PublishConditional(ctx, *c, bucket, key, toKey, meta.SizeBytes, meta.ETag, "", "*", map[string]string{"sha256": sha})
}
func (s *server) publishAnnotationDirectory(ctx context.Context, org, source, target string, manifest map[string]any) (string, int64, error) {
// Publish all media/sidecars before the generation manifest. Limit network
// concurrency independently of encoder concurrency.
group, gctx := errgroup.WithContext(ctx)
group.SetLimit(4)
items := listValue(manifest["items"])
manifest["items"] = items
for i, raw := range items {
item := mapValue(raw)
items[i] = item
dir := stringValue(item["directory"])
files := mapValue(item["files"])
item["files"] = files
for kind, rawFile := range files {
file := mapValue(rawFile)
files[kind] = file
name := stringValue(file["name"])
if name == "" || filepath.Base(name) != name || filepath.IsAbs(dir) || strings.Contains(dir, "..") {
return "", 0, fmt.Errorf("invalid annotation member path")
}
group.Go(func() error {
meta, e := s.publishAnnotationFile(gctx, org, filepath.Join(source, dir, name), filepath.Join(target, dir, name), stringValue(file["sha256"]))
if e == nil && meta.ETag != "" {
file["etag"] = meta.ETag
}
return e
})
}
}
if e := group.Wait(); e != nil {
return "", 0, e
}
path := filepath.Join(source, "revision-manifest.json")
sha, size, e := writeJSONFile(path, manifest)
if e != nil {
return "", 0, e
}
_, e = s.publishAnnotationFile(ctx, org, path, filepath.Join(target, "revision-manifest.json"), sha)
return sha, size, e
}