forked from longtao.wu/droboticflow
207 lines
7.0 KiB
Go
207 lines
7.0 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/objectsync"
|
|
"fmt"
|
|
"golang.org/x/sync/errgroup"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
)
|
|
|
|
func annotationObjectSHA(meta objectsync.ObjectMetadata) string {
|
|
for k, v := range meta.UserMeta {
|
|
if strings.EqualFold(k, "sha256") {
|
|
return v
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Caller has already fenced the current generation and validated this exact
|
|
// older revision path. Virtual BOS directories must not use rmdir/unlinkat.
|
|
func (s *server) removeAnnotationGeneration(ctx context.Context, org, path string) error {
|
|
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, path)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if c == nil {
|
|
return os.RemoveAll(path)
|
|
}
|
|
if !strings.HasPrefix(key, "system/runs/") || !strings.Contains(key, "/human-materializations/") || !strings.HasPrefix(filepath.Base(key), "revision-") {
|
|
return fmt.Errorf("unsafe annotation cleanup prefix")
|
|
}
|
|
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
|
|
prefix := strings.TrimSuffix(key, "/") + "/"
|
|
keys, e := p.ListPrefix(ctx, *c, bucket, prefix)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
group, gctx := errgroup.WithContext(ctx)
|
|
group.SetLimit(4)
|
|
for _, objectKey := range keys {
|
|
if !strings.HasPrefix(objectKey, prefix) {
|
|
return fmt.Errorf("foreign object in annotation cleanup")
|
|
}
|
|
group.Go(func() error { return p.Delete(gctx, *c, bucket, objectKey) })
|
|
}
|
|
return group.Wait()
|
|
}
|
|
|
|
func (s *server) annotationMediaCopyReady(ctx context.Context, org, path string, file map[string]any) error {
|
|
c, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, path, false)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if c == nil {
|
|
return nil
|
|
}
|
|
if stringValue(file["etag"]) == "" {
|
|
return fmt.Errorf("legacy media needs initial verification")
|
|
}
|
|
meta, e := (objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}).Head(ctx, *c, bucket, key)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if !meta.Exists || meta.ETag != stringValue(file["etag"]) || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != stringValue(file["sha256"]) {
|
|
return fmt.Errorf("cached media version changed")
|
|
}
|
|
return nil
|
|
}
|
|
func (s *server) annotationWorkspaceObject(ctx context.Context, org, path string) (*domain.StorageConnectionRecord, string, string, error) {
|
|
return s.annotationWorkspaceObjectAccess(ctx, org, path, true)
|
|
}
|
|
|
|
func (s *server) publishAnnotationFile(ctx context.Context, org, source, target, sha string) (objectsync.ObjectMetadata, error) {
|
|
if e := s.assertAnnotationLease(ctx); e != nil {
|
|
return objectsync.ObjectMetadata{}, e
|
|
}
|
|
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, target)
|
|
if e != nil {
|
|
return objectsync.ObjectMetadata{}, e
|
|
}
|
|
if c == nil {
|
|
e := copyAnnotationFile(ctx, source, target, sha)
|
|
return objectsync.ObjectMetadata{}, e
|
|
}
|
|
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
|
|
info, e := os.Stat(source)
|
|
if e != nil {
|
|
return objectsync.ObjectMetadata{}, e
|
|
}
|
|
meta, e := p.Head(ctx, *c, bucket, key)
|
|
if e != nil {
|
|
return meta, e
|
|
}
|
|
if meta.Exists {
|
|
if meta.SizeBytes == info.Size() && annotationObjectSHA(meta) == sha {
|
|
return meta, nil
|
|
}
|
|
return meta, fmt.Errorf("annotation output conflict: %s", filepath.Base(target))
|
|
}
|
|
f, e := os.Open(source)
|
|
if e != nil {
|
|
return meta, e
|
|
}
|
|
defer f.Close()
|
|
meta, e = p.PutInternal(ctx, *c, bucket, key, contextFileReader{ctx, f}, info.Size(), "application/octet-stream", map[string]string{"sha256": sha})
|
|
if e != nil {
|
|
actual, he := p.Head(ctx, *c, bucket, key)
|
|
if he == nil && actual.Exists && actual.SizeBytes == info.Size() && annotationObjectSHA(actual) == sha {
|
|
return actual, nil
|
|
}
|
|
}
|
|
return meta, e
|
|
}
|
|
|
|
// The bytes stay in BOS. Source ETag and digest metadata fence the copy, while
|
|
// the fresh generation keeps independent objects for existing retention rules.
|
|
func (s *server) copyAnnotationObject(ctx context.Context, org, source, target string, file map[string]any) (objectsync.ObjectMetadata, error) {
|
|
sha := stringValue(file["sha256"])
|
|
if sha == "" {
|
|
return objectsync.ObjectMetadata{}, fmt.Errorf("missing verified media digest")
|
|
}
|
|
c, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, source, false)
|
|
if e != nil {
|
|
return objectsync.ObjectMetadata{}, e
|
|
}
|
|
if c == nil {
|
|
e := copyAnnotationFile(ctx, source, target, sha)
|
|
return objectsync.ObjectMetadata{}, e
|
|
}
|
|
toConnection, toBucket, toKey, e := s.annotationWorkspaceObject(ctx, org, target)
|
|
if e != nil {
|
|
return objectsync.ObjectMetadata{}, e
|
|
}
|
|
if toConnection == nil || c.ID != toConnection.ID || bucket != toBucket {
|
|
return objectsync.ObjectMetadata{}, fmt.Errorf("cross-workspace media copy")
|
|
}
|
|
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
|
|
meta, e := p.Head(ctx, *c, bucket, key)
|
|
if e != nil {
|
|
return meta, e
|
|
}
|
|
// Old generations without authoritative object metadata take the verified
|
|
// normal materialization path once, rather than trusting an unpinned copy.
|
|
if !meta.Exists || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != sha || stringValue(file["etag"]) == "" || meta.ETag != stringValue(file["etag"]) {
|
|
return meta, fmt.Errorf("cached media version unavailable")
|
|
}
|
|
old, e := p.Head(ctx, *c, bucket, toKey)
|
|
if e != nil {
|
|
return old, e
|
|
}
|
|
if old.Exists {
|
|
if old.SizeBytes == meta.SizeBytes && annotationObjectSHA(old) == sha {
|
|
return old, nil
|
|
}
|
|
return old, fmt.Errorf("annotation media destination conflict")
|
|
}
|
|
if e = s.assertAnnotationLease(ctx); e != nil {
|
|
return old, e
|
|
}
|
|
return p.PublishConditional(ctx, *c, bucket, key, toKey, meta.SizeBytes, meta.ETag, "", "*", map[string]string{"sha256": sha})
|
|
}
|
|
|
|
func (s *server) publishAnnotationDirectory(ctx context.Context, org, source, target string, manifest map[string]any) (string, int64, error) {
|
|
// Publish all media/sidecars before the generation manifest. Limit network
|
|
// concurrency independently of encoder concurrency.
|
|
group, gctx := errgroup.WithContext(ctx)
|
|
group.SetLimit(4)
|
|
items := listValue(manifest["items"])
|
|
manifest["items"] = items
|
|
for i, raw := range items {
|
|
item := mapValue(raw)
|
|
items[i] = item
|
|
dir := stringValue(item["directory"])
|
|
files := mapValue(item["files"])
|
|
item["files"] = files
|
|
for kind, rawFile := range files {
|
|
file := mapValue(rawFile)
|
|
files[kind] = file
|
|
name := stringValue(file["name"])
|
|
if name == "" || filepath.Base(name) != name || filepath.IsAbs(dir) || strings.Contains(dir, "..") {
|
|
return "", 0, fmt.Errorf("invalid annotation member path")
|
|
}
|
|
group.Go(func() error {
|
|
meta, e := s.publishAnnotationFile(gctx, org, filepath.Join(source, dir, name), filepath.Join(target, dir, name), stringValue(file["sha256"]))
|
|
if e == nil && meta.ETag != "" {
|
|
file["etag"] = meta.ETag
|
|
}
|
|
return e
|
|
})
|
|
}
|
|
}
|
|
if e := group.Wait(); e != nil {
|
|
return "", 0, e
|
|
}
|
|
path := filepath.Join(source, "revision-manifest.json")
|
|
sha, size, e := writeJSONFile(path, manifest)
|
|
if e != nil {
|
|
return "", 0, e
|
|
}
|
|
_, e = s.publishAnnotationFile(ctx, org, path, filepath.Join(target, "revision-manifest.json"), sha)
|
|
return sha, size, e
|
|
}
|