234 lines
6.7 KiB
Go
234 lines
6.7 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/objectsync"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
// Only compact, validated timelines are cached, never IMU samples or user drafts.
|
|
// Each hit rechecks source metadata; TTL bounds retention even on a quiet API.
|
|
type annotationTimelineEntry struct {
|
|
value map[string]any
|
|
used, expires time.Time
|
|
bytes int
|
|
}
|
|
type annotationTimelineFlight struct {
|
|
done chan struct{}
|
|
value map[string]any
|
|
err error
|
|
}
|
|
type annotationTimelineCache struct {
|
|
mu sync.Mutex
|
|
entries map[string]annotationTimelineEntry
|
|
flights map[string]*annotationTimelineFlight
|
|
}
|
|
|
|
func cloneTimeline(value map[string]any) map[string]any {
|
|
copy := make(map[string]any, len(value))
|
|
for key, v := range value {
|
|
switch a := v.(type) {
|
|
case []float64:
|
|
copy[key] = append([]float64(nil), a...)
|
|
case []int64:
|
|
copy[key] = append([]int64(nil), a...)
|
|
case []int:
|
|
copy[key] = append([]int(nil), a...)
|
|
default:
|
|
copy[key] = v
|
|
}
|
|
}
|
|
return copy
|
|
}
|
|
|
|
func (c *annotationTimelineCache) get(ctx context.Context, key string, build func(context.Context) (map[string]any, error)) (map[string]any, error) {
|
|
c.mu.Lock()
|
|
if c.entries == nil {
|
|
c.entries = make(map[string]annotationTimelineEntry)
|
|
c.flights = make(map[string]*annotationTimelineFlight)
|
|
}
|
|
now := time.Now()
|
|
for k, e := range c.entries {
|
|
if now.After(e.expires) {
|
|
delete(c.entries, k)
|
|
}
|
|
}
|
|
if e, ok := c.entries[key]; ok {
|
|
e.used = now
|
|
c.entries[key] = e
|
|
c.mu.Unlock()
|
|
return cloneTimeline(e.value), nil
|
|
}
|
|
if f, ok := c.flights[key]; ok {
|
|
c.mu.Unlock()
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-f.done:
|
|
return cloneTimeline(f.value), f.err
|
|
}
|
|
}
|
|
f := &annotationTimelineFlight{done: make(chan struct{})}
|
|
c.flights[key] = f
|
|
c.mu.Unlock()
|
|
value, err := build(ctx)
|
|
c.mu.Lock()
|
|
f.value, f.err = value, err
|
|
if err == nil {
|
|
encoded, _ := json.Marshal(value)
|
|
if len(encoded) <= 8<<20 {
|
|
c.entries[key] = annotationTimelineEntry{value: cloneTimeline(value), used: now, expires: time.Now().Add(time.Hour), bytes: len(encoded)}
|
|
for {
|
|
total := 0
|
|
oldest := key
|
|
earliest := time.Now()
|
|
for k, e := range c.entries {
|
|
total += e.bytes
|
|
if e.used.Before(earliest) {
|
|
oldest, earliest = k, e.used
|
|
}
|
|
}
|
|
if len(c.entries) <= 32 && total <= 32<<20 {
|
|
break
|
|
}
|
|
delete(c.entries, oldest)
|
|
}
|
|
}
|
|
}
|
|
delete(c.flights, key)
|
|
close(f.done)
|
|
c.mu.Unlock()
|
|
return cloneTimeline(value), err
|
|
}
|
|
|
|
// No mutable revision/name/disposition belongs in the source identity.
|
|
func (s *server) annotationSourceFingerprint(ctx context.Context, task domain.AnnotationTaskRecord, run domain.RunRecord) (string, error) {
|
|
var execution *domain.NodeExecutionRecord
|
|
for i := range run.NodeExecutions {
|
|
if run.NodeExecutions[i].NodeID == task.NodeID {
|
|
execution = &run.NodeExecutions[i]
|
|
break
|
|
}
|
|
}
|
|
if execution == nil {
|
|
return "", fmt.Errorf("annotation human-gate execution is unavailable")
|
|
}
|
|
left := artifactByID(artifactsForTarget(*execution, "fullLeftVideos"), task.ArtifactID)
|
|
if left == nil {
|
|
left = artifactByID(execution.InputArtifacts, task.ArtifactID)
|
|
}
|
|
if left == nil {
|
|
return "", fmt.Errorf("left source video is unavailable")
|
|
}
|
|
collection := artifactCollectionID(*left)
|
|
inputs := []*domain.ExecutionArtifact{left,
|
|
matchingArtifact(artifactsForTarget(*execution, "fullRightVideos"), collection, "right", ".mp4"),
|
|
matchingArtifact(artifactsForTarget(*execution, "leftTimestamps"), collection, "left", "pts"),
|
|
matchingArtifact(artifactsForTarget(*execution, "rightTimestamps"), collection, "right", "pts"),
|
|
matchingArtifact(artifactsForTarget(*execution, "imu"), collection, "imu", ".csv"),
|
|
}
|
|
parts := make([]string, len(inputs))
|
|
group, ctx := errgroup.WithContext(ctx)
|
|
for i, input := range inputs {
|
|
group.Go(func() error {
|
|
path, err := s.safeMaterializationSourcePath(input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
version, err := s.annotationFileVersion(ctx, task.OrganizationID, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
encoded, err := json.Marshal(input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
parts[i] = fmt.Sprintf("\n%s:%s:%s", path, version, encoded)
|
|
return nil
|
|
})
|
|
}
|
|
if err := group.Wait(); err != nil {
|
|
return "", err
|
|
}
|
|
h := sha256.New()
|
|
fmt.Fprintf(h, "timeline-v3:%s:%s:%s:%s:%s", task.OrganizationID, task.RunID, task.NodeID, task.ArtifactID, task.RightArtifactID)
|
|
for _, part := range parts {
|
|
fmt.Fprint(h, part)
|
|
}
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
|
|
// BOS-FUSE can synthesize "now" as mtime during reads. Use the authoritative
|
|
// object version for a configured workspace; never weaken invalidation to size
|
|
// alone. Local files retain size+mtime semantics.
|
|
func (s *server) annotationFileVersion(ctx context.Context, org, path string) (string, error) {
|
|
connection, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, path, false)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
if connection == nil {
|
|
info, e := os.Stat(path)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
return fmt.Sprintf("local:%d:%d", info.Size(), info.ModTime().UnixNano()), nil
|
|
}
|
|
meta, e := (objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}).Head(ctx, *connection, bucket, filepath.ToSlash(key))
|
|
if e != nil {
|
|
return "", fmt.Errorf("读取源文件对象版本失败:%s", filepath.Base(path))
|
|
}
|
|
if !meta.Exists || meta.ETag == "" {
|
|
return "", fmt.Errorf("源文件对象尚未就绪:%s", filepath.Base(path))
|
|
}
|
|
return fmt.Sprintf("bos:%s:%s:%s:%s:%d", connection.ID, bucket, meta.ETag, meta.VersionID, meta.SizeBytes), nil
|
|
}
|
|
|
|
func (s *server) annotationFrameTimelineForTask(ctx context.Context, task domain.AnnotationTaskRecord, camera string) (map[string]any, error) {
|
|
run, err := s.store.GetRun(ctx, task.RunID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if run == nil {
|
|
return nil, fmt.Errorf("annotation run is unavailable")
|
|
}
|
|
key, err := s.annotationSourceFingerprint(ctx, task, *run)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
value, err := s.annotationTimelines.get(ctx, key, func(ctx context.Context) (map[string]any, error) {
|
|
return s.persistentTimeline(ctx, task, key, func(ctx context.Context) (map[string]any, error) {
|
|
value, err := s.buildAnnotationFrameTimeline(ctx, task, "left")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
after, err := s.annotationSourceFingerprint(ctx, task, *run)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if after != key {
|
|
return nil, fmt.Errorf("源文件在校验期间发生变化,请重试")
|
|
}
|
|
value["sourceFingerprint"] = key
|
|
return value, nil
|
|
})
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
value["camera"] = camera
|
|
if camera == "right" {
|
|
value["frameCount"] = value["rightFrameCount"]
|
|
}
|
|
return value, nil
|
|
}
|