droboticflow-new/internal/api/annotation_performance.go
2026-09-27 05:32:22 -07:00

234 lines
6.7 KiB
Go

package api
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"golang.org/x/sync/errgroup"
)
// Only compact, validated timelines are cached, never IMU samples or user drafts.
// Each hit rechecks source metadata; TTL bounds retention even on a quiet API.
type annotationTimelineEntry struct {
value map[string]any
used, expires time.Time
bytes int
}
type annotationTimelineFlight struct {
done chan struct{}
value map[string]any
err error
}
type annotationTimelineCache struct {
mu sync.Mutex
entries map[string]annotationTimelineEntry
flights map[string]*annotationTimelineFlight
}
func cloneTimeline(value map[string]any) map[string]any {
copy := make(map[string]any, len(value))
for key, v := range value {
switch a := v.(type) {
case []float64:
copy[key] = append([]float64(nil), a...)
case []int64:
copy[key] = append([]int64(nil), a...)
case []int:
copy[key] = append([]int(nil), a...)
default:
copy[key] = v
}
}
return copy
}
func (c *annotationTimelineCache) get(ctx context.Context, key string, build func(context.Context) (map[string]any, error)) (map[string]any, error) {
c.mu.Lock()
if c.entries == nil {
c.entries = make(map[string]annotationTimelineEntry)
c.flights = make(map[string]*annotationTimelineFlight)
}
now := time.Now()
for k, e := range c.entries {
if now.After(e.expires) {
delete(c.entries, k)
}
}
if e, ok := c.entries[key]; ok {
e.used = now
c.entries[key] = e
c.mu.Unlock()
return cloneTimeline(e.value), nil
}
if f, ok := c.flights[key]; ok {
c.mu.Unlock()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-f.done:
return cloneTimeline(f.value), f.err
}
}
f := &annotationTimelineFlight{done: make(chan struct{})}
c.flights[key] = f
c.mu.Unlock()
value, err := build(ctx)
c.mu.Lock()
f.value, f.err = value, err
if err == nil {
encoded, _ := json.Marshal(value)
if len(encoded) <= 8<<20 {
c.entries[key] = annotationTimelineEntry{value: cloneTimeline(value), used: now, expires: time.Now().Add(time.Hour), bytes: len(encoded)}
for {
total := 0
oldest := key
earliest := time.Now()
for k, e := range c.entries {
total += e.bytes
if e.used.Before(earliest) {
oldest, earliest = k, e.used
}
}
if len(c.entries) <= 32 && total <= 32<<20 {
break
}
delete(c.entries, oldest)
}
}
}
delete(c.flights, key)
close(f.done)
c.mu.Unlock()
return cloneTimeline(value), err
}
// No mutable revision/name/disposition belongs in the source identity.
func (s *server) annotationSourceFingerprint(ctx context.Context, task domain.AnnotationTaskRecord, run domain.RunRecord) (string, error) {
var execution *domain.NodeExecutionRecord
for i := range run.NodeExecutions {
if run.NodeExecutions[i].NodeID == task.NodeID {
execution = &run.NodeExecutions[i]
break
}
}
if execution == nil {
return "", fmt.Errorf("annotation human-gate execution is unavailable")
}
left := artifactByID(artifactsForTarget(*execution, "fullLeftVideos"), task.ArtifactID)
if left == nil {
left = artifactByID(execution.InputArtifacts, task.ArtifactID)
}
if left == nil {
return "", fmt.Errorf("left source video is unavailable")
}
collection := artifactCollectionID(*left)
inputs := []*domain.ExecutionArtifact{left,
matchingArtifact(artifactsForTarget(*execution, "fullRightVideos"), collection, "right", ".mp4"),
matchingArtifact(artifactsForTarget(*execution, "leftTimestamps"), collection, "left", "pts"),
matchingArtifact(artifactsForTarget(*execution, "rightTimestamps"), collection, "right", "pts"),
matchingArtifact(artifactsForTarget(*execution, "imu"), collection, "imu", ".csv"),
}
parts := make([]string, len(inputs))
group, ctx := errgroup.WithContext(ctx)
for i, input := range inputs {
group.Go(func() error {
path, err := s.safeMaterializationSourcePath(input)
if err != nil {
return err
}
version, err := s.annotationFileVersion(ctx, task.OrganizationID, path)
if err != nil {
return err
}
encoded, err := json.Marshal(input)
if err != nil {
return err
}
parts[i] = fmt.Sprintf("\n%s:%s:%s", path, version, encoded)
return nil
})
}
if err := group.Wait(); err != nil {
return "", err
}
h := sha256.New()
fmt.Fprintf(h, "timeline-v3:%s:%s:%s:%s:%s", task.OrganizationID, task.RunID, task.NodeID, task.ArtifactID, task.RightArtifactID)
for _, part := range parts {
fmt.Fprint(h, part)
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// BOS-FUSE can synthesize "now" as mtime during reads. Use the authoritative
// object version for a configured workspace; never weaken invalidation to size
// alone. Local files retain size+mtime semantics.
func (s *server) annotationFileVersion(ctx context.Context, org, path string) (string, error) {
connection, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, path, false)
if e != nil {
return "", e
}
if connection == nil {
info, e := os.Stat(path)
if e != nil {
return "", e
}
return fmt.Sprintf("local:%d:%d", info.Size(), info.ModTime().UnixNano()), nil
}
meta, e := (objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}).Head(ctx, *connection, bucket, filepath.ToSlash(key))
if e != nil {
return "", fmt.Errorf("读取源文件对象版本失败:%s", filepath.Base(path))
}
if !meta.Exists || meta.ETag == "" {
return "", fmt.Errorf("源文件对象尚未就绪:%s", filepath.Base(path))
}
return fmt.Sprintf("bos:%s:%s:%s:%s:%d", connection.ID, bucket, meta.ETag, meta.VersionID, meta.SizeBytes), nil
}
func (s *server) annotationFrameTimelineForTask(ctx context.Context, task domain.AnnotationTaskRecord, camera string) (map[string]any, error) {
run, err := s.store.GetRun(ctx, task.RunID)
if err != nil {
return nil, err
}
if run == nil {
return nil, fmt.Errorf("annotation run is unavailable")
}
key, err := s.annotationSourceFingerprint(ctx, task, *run)
if err != nil {
return nil, err
}
value, err := s.annotationTimelines.get(ctx, key, func(ctx context.Context) (map[string]any, error) {
return s.persistentTimeline(ctx, task, key, func(ctx context.Context) (map[string]any, error) {
value, err := s.buildAnnotationFrameTimeline(ctx, task, "left")
if err != nil {
return nil, err
}
after, err := s.annotationSourceFingerprint(ctx, task, *run)
if err != nil {
return nil, err
}
if after != key {
return nil, fmt.Errorf("源文件在校验期间发生变化,请重试")
}
value["sourceFingerprint"] = key
return value, nil
})
})
if err != nil {
return nil, err
}
value["camera"] = camera
if camera == "right" {
value["frameCount"] = value["rightFrameCount"]
}
return value, nil
}