⚡ Persist frame indices and isolate resumable annotation processing

This commit is contained in:
eust-w 2026-09-17 18:40:06 +08:00
parent ffa43bd5b9
commit cc0202d40d
35 changed files with 2425 additions and 169 deletions

View File

@ -8,6 +8,7 @@ import (
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
@ -90,6 +91,43 @@ func main() {
}()
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
if os.Getenv("ANNOTATION_PROCESSOR_MODE") == "processor" {
if err := dataStore.EnsureIndexes(ctx); err != nil {
fmt.Printf("[annotation-processor] indexes: %v\n", err)
os.Exit(1)
}
parallel := 2
if value, err := strconv.Atoi(os.Getenv("ANNOTATION_PROCESSOR_CONCURRENCY")); err == nil {
parallel = value
}
health := &http.Server{Addr: "0.0.0.0:3003", ReadHeaderTimeout: 5 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/ready" {
http.NotFound(w, r)
return
}
if e := dataStore.Ready(r.Context()); e != nil {
http.Error(w, "store unavailable", 503)
return
}
w.WriteHeader(200)
_, _ = w.Write([]byte("annotation processor ready"))
})}
go func() {
if e := health.ListenAndServe(); e != nil && e != http.ErrServerClosed {
stop()
}
}()
fmt.Println("[annotation-processor] durable worker and index precompute started")
err := api.RunAnnotationProcessor(ctx, dataStore, cfg, parallel, os.Getenv("ANNOTATION_INDEX_BACKFILL") == "1")
shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = health.Shutdown(shutdown)
if err != nil && ctx.Err() == nil {
fmt.Printf("[annotation-processor] failed: %v\n", err)
os.Exit(1)
}
return
}
if err := dataStore.RunMigrations(ctx); err != nil {
fmt.Printf("[api] failed to run mongo migrations: %v\n", err)
os.Exit(1)

View File

@ -0,0 +1,30 @@
# Durable annotation processing
## Compatibility boundary
No change to dictionary selection, role/supplier ownership, semantic blanks, explicit invalid intervals, discard evidence, acceptance rules, exact frame boundaries or delivery directory layout. Materialization manifests remain v2/v3. Existing generation files remain readable. New media objects are independently owned by each generation; no symlinks or references into a generation that retention can delete.
## Performance paths
1. New full-video annotation tasks enqueue index preparation. The independent processor builds a source-version-keyed timeline and stores a compressed, checksummed document in `annotationTimelineIndexes`. API restarts and memory-cache eviction no longer require rebuilding an existing index. Source ETag/version changes invalidate it. Legacy/open tasks can be backfilled without creating Runs or changing assignments.
2. When only names/descriptions/disposition change and all frame windows still match, the metadata-only path validates the source version and prior manifest, copies verified media/PTS/IMU objects server-side in BOS, and rewrites only small annotation/synchronization manifests locally. It does not reread original IMU/PTS or decode video. A native-filesystem installation retains checked file copies as a compatibility fallback. Old BOS generations without pinned object ETags receive normal verification once.
3. New/changed slices share input decoding in groups of up to eight outputs. Every output still uses half-open original-frame trims and decoded frame-count checks. Tests compare decoded frame hashes against the single-output implementation.
4. Generation publication uses at most four concurrent BOS transfers and publishes its manifest last. Final delivery also uses bounded BOS concurrency, retaining conditional ownership checks, digest verification, retry semantics and manifest-last ordering.
5. The API validates a submission once and atomically stores the durable outbox in the HumanItem CAS. It returns processing, not success. The processor rechecks current actor/project/role/source ownership and uses an expiring, fenced claim. A stale worker cannot publish after takeover. Worker Run snapshots preserve API-published materializations.
6. Task-pool cards use an authorized task-summary view instead of loading each task's full history. Processing polls use the existing lightweight status view. Histories are loaded for editing, not every progress tick.
## Operation
- Default `ANNOTATION_PROCESSOR_MODE=inprocess` preserves existing deployments until explicitly switched.
- The API uses `ANNOTATION_PROCESSOR_MODE=external` after the independent processor is healthy.
- The processor runs the same API image/binary with `ANNOTATION_PROCESSOR_MODE=processor`. It has no public task-mutation endpoint; `/ready` on port 3003 is for health checks only. It does not need a Kubernetes service-account token.
- `ANNOTATION_PROCESSOR_CONCURRENCY=2` is the default (allowed 1–4 per process). Multiple replicas claim independently from the database. Local `/tmp` needs bounded scratch capacity; the supplied deployment uses a 20 GiB emptyDir.
- `ANNOTATION_INDEX_BACKFILL=1` queues existing pending full-video annotation tasks at startup. Index work has a separate bounded loop. Precomputation may take time for a backlog; it is not a claim that every newly discovered file is immediately ready.
- Processing leases last two minutes and renew every ten seconds. Interrupted work is requeued on graceful shutdown, or reclaimed after lease expiry. Repeated interruptions stop after three attempts with the saved workspace intact. Legacy API-only reservations can be adopted without changing the visible business version.
- BOS workspace mapping uses `ANNOTATION_WORKSPACE_CONNECTION_ID` / `ANNOTATION_WORKSPACE_BUCKET`, with the prior project-deletion workspace mapping as a fallback. Native local files use filesystem versions.
- Manifest rendering: set `ANNOTATION_PROCESSOR_ENABLED=true` plus the workspace mapping to render `k8s/annotation-processor.template.yaml` and switch API mode. Keep the processor on the same code/image release as the API when deploying future changes. Roll out the processor before enabling external mode; inspect existing legacy in-flight submissions before changing modes.
- Project/workflow deletion removes index/cache bookkeeping with the associated scope. Media retention still removes only superseded private generation paths, never original inputs.
## Verification
Focused tests cover durable outbox BSON/JSON privacy, concurrent exclusive claims, expired-owner fencing, process replacement, source changes, persistent index recovery/corruption, proactive indexing, semantic-empty submission, metadata-only generation, decoded frame equivalence and delivery ownership/retry behavior. Live load tests must use a separate database/processor and private Run prefixes; production source objects are read-only. Report the tested concurrency and actual video duration rather than extrapolating unlimited capacity.

View File

@ -0,0 +1,24 @@
package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func (s *server) storeAnnotationMaterializationArtifact(ctx context.Context, t domain.AnnotationTaskRecord, run *domain.RunRecord, a domain.ExecutionArtifact) error {
if e := s.assertAnnotationLease(ctx); e != nil {
return e
}
if db, ok := s.store.(store.AnnotationProcessingStore); ok {
return db.AppendAnnotationArtifact(ctx, t.OrganizationID, t.NodeID, a)
}
run.Artifacts = append(run.Artifacts, a)
for i := range run.NodeExecutions {
if run.NodeExecutions[i].NodeID == t.NodeID {
run.NodeExecutions[i].Artifacts = append(run.NodeExecutions[i].Artifacts, a)
}
}
run.UpdatedAt = nowISO()
return s.store.UpsertRun(ctx, *run)
}

View File

@ -0,0 +1,43 @@
package api
import (
"context"
"fmt"
"os/exec"
"strings"
)
type annotationFrameSlice struct {
Start, End int
Output string
}
// Each input is decoded once per bounded group, not once per output slice.
// Half-open frame trims preserve exact original frame selection; no fast seek
// or keyframe-only stream copy is used.
func writeFrameAccurateSlices(ctx context.Context, source string, slices []annotationFrameSlice) error {
for offset := 0; offset < len(slices); offset += 8 {
group := slices[offset:min(offset+8, len(slices))]
filters := []string{}
split := fmt.Sprintf("[0:v]split=%d", len(group))
for i := range group {
split += fmt.Sprintf("[in%d]", i)
}
filters = append(filters, split)
for i, s := range group {
if s.Start < 0 || s.End <= s.Start {
return fmt.Errorf("invalid frame slice")
}
filters = append(filters, fmt.Sprintf("[in%d]trim=start_frame=%d:end_frame=%d,setpts=PTS-STARTPTS[out%d]", i, s.Start, s.End, i))
}
args := []string{"-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-threads", "2", "-i", source, "-filter_complex_threads", "1", "-filter_complex", strings.Join(filters, ";")}
for i, s := range group {
args = append(args, "-map", fmt.Sprintf("[out%d]", i), "-an", "-c:v", "libx264", "-threads", "2", "-preset", "ultrafast", "-crf", "23", "-pix_fmt", "yuv420p", s.Output)
}
out, e := exec.CommandContext(ctx, "ffmpeg", args...).CombinedOutput()
if e != nil {
return fmt.Errorf("batch frame slicing: %w: %s", e, out)
}
}
return nil
}

View File

@ -0,0 +1,41 @@
package api
import (
"context"
"os/exec"
"path/filepath"
"strings"
"testing"
)
func TestBatchDecoderMatchesSingleDecoderFrameContent(t *testing.T) {
f, root := annotationProcessorFixture(t)
_ = f
source := filepath.Join(root, "left.mp4")
cuts := []annotationFrameSlice{{0, 9, filepath.Join(root, "batch-a.mp4")}, {9, 24, filepath.Join(root, "batch-b.mp4")}, {24, 32, filepath.Join(root, "batch-c.mp4")}}
if e := writeFrameAccurateSlices(context.Background(), source, cuts); e != nil {
t.Fatal(e)
}
frames := func(path string) string {
out, e := exec.Command("ffmpeg", "-v", "error", "-i", path, "-f", "framemd5", "-").Output()
if e != nil {
t.Fatal(e)
}
lines := []string{}
for _, line := range strings.Split(string(out), "\n") {
if line != "" && !strings.HasPrefix(line, "#") {
lines = append(lines, line)
}
}
return strings.Join(lines, "\n")
}
for i, c := range cuts {
single := filepath.Join(root, []string{"single-a.mp4", "single-b.mp4", "single-c.mp4"}[i])
if e := writeFrameAccurateSlice(context.Background(), source, single, c.Start, c.End); e != nil {
t.Fatal(e)
}
if frames(single) != frames(c.Output) {
t.Fatalf("batch changed decoded frame content for %d", i)
}
}
}

View File

@ -74,8 +74,12 @@ func (s *server) retainCurrentAnnotationData(ctx context.Context, task domain.An
if pathErr != nil && !os.IsNotExist(pathErr) {
return pathErr
}
if pathErr == nil && resolved != dir {
return fmt.Errorf("refusing symlinked annotation directory")
if pathErr == nil {
rootReal, rootErr := filepath.EvalSymlinks(s.workspaceMountPath)
relative, relErr := filepath.Rel(s.workspaceMountPath, dir)
if rootErr != nil || relErr != nil || resolved != filepath.Join(rootReal, relative) {
return fmt.Errorf("refusing symlinked annotation directory")
}
}
if err := os.RemoveAll(dir); err != nil {
return err

View File

@ -0,0 +1,282 @@
package api
import (
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"encoding/hex"
"encoding/json"
"fmt"
"golang.org/x/sync/errgroup"
"log"
"math"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
)
func readVerifiedAnnotationJSON(path, sha string) (map[string]any, error) {
b, e := os.ReadFile(path)
if e != nil {
return nil, e
}
sum := sha256.Sum256(b)
if sha == "" || hex.EncodeToString(sum[:]) != sha {
return nil, fmt.Errorf("annotation manifest checksum mismatch")
}
var m map[string]any
e = json.Unmarshal(b, &m)
return m, e
}
// No source IMU/PTS reads, video decoding or application-side media transfer
// on the BOS fast path. New generations own independent server-side copies.
func (s *server) tryAnnotationMetadataRevision(ctx context.Context, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, run *domain.RunRecord, payload combinedAnnotationSubmissionInput, revision int, fingerprint string) (*humanSliceMaterialization, error) {
prefix := fmt.Sprintf("%s-%s-%s-materialized-revision-", task.RunID, task.NodeID, task.ID)
var prior *domain.ExecutionArtifact
highest := 0
slices := append([]domain.SliceSegment(nil), payload.SliceSegments...)
sort.Slice(slices, func(i, j int) bool { return slices[i].Start < slices[j].Start })
for i := range run.Artifacts {
a := &run.Artifacts[i]
n, e := strconv.Atoi(strings.TrimPrefix(a.ID, prefix))
if e != nil || !strings.HasPrefix(a.ID, prefix) || n >= revision || n <= highest || stringValue(a.Data["sourceFingerprint"]) != fingerprint {
continue
}
old := decodeMaterializationValue[domain.SliceSegment](a.Data["normalizedSlices"])
if len(old) != len(slices) || len(old) == 0 {
continue
}
same := true
for j := range old {
if math.Abs(old[j].Start-slices[j].Start) > 1e-9 || math.Abs(old[j].End-slices[j].End) > 1e-9 {
same = false
}
}
if same {
prior = a
highest = n
}
}
if prior == nil || prior.Metadata == nil {
return nil, nil
}
base := filepath.Join(s.workspaceMountPath, "system", "runs", task.RunID, "human-materializations", item.ID, task.ID)
if prior.Path != filepath.Join(base, fmt.Sprintf("revision-%06d", highest)) {
return nil, nil
}
manifest, e := readVerifiedAnnotationJSON(filepath.Join(prior.Path, "revision-manifest.json"), prior.Metadata.SHA256)
if e != nil {
return nil, nil
}
entries := listValue(manifest["items"])
if len(entries) != len(slices) {
return nil, nil
}
timeline, e := s.annotationFrameTimelineForTask(ctx, task, "left")
if e != nil {
return nil, e
}
if stringValue(timeline["sourceFingerprint"]) != fingerprint {
return nil, fmt.Errorf("source changed during annotation update")
}
grid := annotationBoundaryGrid{Seconds: timeline["boundariesSeconds"].([]float64), Timestamps: timeline["boundaryTimestampUs"].([]int64)}
segments := append([]domain.AnnotationSegment{}, payload.Segments...)
for i := range segments {
a, e := grid.resolve(segments[i].Start)
if e != nil {
return nil, e
}
b, e := grid.resolve(segments[i].End)
if e != nil || b <= a {
return nil, fmt.Errorf("invalid annotation boundary")
}
segments[i].Start, segments[i].End = grid.Seconds[a], grid.Seconds[b]
segments[i].StartTimestampUS, segments[i].EndTimestampUS = grid.Timestamps[a], grid.Timestamps[b]
}
if issues := validateDescriptionIntervals(slices, segments, task.TemplateVersion < 6); len(issues) > 0 {
return nil, fmt.Errorf("%s", issues[0].Message)
}
stage, e := os.MkdirTemp("", "annotation-metadata-")
if e != nil {
return nil, e
}
defer os.RemoveAll(stage)
destination := filepath.Join(base, fmt.Sprintf("revision-%06d", revision))
kept, discarded := 0, 0
keptDuration, discardedDuration := 0.0, 0.0
for i, raw := range entries {
entry := mapValue(raw)
entries[i] = entry
slice := &slices[i]
oldDir := stringValue(entry["directory"])
if oldDir != filepath.ToSlash(filepath.Join("slices", fmt.Sprintf("slice-%06d", i+1))) {
return nil, nil
}
files := mapValue(entry["files"])
entry["files"] = files
syncFile := mapValue(files["synchronization"])
synchronized, e := readVerifiedAnnotationJSON(filepath.Join(prior.Path, oldDir, "synchronized-slice.json"), stringValue(syncFile["sha256"]))
if e != nil {
return nil, nil
}
slice.StartTimestampUS = int64Value(entry["startTimestampUs"])
slice.EndTimestampUS = int64Value(entry["endTimestampUs"])
if slice.IsDiscarded() {
discarded++
discardedDuration += slice.End - slice.Start
} else {
kept++
keptDuration += slice.End - slice.Start
}
entry["sourceSliceId"], entry["sliceName"], entry["deliveryDisposition"] = slice.ID, slice.Name, sliceDisposition(*slice)
synchronized["sliceName"], synchronized["deliveryDisposition"] = slice.Name, sliceDisposition(*slice)
delete(entry, "discardEvidence")
delete(synchronized, "discardEvidence")
if slice.IsDiscarded() {
evidence := map[string]any{"reason": slice.DiscardReason, "at": slice.DiscardedAt, "previousAnnotations": slice.DiscardedAnnotations}
entry["discardEvidence"] = evidence
synchronized["discardEvidence"] = evidence
}
actions := []map[string]any{}
for _, segment := range segments {
if slice.IsDiscarded() || segment.End <= slice.Start || segment.Start >= slice.End {
continue
}
actions = append(actions, map[string]any{"id": segment.ID, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": (segment.Start - slice.Start) * 1000, "clipEndMs": (segment.End - slice.Start) * 1000, "startTimestampUs": segment.StartTimestampUS, "endTimestampUs": segment.EndTimestampUS})
}
annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": entry["sliceId"], "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions}
if e = os.MkdirAll(filepath.Join(stage, oldDir), 0755); e != nil {
return nil, e
}
for name, value := range map[string]map[string]any{"annotation": annotation, "synchronization": synchronized} {
filename := "annotation.json"
if name == "synchronization" {
filename = "synchronized-slice.json"
}
sha, size, e := writeJSONFile(filepath.Join(stage, oldDir, filename), value)
if e != nil {
return nil, e
}
files[name] = map[string]any{"name": filename, "sha256": sha, "bytes": size}
}
}
// Verify all old metadata before copying any members. A legacy generation
// without BOS digest/ETag metadata safely falls back to normal generation.
var copiedVersions sync.Map
preflight, pctx := errgroup.WithContext(ctx)
preflight.SetLimit(4)
for _, raw := range entries {
entry := mapValue(raw)
dir := stringValue(entry["directory"])
files := mapValue(entry["files"])
for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
file := mapValue(files[kind])
name := stringValue(file["name"])
if name == "" || filepath.Base(name) != name {
return nil, nil
}
preflight.Go(func() error {
return s.annotationMediaCopyReady(pctx, task.OrganizationID, filepath.Join(prior.Path, dir, name), file)
})
}
}
if e = preflight.Wait(); e != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, nil
}
group, gctx := errgroup.WithContext(ctx)
group.SetLimit(4)
for _, raw := range entries {
entry := mapValue(raw)
dir := stringValue(entry["directory"])
files := mapValue(entry["files"])
for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
file := mapValue(files[kind])
name := stringValue(file["name"])
if name == "" || filepath.Base(name) != name {
return nil, nil
}
group.Go(func() error {
target := filepath.Join(destination, dir, name)
meta, e := s.copyAnnotationObject(gctx, task.OrganizationID, filepath.Join(prior.Path, dir, name), target, file)
if e == nil {
copiedVersions.Store(target, meta.ETag)
}
return e
})
}
}
if e = group.Wait(); e != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
log.Printf("[api] metadata reuse fallback task=%s: %v", task.ID, e)
return nil, nil
}
// New copy ETags may differ for multipart/server-side copies. Pin them in
// the next manifest, without downloading media or parsing the original IMU.
for i, raw := range entries {
entry := mapValue(raw)
entries[i] = entry
files := mapValue(entry["files"])
entry["files"] = files
dir := stringValue(entry["directory"])
for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
file := mapValue(files[kind])
files[kind] = file
if tag, ok := copiedVersions.Load(filepath.Join(destination, dir, stringValue(file["name"]))); ok && tag.(string) != "" {
file["etag"] = tag.(string)
}
}
for _, kind := range []string{"annotation", "synchronization"} {
file := mapValue(files[kind])
files[kind] = file
name := stringValue(file["name"])
meta, e := s.publishAnnotationFile(ctx, task.OrganizationID, filepath.Join(stage, dir, name), filepath.Join(destination, dir, name), stringValue(file["sha256"]))
if e != nil {
return nil, e
}
if meta.ETag != "" {
file["etag"] = meta.ETag
}
}
}
selection := map[string]any{"keptSliceCount": kept, "discardedSliceCount": discarded, "keptDurationSeconds": keptDuration, "discardedDurationSeconds": discardedDuration}
manifest["items"], manifest["revision"], manifest["schemaVersion"] = entries, revision, "droboticflow.slice-materialization/v2"
delete(manifest, "deliverySelection")
if discarded > 0 {
manifest["schemaVersion"] = "droboticflow.slice-materialization/v3"
manifest["deliverySelection"] = selection
}
sha, size, e := writeJSONFile(filepath.Join(stage, "revision-manifest.json"), manifest)
if e != nil {
return nil, e
}
if e = s.assertAnnotationLease(ctx); e != nil {
return nil, e
}
if _, e = s.publishAnnotationFile(ctx, task.OrganizationID, filepath.Join(stage, "revision-manifest.json"), filepath.Join(destination, "revision-manifest.json"), sha); e != nil {
return nil, e
}
artifact := *prior
artifact.ID = prefix + fmt.Sprintf("%06d", revision)
artifact.Name = fmt.Sprintf("revision-%06d.directory.index", revision)
artifact.Path = destination
artifact.SizeBytes = size
artifact.Metadata = &domain.ArtifactMetadata{SHA256: sha, MimeType: "application/json"}
artifact.Data = mapValue(prior.Data)
artifact.Data["rootPath"], artifact.Data["manifestPath"], artifact.Data["revision"] = destination, filepath.Join(destination, "revision-manifest.json"), revision
artifact.Data["normalizedSlices"], artifact.Data["normalizedSegments"], artifact.Data["deliverySelection"], artifact.Data["reusedSliceCount"], artifact.Data["metadataOnly"] = slices, segments, selection, len(slices), true
if e = s.storeAnnotationMaterializationArtifact(ctx, task, run, artifact); e != nil {
return nil, e
}
coverage := mapValue(manifest["coverage"])
log.Printf("[api] metadata-only revision task=%s revision=%d slices=%d", task.ID, revision, len(slices))
return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(slices), "coverage": coverage, "deliverySelection": selection, "runId": task.RunID, "artifactId": artifact.ID, "manifestSha256": sha, "metadataOnly": true}, Slices: slices, Segments: segments, CoverageStartSecond: floatValue(coverage["startSecond"]), CoverageEndSecond: floatValue(coverage["endSecond"])}, nil
}

View File

@ -1,13 +1,11 @@
package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"log"
"math"
"net/http"
@ -629,6 +627,10 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques
}
// Polling must not load histories, enrich media, recover state, or rebuild
// timelines. A transition/interrupted job asks the client to fetch detail once.
if request.URL.Query().Get("view") == "summary" {
renderJSON(writer, http.StatusOK, map[string]any{"task": item})
return
}
if request.URL.Query().Get("view") == "status" {
status := map[string]any{"taskId": item.ID, "taskStatus": item.Status}
if progress, ok := server.materializationProgress.Load(item.OrganizationID + ":" + item.ID); ok {
@ -650,6 +652,15 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques
status["needsRefresh"] = true
}
}
if j := linkedHumanItem.ProcessingJob; j != nil {
status["progress"] = map[string]any{"stage": j.Stage, "completedSlices": j.CompletedSlices, "totalSlices": j.TotalSlices, "startedAt": j.CreatedAt, "attempt": j.Attempt}
if durableAnnotationActive(linkedHumanItem) {
status["needsRefresh"] = false
}
if j.Status == "failed" {
status["error"] = j.Error
}
}
}
renderJSON(writer, http.StatusOK, status)
return
@ -666,7 +677,7 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques
if linkedHumanItem.PendingCombinedSubmissionID != "" {
jobKey := linkedHumanItem.OrganizationID + ":" + linkedHumanItem.PendingCombinedSubmissionID
_, jobActive := server.combinedSubmissionJobs.Load(jobKey)
if !jobActive {
if !jobActive && !durableAnnotationActive(linkedHumanItem) {
revisions, _ := server.store.ListAnnotationRevisions(request.Context(), item.OrganizationID, item.ID)
submissionRevisionFound := false
submissionCreatedAt := ""
@ -884,7 +895,7 @@ func (server *server) buildAnnotationFrameTimeline(ctx context.Context, task dom
for i, f := range grid.RightFrames {
rightSeconds[i] = float64(f) * rightDuration / float64(len(rightRows))
}
return map[string]any{"camera": camera, "frameCount": len(rows), "originTimestampUs": grid.Origin, "coverageEndTimestampUs": end, "boundariesSeconds": grid.Seconds, "rightBoundariesSeconds": rightSeconds, "boundaryTimestampUs": grid.Timestamps, "leftFrameIndices": grid.LeftFrames, "rightFrameIndices": grid.RightFrames, "rangeStartSecond": grid.Seconds[0], "rangeEndSecond": grid.Seconds[len(grid.Seconds)-1], "boundaryContract": "hardware-frame-v1", "ptsToMediaScale": grid.Scale}, nil
return map[string]any{"camera": camera, "frameCount": len(rows), "leftFrameCount": lc, "rightFrameCount": rc, "originTimestampUs": grid.Origin, "coverageEndTimestampUs": end, "boundariesSeconds": grid.Seconds, "rightBoundariesSeconds": rightSeconds, "boundaryTimestampUs": grid.Timestamps, "leftFrameIndices": grid.LeftFrames, "rightFrameIndices": grid.RightFrames, "rangeStartSecond": grid.Seconds[0], "rangeEndSecond": grid.Seconds[len(grid.Seconds)-1], "boundaryContract": "hardware-frame-v1", "ptsToMediaScale": grid.Scale}, nil
}
func (server *server) handleAnnotationTaskFrameTimelineGet(writer http.ResponseWriter, request *http.Request) {
@ -1300,7 +1311,7 @@ func validateDraftStatuses(statuses map[string]domain.SliceAnnotationStatus, sli
// reserveFanyuV5CombinedSubmission wins a CAS before either revision write.
// A retry with the same payload token can resume; a different same-version
// submission cannot overwrite partial or completed revision records.
func (server *server) reserveFanyuV5CombinedSubmission(ctx context.Context, item domain.HumanItemRecord, expectedVersion int, submissionID string, payloadSHA string) (*domain.HumanItemRecord, bool, error) {
func (server *server) reserveFanyuV5CombinedSubmission(ctx context.Context, item domain.HumanItemRecord, expectedVersion int, submissionID string, payloadSHA string, jobs ...*domain.AnnotationProcessingJob) (*domain.HumanItemRecord, bool, error) {
if item.PendingCombinedSubmissionID != "" {
// Older clients could keep confirming descriptions after a submission
// reserved its version. That writer can no longer finish its CAS. Only
@ -1318,6 +1329,20 @@ func (server *server) reserveFanyuV5CombinedSubmission(ctx context.Context, item
item.PendingCombinedPayloadSHA = ""
} else {
if item.PendingCombinedPayloadSHA == payloadSHA {
if len(jobs) > 0 && jobs[0] != nil && (item.ProcessingJob == nil || item.ProcessingJob.ID != item.PendingCombinedSubmissionID) {
adopted := item
job := *jobs[0]
job.ID = item.PendingCombinedSubmissionID
adopted.ProcessingJob = &job
ok, e := server.store.UpdateHumanItemVersioned(ctx, adopted, item.Version)
if e != nil {
return nil, false, e
}
if !ok {
return nil, false, fmt.Errorf("combined submission adoption conflict")
}
return &adopted, true, nil
}
return &item, true, nil
}
return nil, false, fmt.Errorf("a different combined submission is already reserved")
@ -1328,6 +1353,17 @@ func (server *server) reserveFanyuV5CombinedSubmission(ctx context.Context, item
reserved.Version++
reserved.PendingCombinedSubmissionID = submissionID
reserved.PendingCombinedPayloadSHA = payloadSHA
if len(jobs) > 0 && jobs[0] != nil {
reserved.ProcessingJob = jobs[0]
if jobs[0].Kind == "submit" {
reserved.Subject.Payload = mapValue(item.Subject.Payload)
if reserved.Subject.Payload == nil {
reserved.Subject.Payload = map[string]any{}
}
in := jobs[0].Input
reserved.Subject.Payload["annotationWorkspace"] = map[string]any{"sliceSegments": in["sliceSegments"], "operations": in["sliceOperations"], "segments": in["segments"], "userId": jobs[0].ActorID, "savedAt": nowISO()}
}
}
updated, err := server.store.UpdateHumanItemVersioned(ctx, reserved, item.Version)
if err != nil {
return nil, false, err
@ -1583,7 +1619,11 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
}
done := make(chan struct{})
heartbeatStopped := make(chan struct{})
go func() { defer close(heartbeatStopped); server.keepSliceLease(ctx, cancel, itemID, requestID, done) }()
if _, durable := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease); durable {
close(heartbeatStopped)
} else {
go func() { defer close(heartbeatStopped); server.keepSliceLease(ctx, cancel, itemID, requestID, done) }()
}
slices := descriptionOnlySlices(payload.SliceSegments)
sliceRevision := domain.SliceRevisionRecord{
ID: "slice-confirmation-" + requestID, OrganizationID: task.OrganizationID, TaskID: task.ID,
@ -1593,6 +1633,9 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
materialization, processErr := server.materializeCombinedSliceSubmission(ctx, *task, *item, combinedAnnotationSubmissionInput{SliceSegments: slices, Segments: []domain.AnnotationSegment{}}, revision)
close(done)
<-heartbeatStopped
if _, durable := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease); durable && (ctx.Err() != nil || server.assertAnnotationLease(ctx) != nil) {
return
}
// Persist failure even when ffmpeg was cancelled by the deadline/lease.
ctx, cleanup := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cleanup()
@ -1600,6 +1643,9 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
if currentErr != nil || current == nil {
return
}
if lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease); ok && (current.ProcessingJob == nil || current.ProcessingJob.Owner != lease.Owner) {
return
}
workflow := mapValue(current.Subject.Payload["sliceWorkflow"])
if strings.TrimSpace(fmt.Sprint(workflow["requestId"])) != requestID || stringValue(workflow["status"]) != "processing" {
return
@ -1744,6 +1790,14 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter,
"segments": []domain.AnnotationSegment{}, "statuses": pendingSliceStatuses(payload.SliceSegments),
"userId": principal.User.ID, "savedAt": now,
}
if server.externalAnnotationProcessor {
job, err := server.newAnnotationProcessingJob(request.Context(), *task, principal.User.ID, requestID, "slice", payload, revision)
if err != nil {
renderError(writer, http.StatusUnprocessableEntity, err)
return
}
item.ProcessingJob = job
}
expectedVersion := item.Version
item.Version++
item.UpdatedAt = now
@ -1757,7 +1811,9 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter,
return
}
_ = server.appendHumanEvent(request.Context(), *item, "slice_confirmation_started", principal.User.ID, map[string]any{"requestId": requestID, "revision": revision, "sliceCount": len(payload.SliceSegments)})
go server.finishSliceConfirmation(context.WithoutCancel(request.Context()), task.ID, item.ID, requestID, payload, revision, principal.User.ID)
if !server.externalAnnotationProcessor {
go server.finishSliceConfirmation(context.WithoutCancel(request.Context()), task.ID, item.ID, requestID, payload, revision, principal.User.ID)
}
renderJSON(writer, http.StatusAccepted, map[string]any{"item": item, "sliceWorkflow": item.Subject.Payload["sliceWorkflow"]})
}
@ -1832,6 +1888,11 @@ func (server *server) handleSliceResetCreate(writer http.ResponseWriter, request
}
delete(item.Subject.Payload, "annotationDraft")
item.Subject.Payload["annotationWorkspace"] = resetDraft
if item.ProcessingJob != nil {
item.ProcessingJob.Status = "cancelled"
item.ProcessingJob.Input = nil
item.ProcessingJob.Owner = ""
}
item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "editing", "phase": "slice", "resetAt": now, "revision": intValue(mapValue(item.Subject.Payload["sliceWorkflow"])["revision"])}
expectedVersion := item.Version
item.Version++
@ -1935,7 +1996,15 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
return
}
submissionRef := fanyuV5CombinedSubmissionID(humanItem.ID, payload.Version)
reservedItem, resumed, reserveErr := server.reserveFanyuV5CombinedSubmission(request.Context(), *humanItem, payload.Version, submissionRef, payloadSHA)
var processingJob *domain.AnnotationProcessingJob
if server.externalAnnotationProcessor && strings.Contains(strings.ToLower(request.Header.Get("Prefer")), "respond-async") {
processingJob, err = server.newAnnotationProcessingJob(request.Context(), *task, principal.User.ID, submissionRef, "submit", payload, 0)
if err != nil {
renderError(writer, http.StatusUnprocessableEntity, err)
return
}
}
reservedItem, resumed, reserveErr := server.reserveFanyuV5CombinedSubmission(request.Context(), *humanItem, payload.Version, submissionRef, payloadSHA, processingJob)
if reserveErr != nil || reservedItem == nil {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"})
return
@ -1944,6 +2013,33 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
if resumed {
submissionRef = humanItem.PendingCombinedSubmissionID
}
if strings.Contains(strings.ToLower(request.Header.Get("Prefer")), "respond-async") {
if !server.externalAnnotationProcessor {
jobKey := humanItem.OrganizationID + ":" + submissionRef
if _, loaded := server.combinedSubmissionJobs.LoadOrStore(jobKey, struct{}{}); !loaded {
backgroundRequest := request.Clone(context.WithoutCancel(request.Context()))
go func() {
defer server.combinedSubmissionJobs.Delete(jobKey)
freshTask, te := server.store.GetAnnotationTask(backgroundRequest.Context(), task.ID)
freshItem, he := server.store.GetHumanItem(backgroundRequest.Context(), humanItem.ID)
if te != nil || he != nil || freshTask == nil || freshItem == nil || freshItem.PendingCombinedSubmissionID != submissionRef {
return
}
server.completeCombinedAnnotationSubmission(&asynchronousAnnotationResponseWriter{}, backgroundRequest, freshTask, freshItem, payload, principal, submissionRef, resumed)
}()
}
}
renderJSON(writer, http.StatusAccepted, map[string]any{"item": humanItem, "submissionId": submissionRef, "status": "processing", "idempotent": resumed})
return
}
server.completeCombinedAnnotationSubmission(writer, request, task, humanItem, payload, principal, submissionRef, resumed)
}
func (server *server) completeCombinedAnnotationSubmission(writer http.ResponseWriter, request *http.Request, task *domain.AnnotationTaskRecord, humanItem *domain.HumanItemRecord, payload combinedAnnotationSubmissionInput, principal *authenticatedPrincipal, submissionRef string, resumed bool) {
if err := server.assertAnnotationLease(request.Context()); err != nil {
renderError(writer, http.StatusConflict, err)
return
}
sliceRevisions, sliceErr := server.store.ListSliceRevisions(request.Context(), task.OrganizationID, task.ID)
annotationRevisions, annotationErr := server.store.ListAnnotationRevisions(request.Context(), task.OrganizationID, task.ID)
reviews, reviewErr := server.store.ListAnnotationReviews(request.Context(), task.OrganizationID, task.ID)
@ -1983,35 +2079,11 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
now := nowISO()
sliceRevision := domain.SliceRevisionRecord{ID: sliceID, OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: revisionNumber, Segments: payload.SliceSegments, Operations: payload.SliceOperations, AuthorUserID: principal.User.ID, ChangeReason: submissionRef + ":" + strings.TrimSpace(payload.ChangeReason), DurationSeconds: payload.DurationSeconds, CreatedAt: now}
annotationRevision := domain.AnnotationRevisionRecord{ID: annotationID, OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: revisionNumber, TemplateVersion: task.TemplateVersion, Segments: payload.Segments, ObjectHints: payload.ObjectHints, AuthorUserID: principal.User.ID, ChangeReason: submissionRef + ":" + strings.TrimSpace(payload.ChangeReason), IsRework: isRework, CountsAsOutput: !isRework, DurationSeconds: payload.DurationSeconds, CreatedAt: now}
if strings.Contains(strings.ToLower(request.Header.Get("Prefer")), "respond-async") {
jobKey := humanItem.OrganizationID + ":" + submissionRef
if _, loaded := server.combinedSubmissionJobs.LoadOrStore(jobKey, struct{}{}); !loaded {
payloadBytes, marshalErr := json.Marshal(payload)
if marshalErr != nil {
server.combinedSubmissionJobs.Delete(jobKey)
renderError(writer, http.StatusInternalServerError, marshalErr)
return
}
backgroundRequest := request.Clone(context.WithoutCancel(request.Context()))
backgroundRequest.Header = request.Header.Clone()
backgroundRequest.Header.Del("Prefer")
backgroundRequest.Body = io.NopCloser(bytes.NewReader(payloadBytes))
backgroundRequest.ContentLength = int64(len(payloadBytes))
go func() {
defer server.combinedSubmissionJobs.Delete(jobKey)
backgroundWriter := &asynchronousAnnotationResponseWriter{}
server.handleCombinedAnnotationSubmission(backgroundWriter, backgroundRequest)
if backgroundWriter.status >= http.StatusBadRequest {
fmt.Printf("[api] asynchronous combined submission %s failed status=%d body=%s\n", submissionRef, backgroundWriter.status, strings.TrimSpace(backgroundWriter.body.String()))
}
}()
}
renderJSON(writer, http.StatusAccepted, map[string]any{
"item": humanItem, "submissionId": submissionRef, "status": "processing", "idempotent": resumed,
})
materialization, materializationErr := server.materializeCombinedSliceSubmission(request.Context(), *task, *humanItem, payload, revisionNumber)
if request.Context().Err() != nil {
renderError(writer, http.StatusConflict, request.Context().Err())
return
}
materialization, materializationErr := server.materializeCombinedSliceSubmission(request.Context(), *task, *humanItem, payload, revisionNumber)
if materializationErr != nil {
previousVersion := humanItem.Version
humanItem.PendingCombinedSubmissionID = ""
@ -2042,6 +2114,10 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
return
}
previousVersion := humanItem.Version
if err := server.assertAnnotationLease(request.Context()); err != nil {
renderError(writer, http.StatusConflict, err)
return
}
humanItem.CurrentRevision = annotationRevision.Revision
humanItem.CurrentStageID = "quality_review"
humanItem.Status = domain.HumanItemReady

View File

@ -220,19 +220,29 @@ func (s *server) annotationFrameTimelineForTask(ctx context.Context, task domain
if err != nil {
return nil, err
}
return s.annotationTimelines.get(ctx, key+":"+camera, func(ctx context.Context) (map[string]any, error) {
value, err := s.buildAnnotationFrameTimeline(ctx, task, camera)
if err != nil {
return nil, err
}
after, err := s.annotationSourceFingerprint(ctx, task, *run)
if err != nil {
return nil, err
}
if after != key {
return nil, fmt.Errorf("源文件在校验期间发生变化,请重试")
}
value["sourceFingerprint"] = key
return value, nil
value, err := s.annotationTimelines.get(ctx, key, func(ctx context.Context) (map[string]any, error) {
return s.persistentTimeline(ctx, task, key, func(ctx context.Context) (map[string]any, error) {
value, err := s.buildAnnotationFrameTimeline(ctx, task, "left")
if err != nil {
return nil, err
}
after, err := s.annotationSourceFingerprint(ctx, task, *run)
if err != nil {
return nil, err
}
if after != key {
return nil, fmt.Errorf("源文件在校验期间发生变化,请重试")
}
value["sourceFingerprint"] = key
return value, nil
})
})
if err != nil {
return nil, err
}
value["camera"] = camera
if camera == "right" {
value["frameCount"] = value["rightFrameCount"]
}
return value, nil
}

View File

@ -0,0 +1,188 @@
package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/json"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"os"
"regexp"
"strconv"
"testing"
"time"
)
// Opt-in seeding only. Original production task/files remain read-only; every
// write goes to a uniquely named sandbox database and private test Run prefixes.
func TestSeedAnnotationPerformanceSandbox(t *testing.T) {
if os.Getenv("FANYU_PERF_SEED") != "1" {
t.Skip("explicit isolated performance fixture")
}
nonce := os.Getenv("FANYU_PERF_NONCE")
if !regexp.MustCompile(`^2026[0-9a-z]{8,24}$`).MatchString(nonce) {
t.Fatal("invalid nonce")
}
name := "annotation_perf_" + nonce
if name == os.Getenv("MONGO_DB") {
t.Fatal("sandbox must differ from source")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
client, e := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")))
if e != nil {
t.Fatal("mongo connect")
}
defer client.Disconnect(ctx)
source := store.NewMongoStore(client.Database(os.Getenv("MONGO_DB")))
target := store.NewMongoStore(client.Database(name))
mode := os.Getenv("FANYU_PERF_MODE")
if mode == "disable-users" {
users, e := target.ListUsers(ctx)
if e != nil {
t.Fatal(e)
}
for _, u := range users {
if !regexp.MustCompile(`^perf-` + nonce + `-`).MatchString(u.ID) {
continue
}
u.Status = domain.UserStatusDisabled
if e = target.UpsertUser(ctx, u); e != nil {
t.Fatal(e)
}
}
fmt.Println("disabled only isolated performance accounts")
return
}
collections, e := client.Database(name).ListCollectionNames(ctx, bson.M{})
if e != nil {
t.Fatal(e)
}
if len(collections) > 0 {
t.Fatal("sandbox already exists; use a fresh nonce")
}
password := os.Getenv("FANYU_PERF_PASSWORD")
if len(password) < 24 {
t.Fatal("ephemeral password required")
}
count := 5
if v, e := strconv.Atoi(os.Getenv("FANYU_PERF_USERS")); e == nil {
count = v
}
if count < 2 || count > 10 {
t.Fatal("bounded concurrency 2..10 required")
}
original, e := source.GetAnnotationTask(ctx, "annotation-task-7d738b9fc2bd9e9e778308029a7d6a50")
if e != nil || original == nil || original.OrganizationID != "org-29cd7a684faba19f630f8ad5890e18bb" || original.SourceDurationSeconds < 800 {
t.Fatal("authorized long source task unavailable")
}
run, e := source.GetRun(ctx, original.RunID)
if e != nil || run == nil {
t.Fatal("source run unavailable")
}
var input domain.NodeExecutionRecord
for _, n := range run.NodeExecutions {
if n.NodeID == original.NodeID {
input = n
}
}
org, e := source.GetOrganization(ctx, original.OrganizationID)
if e != nil || org == nil {
t.Fatal("source organization")
}
connection, e := source.GetStorageConnection(ctx, "storage-d0a49fe4b9881c324f0640ab62359603")
if e != nil || connection == nil || connection.OrganizationID != org.ID {
t.Fatal("workspace connection scope")
}
if e = target.UpsertOrganization(ctx, *org); e != nil {
t.Fatal(e)
}
if e = target.UpsertStorageConnection(ctx, *connection); e != nil {
t.Fatal("copy scoped storage configuration")
}
if e = target.EnsureIndexes(ctx); e != nil {
t.Fatal(e)
}
projectID := "project-perf-" + nonce
workflowID := "workflow-perf-" + nonce
supplierID := "supplier-perf-" + nonce
must := func(e error) {
t.Helper()
if e != nil {
t.Fatal(e)
}
}
must(target.UpsertProject(ctx, domain.ProjectRecord{ID: projectID, OrganizationID: org.ID, Name: "长视频并发隔离验收", Description: "Isolated performance fixture " + nonce}))
must(target.UpsertSupplier(ctx, domain.SupplierRecord{ID: supplierID, OrganizationID: org.ID, Name: "性能测试专用供应商", Status: "active"}))
must(target.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: workflowID, OrganizationID: org.ID, ProjectID: projectID, Name: "隔离人工流程"}))
templateID := "template-perf-" + nonce
must(target.UpsertAnnotationTemplate(ctx, domain.AnnotationTemplateRecord{ID: templateID, OrganizationID: org.ID, ProjectID: projectID, Version: 6, Status: domain.AnnotationTemplatePublished, Labels: domain.ConvenienceStore22SemanticLabels()}))
labelID := "slice-perf-" + nonce
must(target.UpsertAnnotationLabel(ctx, domain.AnnotationLabelRecord{ID: labelID, OrganizationID: org.ID, Kind: "slice", Code: labelID, Name: "压力切片", Status: domain.AnnotationLabelActive}))
must(target.UpsertProjectSliceLabels(ctx, domain.ProjectAnnotationLabelsRecord{ID: "labels-" + nonce, OrganizationID: org.ID, ProjectID: projectID, SliceLabelIDs: []string{labelID}}))
users, tasks := []map[string]string{}, []string{}
for i := 0; i < count; i++ {
suffix := fmt.Sprintf("%02d", i+1)
uid := "perf-" + nonce + "-ann-" + suffix
email := uid + "@example.invalid"
hash, e := hashPassword(password)
must(e)
must(target.UpsertUser(ctx, domain.UserRecord{ID: uid, OrganizationID: org.ID, Name: "压力标注员" + suffix, Email: email, PasswordHash: hash, Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive, SupplierID: supplierID}))
must(target.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + uid, OrganizationID: org.ID, ProjectID: projectID, UserID: uid, SupplierID: supplierID, Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}}))
runID := "run-perf-" + nonce + "-" + suffix
taskID := "task-perf-" + nonce + "-" + suffix
batchID := "batch-perf-" + nonce + "-" + suffix
itemID := "item-perf-" + nonce + "-" + suffix
bindings := []domain.NodeInputBinding{}
artifacts := []domain.ExecutionArtifact{}
leftID, rightID := "", ""
for _, port := range []string{"fullLeftVideos", "fullRightVideos", "leftTimestamps", "rightTimestamps", "imu"} {
a := artifactsForTarget(input, port)
if len(a) != 1 {
t.Fatalf("expected one source per port %s, got %d", port, len(a))
}
v := a[0]
v.ID = runID + "-" + port
v.RunID = runID
v.NodeID = "source"
bindings = append(bindings, domain.NodeInputBinding{TargetPortID: port, SourceNodeID: "source", SourcePortID: port, Artifacts: []domain.ExecutionArtifact{v}})
artifacts = append(artifacts, v)
if port == "fullLeftVideos" {
leftID = v.ID
}
if port == "fullRightVideos" {
rightID = v.ID
}
}
must(target.UpsertRun(ctx, domain.RunRecord{ID: runID, OrganizationID: org.ID, ProjectID: projectID, WorkflowID: workflowID, ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.RunStatusWaitingForHuman, Mode: domain.RunModeProduction, Artifacts: artifacts, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "source", Status: domain.NodeExecutionStatusCompleted, Artifacts: artifacts}, {NodeID: "human", Status: domain.NodeExecutionStatusWaitingForHuman, InputBindings: bindings, InputArtifacts: artifacts}}}))
must(target.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, OrganizationID: org.ID, PluginID: "annotation-qa", ProjectID: projectID, SupplierID: supplierID, WorkflowID: workflowID, RunID: runID, GateNodeID: "human", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages(), RequirePrePackageAcceptance: true, ReleasePolicy: domain.HumanReleaseAllAccepted, ItemCount: 1}))
must(target.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: itemID, OrganizationID: org.ID, PluginID: "annotation-qa", ProjectID: projectID, SupplierID: supplierID, BatchID: batchID, WorkflowID: workflowID, RunID: runID, GateNodeID: "human", LegacyTaskID: taskID, ProcessingProfile: domain.ProcessingProfileRawHuman, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Version: 1, Subject: domain.HumanSubjectRef{ItemKey: "pressure-" + suffix, Payload: map[string]any{"captureDirectory": "真实长视频压力-" + suffix, "sliceWorkflow": map[string]any{"status": "editing", "phase": "slice"}}}}))
copy := *original
copy.ID = taskID
copy.ProjectID = projectID
copy.WorkflowID = workflowID
copy.RunID = runID
copy.NodeID = "human"
copy.ArtifactID = leftID
copy.RightArtifactID = rightID
copy.SourceVideo = "/api/runs/" + runID + "/artifacts/" + leftID + "/content"
copy.RightSourceVideo = "/api/runs/" + runID + "/artifacts/" + rightID + "/content"
copy.TemplateID = templateID
copy.TemplateVersion = 6
copy.SupplierID = supplierID
copy.AssignedUserID = ""
copy.Status = domain.AnnotationTaskStatusPendingAnnotation
copy.CurrentRevision = 0
copy.ProcessingProfile = domain.ProcessingProfileRawHuman
must(target.UpsertAnnotationTask(ctx, copy))
users = append(users, map[string]string{"id": uid, "email": email})
tasks = append(tasks, taskID)
}
_, e = client.Database(name).Collection("performanceScope").InsertOne(ctx, bson.M{"id": nonce, "sourceTaskId": original.ID, "projectId": projectID})
must(e)
out, _ := json.Marshal(map[string]any{"database": name, "nonce": nonce, "projectId": projectID, "users": users, "tasks": tasks, "sourceTaskId": original.ID, "sourceDurationSeconds": original.SourceDurationSeconds})
fmt.Println("PERFORMANCE_FIXTURE " + string(out))
}

View File

@ -0,0 +1,352 @@
package api
import (
"context"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/json"
"fmt"
"log"
"net/http"
"sync"
"time"
)
type annotationJobLease struct{ ItemID, JobID, Owner string }
type annotationJobLeaseKey struct{}
func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.AnnotationTaskRecord, actor, id, kind string, input any, revision int) (*domain.AnnotationProcessingJob, error) {
run, e := s.store.GetRun(ctx, t.RunID)
if e != nil {
return nil, e
}
if run == nil {
return nil, fmt.Errorf("annotation source run unavailable")
}
fingerprint, e := s.annotationSourceFingerprint(ctx, t, *run)
if e != nil {
return nil, e
}
b, e := json.Marshal(input)
if e != nil {
return nil, e
}
var values map[string]any
if e = json.Unmarshal(b, &values); e != nil {
return nil, e
}
return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil
}
func durableAnnotationActive(item *domain.HumanItemRecord) bool {
if item == nil || item.ProcessingJob == nil {
return false
}
j := item.ProcessingJob
return (j.Status == "queued" || j.Status == "running") && ((j.Kind == "submit" && j.ID == item.PendingCombinedSubmissionID) || (j.Kind == "slice" && j.ID == stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["requestId"])))
}
func (s *server) assertAnnotationLease(ctx context.Context) error {
lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease)
if !ok {
return ctx.Err()
}
h, e := s.store.GetHumanItem(ctx, lease.ItemID)
if e != nil {
return e
}
if h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != lease.JobID || h.ProcessingJob.Owner != lease.Owner || h.ProcessingJob.Status != "running" {
return fmt.Errorf("annotation processing lease lost")
}
if h.Status == domain.HumanItemCancelled {
return fmt.Errorf("annotation task cancelled")
}
project, e := s.store.GetProject(ctx, h.ProjectID)
if e != nil {
return e
}
if project == nil || project.DeletionTaskID != "" {
return fmt.Errorf("annotation project removed or deleting")
}
expires, e := time.Parse(time.RFC3339Nano, h.ProcessingJob.LeaseUntil)
if e != nil || time.Now().After(expires) {
return fmt.Errorf("annotation processing lease expired")
}
return ctx.Err()
}
// RunAnnotationProcessor has no public mutation endpoints. Authenticated API
// requests create fenced outboxes; this independent process performs the work.
func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppConfig, parallel int, backfill bool) error {
queue, ok := db.(store.AnnotationProcessingStore)
if !ok {
return fmt.Errorf("annotation processing store unavailable")
}
if parallel < 1 || parallel > 4 {
return fmt.Errorf("annotation processor concurrency must be 1..4")
}
s := &server{store: db, workspaceMountPath: cfg.WorkspaceMountPath, globalBOSEndpoint: cfg.GlobalBOSEndpoint, externalAnnotationProcessor: true}
s.materializationSlotsOnce.Do(func() { s.materializationSlots = make(chan struct{}, parallel) })
if backfill {
orgs, e := db.ListOrganizations(ctx)
if e != nil {
return e
}
for _, org := range orgs {
tasks, e := db.ListAnnotationTasks(ctx, org.ID, "")
if e != nil {
return e
}
for _, t := range tasks {
if t.SourceIsFullVideo && t.TemplateVersion >= 5 && t.Status == domain.AnnotationTaskStatusPendingAnnotation {
if e = queue.EnqueueAnnotationIndex(ctx, t); e != nil {
return e
}
}
}
}
}
var wg sync.WaitGroup
for n := 0; n < parallel; n++ {
wg.Add(1)
go func() {
defer wg.Done()
for ctx.Err() == nil {
now := time.Now().UTC()
item, e := queue.ClaimAnnotationProcessing(ctx, domain.NewID("annotation-lease"), now.Format(time.RFC3339Nano), now.Add(2*time.Minute).Format(time.RFC3339Nano))
if e != nil {
log.Printf("[annotation-processor] claim: %v", e)
} else if item != nil {
s.executeAnnotationJob(ctx, *item)
continue
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}()
}
wg.Add(1)
go func() {
defer wg.Done()
for ctx.Err() == nil {
now := time.Now().UTC()
j, e := queue.ClaimAnnotationIndex(ctx, domain.NewID("index-lease"), now.Format(time.RFC3339Nano), now.Add(20*time.Minute).Format(time.RFC3339Nano))
if e != nil {
log.Printf("[annotation-processor] index claim: %v", e)
} else if j != nil {
indexCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
task, readErr := db.GetAnnotationTask(indexCtx, j.ID)
if readErr == nil && task != nil {
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
} else if task == nil {
readErr = fmt.Errorf("annotation task no longer exists")
}
cancel()
j.Status = "completed"
j.Error = ""
if readErr != nil {
j.Error = readErr.Error()
j.Status = "queued"
j.NextAttemptAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)
if j.Attempt >= 5 {
j.Status = "failed"
}
}
_, _ = queue.FinishAnnotationIndex(context.WithoutCancel(ctx), *j)
continue
}
select {
case <-ctx.Done():
return
case <-time.After(2 * time.Second):
}
}
}()
wg.Wait()
return ctx.Err()
}
func (s *server) executeAnnotationJob(parent context.Context, claimed domain.HumanItemRecord) {
queue := s.store.(store.AnnotationProcessingStore)
j := *claimed.ProcessingJob
ctx, cancel := context.WithTimeout(parent, 24*time.Hour)
defer cancel()
ctx = context.WithValue(ctx, annotationJobLeaseKey{}, annotationJobLease{claimed.ID, j.ID, j.Owner})
done, stopped := make(chan struct{}), make(chan struct{})
go func() {
defer close(stopped)
tick := time.NewTicker(10 * time.Second)
defer tick.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-tick.C:
stage, completed, total := "validating", 0, j.TotalSlices
if raw, ok := s.materializationProgress.Load(claimed.OrganizationID + ":" + j.TaskID); ok {
p := mapValue(raw)
stage = stringValue(p["stage"])
completed = intValue(p["completedSlices"])
total = intValue(p["totalSlices"])
}
ok, e := queue.TouchAnnotationProcessing(ctx, claimed.ID, j.ID, j.Owner, time.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), stage, completed, total)
if e != nil || !ok {
cancel()
return
}
}
}
}()
e := s.processAnnotationJob(ctx, claimed)
close(done)
<-stopped
finishCtx, stop := context.WithTimeout(context.WithoutCancel(parent), 15*time.Second)
defer stop()
if parent.Err() != nil || ctx.Err() != nil {
j.Status = "queued"
j.Stage = "queued"
j.Error = "processing interrupted; resuming"
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
return
}
j.Status = "completed"
j.Stage = "completed"
j.Input = nil
j.CompletedSlices = j.TotalSlices
j.UpdatedAt = nowISO()
if e != nil {
j.Status = "failed"
j.Stage = "failed"
j.Error = e.Error()
log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e)
}
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
if e != nil {
s.failAnnotationJob(finishCtx, claimed.ID, j)
}
}
func (s *server) processAnnotationJob(ctx context.Context, h domain.HumanItemRecord) error {
j := h.ProcessingJob
if j.Attempt > 3 {
return fmt.Errorf("后台处理连续中断,已保留编辑内容,请重试")
}
t, e := s.store.GetAnnotationTask(ctx, j.TaskID)
if e != nil {
return e
}
if t == nil || t.OrganizationID != h.OrganizationID || t.RunID != h.RunID {
return fmt.Errorf("annotation job task scope changed")
}
// Repair a crash after authoritative handoff but before legacy task publication.
if j.Kind == "submit" && h.PendingCombinedSubmissionID == "" && h.CurrentStageID == "quality_review" && h.CurrentRevision > 0 {
t.CurrentRevision = h.CurrentRevision
t.Status = domain.AnnotationTaskStatusPendingQuality
t.AssignedUserID = ""
return s.store.UpsertAnnotationTask(ctx, *t)
}
actor, e := s.store.GetUser(ctx, j.ActorID)
if e != nil {
return e
}
if actor == nil || actor.Status != domain.UserStatusActive || actor.OrganizationID != h.OrganizationID || h.AssignedUserID != actor.ID || h.CurrentStageID != "annotation_combined" {
return fmt.Errorf("annotation processing ownership or user status changed")
}
org, e := s.store.GetOrganization(ctx, h.OrganizationID)
if e != nil {
return e
}
if org == nil || org.Status != domain.OrganizationStatusActive {
return fmt.Errorf("annotation organization unavailable")
}
principal := &authenticatedPrincipal{User: *actor, Organization: *org}
r, _ := http.NewRequestWithContext(context.WithValue(ctx, authContextKey{}, principal), http.MethodPost, "http://annotation-processor/internal", nil)
w := &asynchronousAnnotationResponseWriter{}
if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *t); !ok {
return fmt.Errorf("annotation job no longer authorized")
}
if !annotationManager(principal) && !s.canAccessCurrentHumanStage(ctx, r, h) {
return fmt.Errorf("annotation job role changed")
}
run, e := s.store.GetRun(ctx, t.RunID)
if e != nil {
return e
}
if run == nil {
return fmt.Errorf("annotation run unavailable")
}
key, e := s.annotationSourceFingerprint(ctx, *t, *run)
if e != nil {
return e
}
if key != j.SourceFingerprint {
return fmt.Errorf("源文件已改变,已保留编辑内容,请重新确认")
}
b, e := json.Marshal(normalizeStructuredValue(j.Input))
if e != nil {
return e
}
switch j.Kind {
case "slice":
var p sliceConfirmationInput
if e = json.Unmarshal(b, &p); e != nil {
return e
}
s.finishSliceConfirmation(r.Context(), t.ID, h.ID, j.ID, p, j.Revision, j.ActorID)
fresh, e := s.store.GetHumanItem(ctx, h.ID)
if e != nil {
return e
}
if fresh == nil {
return fmt.Errorf("annotation task removed")
}
flow := mapValue(fresh.Subject.Payload["sliceWorkflow"])
if stringValue(flow["status"]) != "completed" {
return fmt.Errorf("%s", firstNonEmptyText(stringValue(flow["error"]), "slice processing did not complete"))
}
case "submit":
if h.PendingCombinedSubmissionID != j.ID {
return fmt.Errorf("annotation submission superseded")
}
var p combinedAnnotationSubmissionInput
if e = json.Unmarshal(b, &p); e != nil {
return e
}
s.completeCombinedAnnotationSubmission(w, r, t, &h, p, principal, j.ID, j.Attempt > 1)
if w.status >= 400 {
return fmt.Errorf("annotation submission failed: %s", w.body.String())
}
default:
return fmt.Errorf("unknown annotation job kind")
}
return nil
}
func (s *server) failAnnotationJob(ctx context.Context, id string, j domain.AnnotationProcessingJob) {
h, e := s.store.GetHumanItem(ctx, id)
if e != nil || h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner {
return
}
version := h.Version
if j.Kind == "submit" && h.PendingCombinedSubmissionID == j.ID {
h.PendingCombinedSubmissionID = ""
h.PendingCombinedPayloadSHA = ""
}
if j.Kind == "slice" {
w := mapValue(h.Subject.Payload["sliceWorkflow"])
if stringValue(w["requestId"]) == j.ID {
w["status"] = "failed"
w["error"] = j.Error
h.Subject.Payload["sliceWorkflow"] = w
}
}
h.Version++
h.UpdatedAt = nowISO()
_, _ = s.store.UpdateHumanItemVersioned(ctx, *h, version)
}

View File

@ -0,0 +1,141 @@
package api
import (
"context"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
func annotationProcessorFixture(t *testing.T) (combinedDraftFixture, string) {
t.Helper()
if _, e := exec.LookPath("ffmpeg"); e != nil {
t.Skip("ffmpeg required")
}
f := newCombinedDraftFixture(t)
if e := f.store.UpsertOrganization(f.ctx, domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Name: "Processor test", Status: domain.OrganizationStatusActive}); e != nil {
t.Fatal(e)
}
root := t.TempDir()
seedSliceDictionary(t, f, "测试切片")
task := f.task
task.NodeID = "human"
task.ArtifactID = "left-video"
task.RightArtifactID = "right-video"
task.ClipEnd = 4
task.SourceIsFullVideo = true
task.SourceDurationSeconds = 4
task.CoverageStartSecond = 0
task.CoverageEndSecond = 4
task.CoverageMode = "full_common_range"
pts, imu := "timestamp_us\n", "timestamp_us,x,y,z,type\n"
for i := 0; i < 32; i++ {
stamp := 1000000 + i*125000
pts += fmt.Sprintf("%d\n", stamp)
imu += fmt.Sprintf("%d,1,2,3,accel\n%d,4,5,6,gyro\n", stamp, stamp)
}
bindings := []domain.NodeInputBinding{}
for _, side := range []string{"left", "right"} {
video := filepath.Join(root, side+".mp4")
if out, e := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=64x48:r=8:d=4", "-c:v", "libx264", "-pix_fmt", "yuv420p", video).CombinedOutput(); e != nil {
t.Fatalf("%v: %s", e, out)
}
p := filepath.Join(root, side+"_pts.csv")
os.WriteFile(p, []byte(pts), 0600)
videoPort := "fullLeftVideos"
if side == "right" {
videoPort = "fullRightVideos"
}
bindings = append(bindings, domain.NodeInputBinding{TargetPortID: videoPort, Artifacts: []domain.ExecutionArtifact{{ID: side + "-video", Name: side + ".mp4", Path: video, Data: map[string]any{"collectionId": "capture"}}}}, domain.NodeInputBinding{TargetPortID: side + "Timestamps", Artifacts: []domain.ExecutionArtifact{{ID: side + "-pts", Name: side + "_pts.csv", Path: p, Data: map[string]any{"collectionId": "capture"}}}})
}
p := filepath.Join(root, "imu.csv")
os.WriteFile(p, []byte(imu), 0600)
bindings = append(bindings, domain.NodeInputBinding{TargetPortID: "imu", Artifacts: []domain.ExecutionArtifact{{ID: "imu", Name: "imu.csv", Path: p, Data: map[string]any{"collectionId": "capture"}}}})
f.store.UpsertRun(f.ctx, domain.RunRecord{ID: task.RunID, OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, WorkflowID: task.WorkflowID, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: task.NodeID, InputBindings: bindings}}})
f.store.UpsertAnnotationTask(f.ctx, task)
f.task = task
f.handler = NewHandlerWithConfig(f.store, nil, config.AppConfig{WorkspaceMountPath: root})
return f, root
}
func TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, root := annotationProcessorFixture(t)
slices := []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 0, End: 2}, {ID: "b", Name: "测试切片", Start: 2, End: 4}}
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": f.item.Version, "sliceSegments": slices})
if r.Code != 202 {
t.Fatal(r.Code, r.Body.String())
}
h, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
if h.ProcessingJob == nil || h.ProcessingJob.Status != "queued" || len(h.ProcessingJob.Input) == 0 {
t.Fatal("missing atomic outbox")
}
if b, _ := json.Marshal(h); stringContains(string(b), "processingJob") || stringContains(string(b), "sourceFingerprint") {
t.Fatal("private outbox leaked")
}
// A new API instance must not clear a durable job when it has no local map.
f.handler = NewHandlerWithConfig(f.store, nil, config.AppConfig{WorkspaceMountPath: root})
r = f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"?view=status", f.annotator, nil)
if r.Code != 200 {
t.Fatal(r.Code, r.Body.String())
}
now := time.Now().UTC()
old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "dead-worker", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano))
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "replacement-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 {
t.Fatal("lease not recovered")
}
if ok, _ := f.store.UpdateHumanItemVersioned(f.ctx, *old, old.Version); ok {
t.Fatal("expired worker can publish")
}
p := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
p.executeAnnotationJob(context.Background(), *claimed)
h, _ = f.store.GetHumanItem(f.ctx, f.item.ID)
if h.ProcessingJob.Status != "completed" || len(h.ProcessingJob.Input) != 0 || stringValue(mapValue(h.Subject.Payload["sliceWorkflow"])["status"]) != "completed" {
t.Fatalf("slice job failed: %+v", h.ProcessingJob)
}
baseHandler := f.handler
f.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.Header.Set("Prefer", "respond-async")
baseHandler.ServeHTTP(w, r)
})
r = f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/combined-submissions", f.annotator, map[string]any{"version": h.Version, "sliceSegments": slices, "segments": []any{}, "requireSliceCompletion": true})
if r.Code != 202 {
t.Fatal(r.Code, r.Body.String())
}
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
if before.PendingCombinedSubmissionID == "" || before.ProcessingJob.Kind != "submit" {
t.Fatal("submission not atomically queued")
}
job, _ := f.store.ClaimAnnotationProcessing(f.ctx, "submit-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if job == nil {
t.Fatal("submission not claimable")
}
p.executeAnnotationJob(context.Background(), *job)
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
if after.CurrentStageID != "quality_review" || after.ProcessingJob.Status != "completed" || after.PendingCombinedSubmissionID != "" {
t.Fatalf("submission failed %+v", after.ProcessingJob)
}
run, _ := f.store.GetRun(f.ctx, f.task.RunID)
found := false
for _, a := range run.Artifacts {
if boolValue(a.Data["metadataOnly"]) {
found = true
}
}
if !found {
t.Fatal("unchanged media was rebuilt")
}
}
func stringContains(s, part string) bool {
return len(part) > 0 && len(s) >= len(part) && strings.Contains(s, part)
}

View File

@ -0,0 +1,197 @@
package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"fmt"
"golang.org/x/sync/errgroup"
"os"
"path/filepath"
"strings"
)
func annotationObjectSHA(meta objectsync.ObjectMetadata) string {
for k, v := range meta.UserMeta {
if strings.EqualFold(k, "sha256") {
return v
}
}
return ""
}
func (s *server) annotationMediaCopyReady(ctx context.Context, org, path string, file map[string]any) error {
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, path)
if e != nil {
return e
}
if c == nil {
return nil
}
if stringValue(file["etag"]) == "" {
return fmt.Errorf("legacy media needs initial verification")
}
meta, e := (objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}).Head(ctx, *c, bucket, key)
if e != nil {
return e
}
if !meta.Exists || meta.ETag != stringValue(file["etag"]) || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != stringValue(file["sha256"]) {
return fmt.Errorf("cached media version changed")
}
return nil
}
func (s *server) annotationWorkspaceObject(ctx context.Context, org, path string) (*domain.StorageConnectionRecord, string, string, error) {
id, bucket := os.Getenv("ANNOTATION_WORKSPACE_CONNECTION_ID"), os.Getenv("ANNOTATION_WORKSPACE_BUCKET")
if id == "" && bucket == "" {
id, bucket = os.Getenv("PROJECT_DELETION_WORKSPACE_CONNECTION_ID"), os.Getenv("PROJECT_DELETION_WORKSPACE_BUCKET")
}
if id == "" && bucket == "" {
return nil, "", "", nil
}
if id == "" || bucket == "" {
return nil, "", "", fmt.Errorf("annotation workspace mapping incomplete")
}
c, e := s.store.GetStorageConnection(ctx, id)
if e != nil {
return nil, "", "", e
}
if c == nil || c.OrganizationID != org || c.Provider != domain.StorageProviderBOS {
return nil, "", "", fmt.Errorf("annotation workspace scope mismatch")
}
rel, e := filepath.Rel(s.workspaceMountPath, path)
if e != nil || rel == "." || strings.HasPrefix(rel, "..") {
return nil, "", "", fmt.Errorf("annotation output outside workspace")
}
return c, bucket, filepath.ToSlash(rel), nil
}
func (s *server) publishAnnotationFile(ctx context.Context, org, source, target, sha string) (objectsync.ObjectMetadata, error) {
if e := s.assertAnnotationLease(ctx); e != nil {
return objectsync.ObjectMetadata{}, e
}
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, target)
if e != nil {
return objectsync.ObjectMetadata{}, e
}
if c == nil {
e := copyAnnotationFile(ctx, source, target, sha)
return objectsync.ObjectMetadata{}, e
}
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
info, e := os.Stat(source)
if e != nil {
return objectsync.ObjectMetadata{}, e
}
meta, e := p.Head(ctx, *c, bucket, key)
if e != nil {
return meta, e
}
if meta.Exists {
if meta.SizeBytes == info.Size() && annotationObjectSHA(meta) == sha {
return meta, nil
}
return meta, fmt.Errorf("annotation output conflict: %s", filepath.Base(target))
}
f, e := os.Open(source)
if e != nil {
return meta, e
}
defer f.Close()
meta, e = p.PutInternal(ctx, *c, bucket, key, contextFileReader{ctx, f}, info.Size(), "application/octet-stream", map[string]string{"sha256": sha})
if e != nil {
actual, he := p.Head(ctx, *c, bucket, key)
if he == nil && actual.Exists && actual.SizeBytes == info.Size() && annotationObjectSHA(actual) == sha {
return actual, nil
}
}
return meta, e
}
// The bytes stay in BOS. Source ETag and digest metadata fence the copy, while
// the fresh generation keeps independent objects for existing retention rules.
func (s *server) copyAnnotationObject(ctx context.Context, org, source, target string, file map[string]any) (objectsync.ObjectMetadata, error) {
sha := stringValue(file["sha256"])
if sha == "" {
return objectsync.ObjectMetadata{}, fmt.Errorf("missing verified media digest")
}
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, source)
if e != nil {
return objectsync.ObjectMetadata{}, e
}
if c == nil {
e := copyAnnotationFile(ctx, source, target, sha)
return objectsync.ObjectMetadata{}, e
}
_, toBucket, toKey, e := s.annotationWorkspaceObject(ctx, org, target)
if e != nil {
return objectsync.ObjectMetadata{}, e
}
if bucket != toBucket {
return objectsync.ObjectMetadata{}, fmt.Errorf("cross-workspace media copy")
}
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
meta, e := p.Head(ctx, *c, bucket, key)
if e != nil {
return meta, e
}
// Old generations without authoritative object metadata take the verified
// normal materialization path once, rather than trusting an unpinned copy.
if !meta.Exists || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != sha || stringValue(file["etag"]) == "" || meta.ETag != stringValue(file["etag"]) {
return meta, fmt.Errorf("cached media version unavailable")
}
old, e := p.Head(ctx, *c, bucket, toKey)
if e != nil {
return old, e
}
if old.Exists {
if old.SizeBytes == meta.SizeBytes && annotationObjectSHA(old) == sha {
return old, nil
}
return old, fmt.Errorf("annotation media destination conflict")
}
if e = s.assertAnnotationLease(ctx); e != nil {
return old, e
}
return p.PublishConditional(ctx, *c, bucket, key, toKey, meta.SizeBytes, meta.ETag, "", "*", map[string]string{"sha256": sha})
}
func (s *server) publishAnnotationDirectory(ctx context.Context, org, source, target string, manifest map[string]any) (string, int64, error) {
// Publish all media/sidecars before the generation manifest. Limit network
// concurrency independently of encoder concurrency.
group, gctx := errgroup.WithContext(ctx)
group.SetLimit(4)
items := listValue(manifest["items"])
manifest["items"] = items
for i, raw := range items {
item := mapValue(raw)
items[i] = item
dir := stringValue(item["directory"])
files := mapValue(item["files"])
item["files"] = files
for kind, rawFile := range files {
file := mapValue(rawFile)
files[kind] = file
name := stringValue(file["name"])
if name == "" || filepath.Base(name) != name || filepath.IsAbs(dir) || strings.Contains(dir, "..") {
return "", 0, fmt.Errorf("invalid annotation member path")
}
group.Go(func() error {
meta, e := s.publishAnnotationFile(gctx, org, filepath.Join(source, dir, name), filepath.Join(target, dir, name), stringValue(file["sha256"]))
if e == nil && meta.ETag != "" {
file["etag"] = meta.ETag
}
return e
})
}
}
if e := group.Wait(); e != nil {
return "", 0, e
}
path := filepath.Join(source, "revision-manifest.json")
sha, size, e := writeJSONFile(path, manifest)
if e != nil {
return "", 0, e
}
_, e = s.publishAnnotationFile(ctx, org, path, filepath.Join(target, "revision-manifest.json"), sha)
return sha, size, e
}

View File

@ -0,0 +1,104 @@
package api
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"math"
)
type persistedAnnotationTimeline struct {
Camera string `json:"camera"`
FrameCount int `json:"frameCount"`
LeftFrameCount int `json:"leftFrameCount"`
RightFrameCount int `json:"rightFrameCount"`
Origin int64 `json:"originTimestampUs"`
End int64 `json:"coverageEndTimestampUs"`
Seconds []float64 `json:"boundariesSeconds"`
RightSeconds []float64 `json:"rightBoundariesSeconds"`
Stamps []int64 `json:"boundaryTimestampUs"`
Left []int `json:"leftFrameIndices"`
Right []int `json:"rightFrameIndices"`
StartSecond float64 `json:"rangeStartSecond"`
EndSecond float64 `json:"rangeEndSecond"`
Contract string `json:"boundaryContract"`
Scale float64 `json:"ptsToMediaScale"`
Fingerprint string `json:"sourceFingerprint"`
}
func decodePersistentTimeline(record *domain.AnnotationTimelineIndex, fingerprint string) (map[string]any, error) {
if record == nil {
return nil, nil
}
sum := sha256.Sum256(record.Body)
if hex.EncodeToString(sum[:]) != record.SHA256 {
return nil, fmt.Errorf("timeline index checksum mismatch")
}
z, e := gzip.NewReader(bytes.NewReader(record.Body))
if e != nil {
return nil, e
}
defer z.Close()
b, e := io.ReadAll(io.LimitReader(z, 8<<20+1))
if e != nil || len(b) > 8<<20 {
return nil, fmt.Errorf("invalid timeline index size")
}
var t persistedAnnotationTimeline
if e = json.Unmarshal(b, &t); e != nil {
return nil, e
}
n := len(t.Seconds)
if t.Contract != "hardware-frame-v1" || t.Fingerprint != fingerprint || n < 2 || len(t.Stamps) != n || len(t.RightSeconds) != n || len(t.Left) != n || len(t.Right) != n || t.LeftFrameCount < 1 || t.RightFrameCount < 1 {
return nil, fmt.Errorf("invalid timeline index contract")
}
for i := 0; i < n; i++ {
if math.IsNaN(t.Seconds[i]) || math.IsInf(t.Seconds[i], 0) || i > 0 && (t.Stamps[i] <= t.Stamps[i-1] || t.Seconds[i] <= t.Seconds[i-1]) {
return nil, fmt.Errorf("invalid timeline index ordering")
}
}
return map[string]any{"camera": "left", "frameCount": t.LeftFrameCount, "leftFrameCount": t.LeftFrameCount, "rightFrameCount": t.RightFrameCount, "originTimestampUs": t.Origin, "coverageEndTimestampUs": t.End, "boundariesSeconds": t.Seconds, "rightBoundariesSeconds": t.RightSeconds, "boundaryTimestampUs": t.Stamps, "leftFrameIndices": t.Left, "rightFrameIndices": t.Right, "rangeStartSecond": t.StartSecond, "rangeEndSecond": t.EndSecond, "boundaryContract": t.Contract, "ptsToMediaScale": t.Scale, "sourceFingerprint": t.Fingerprint}, nil
}
func (s *server) persistentTimeline(ctx context.Context, t domain.AnnotationTaskRecord, key string, build func(context.Context) (map[string]any, error)) (map[string]any, error) {
db, ok := s.store.(store.AnnotationProcessingStore)
if !ok {
return build(ctx)
}
id := "annotation-timeline-v1:" + t.OrganizationID + ":" + key
record, e := db.GetAnnotationTimelineIndex(ctx, id)
if e != nil {
return nil, e
}
if cached, e := decodePersistentTimeline(record, key); e == nil && cached != nil {
return cached, nil
}
v, e := build(ctx)
if e != nil {
return nil, e
}
b, e := json.Marshal(v)
if e != nil {
return nil, e
}
if len(b) > 8<<20 {
return v, nil
}
var compressed bytes.Buffer
z := gzip.NewWriter(&compressed)
if _, e = z.Write(b); e != nil {
return nil, e
}
if e = z.Close(); e != nil {
return nil, e
}
sum := sha256.Sum256(compressed.Bytes())
e = db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: id, OrganizationID: t.OrganizationID, ProjectID: t.ProjectID, WorkflowID: t.WorkflowID, RunID: t.RunID, Body: compressed.Bytes(), SHA256: hex.EncodeToString(sum[:]), CreatedAt: nowISO()})
return v, e
}

View File

@ -0,0 +1,82 @@
package api
import (
"context"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"testing"
"time"
)
func TestPersistentTimelineSurvivesProcessCacheAndRebuildsCorruption(t *testing.T) {
db := store.NewMemoryStore()
task := domain.AnnotationTaskRecord{ID: "task", OrganizationID: "org", ProjectID: "project", RunID: "run"}
count := 0
v := map[string]any{"camera": "left", "frameCount": 2, "leftFrameCount": 2, "rightFrameCount": 2, "originTimestampUs": int64(100), "coverageEndTimestampUs": int64(300), "boundariesSeconds": []float64{0, 0.1, 0.2}, "rightBoundariesSeconds": []float64{0, 0.1, 0.2}, "boundaryTimestampUs": []int64{100, 200, 300}, "leftFrameIndices": []int{0, 1, 2}, "rightFrameIndices": []int{0, 1, 2}, "rangeStartSecond": 0.0, "rangeEndSecond": 0.2, "boundaryContract": "hardware-frame-v1", "ptsToMediaScale": 1.0, "sourceFingerprint": "source"}
build := func(context.Context) (map[string]any, error) { count++; return cloneTimeline(v), nil }
s := &server{store: db}
if _, e := s.persistentTimeline(context.Background(), task, "source", build); e != nil {
t.Fatal(e)
}
s = &server{store: db}
got, e := s.persistentTimeline(context.Background(), task, "source", build)
if e != nil || count != 1 {
t.Fatal("persistent cache missed", count, e)
}
if got["boundaryTimestampUs"].([]int64)[1] != 200 {
t.Fatal("index changed integer timestamps")
}
id := "annotation-timeline-v1:org:source"
bad, _ := db.GetAnnotationTimelineIndex(context.Background(), id)
bad.Body[0] ^= 0xff
db.PutAnnotationTimelineIndex(context.Background(), *bad)
if _, e = s.persistentTimeline(context.Background(), task, "source", build); e != nil || count != 2 {
t.Fatal("corrupt cache not rebuilt", e)
}
other := task
other.OrganizationID = "other"
if _, e = s.persistentTimeline(context.Background(), other, "source", build); e != nil || count != 3 {
t.Fatal("organization cache isolation", e)
}
}
func TestTaskCreationPrecomputesPersistentTimeline(t *testing.T) {
f, root := annotationProcessorFixture(t)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
done <- RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: root}, 1, false)
}()
s := &server{store: f.store, workspaceMountPath: root}
run, _ := f.store.GetRun(ctx, f.task.RunID)
key, e := s.annotationSourceFingerprint(ctx, f.task, *run)
if e != nil {
cancel()
t.Fatal(e)
}
deadline := time.Now().Add(5 * time.Second)
found := false
for time.Now().Before(deadline) {
index, e := f.store.GetAnnotationTimelineIndex(ctx, "annotation-timeline-v1:"+f.task.OrganizationID+":"+key)
if e == nil && index != nil {
found = true
break
}
time.Sleep(20 * time.Millisecond)
}
cancel()
<-done
if !found {
t.Fatal("new task not proactively indexed")
}
restarted := &server{store: f.store, workspaceMountPath: root}
left, e := restarted.annotationFrameTimelineForTask(context.Background(), f.task, "left")
if e != nil {
t.Fatal(e)
}
right, e := restarted.annotationFrameTimelineForTask(context.Background(), f.task, "right")
if e != nil || left["frameCount"] != right["frameCount"] {
t.Fatal("stereo index", e)
}
}

View File

@ -503,6 +503,9 @@ func absInt64(value int64) int64 {
}
func (server *server) materializeCombinedSliceSubmission(ctx context.Context, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, payload combinedAnnotationSubmissionInput, revision int) (*humanSliceMaterialization, error) {
if err := server.assertAnnotationLease(ctx); err != nil {
return nil, err
}
started := time.Now()
ctx, cancel := context.WithTimeout(ctx, 24*time.Hour)
defer cancel()
@ -561,6 +564,11 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
return nil, err
}
reusable := reusableAnnotationMedia(*run, task, item, server.workspaceMountPath, sourceFingerprint, revision)
if metadata, e := server.tryAnnotationMetadataRevision(ctx, task, item, run, payload, revision, sourceFingerprint); e != nil {
return nil, e
} else if metadata != nil {
return metadata, nil
}
stage, err := os.MkdirTemp("", "annotation-materialize-")
if err != nil {
return nil, err
@ -775,6 +783,45 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
manifestItems := make([]map[string]any, 0, len(slices))
keptSliceCount, discardedSliceCount := 0, 0
reusedSlices := 0
reusedByIndex := map[int]bool{}
leftCuts, rightCuts := []annotationFrameSlice{}, []annotationFrameSlice{}
for i, slice := range normalizedSlices {
a, b := mediaSecondToTimestamp(slice.Start), mediaSecondToTimestamp(slice.End)
ls := sort.Search(len(leftPTS), func(j int) bool { return leftPTS[j].TimestampUS >= a })
le := sort.Search(len(leftPTS), func(j int) bool { return leftPTS[j].TimestampUS >= b })
rs := sort.Search(len(rightPTS), func(j int) bool { return rightPTS[j].TimestampUS >= a })
re := sort.Search(len(rightPTS), func(j int) bool { return rightPTS[j].TimestampUS >= b })
dir := filepath.Join(root, "slices", fmt.Sprintf("slice-%06d", i+1))
if e := os.MkdirAll(dir, 0755); e != nil {
return nil, e
}
leftOut, rightOut := filepath.Join(dir, "left.mp4"), filepath.Join(dir, "right.mp4")
key := fmt.Sprintf("%s:h264-ultrafast-crf23-v1:%d:%d:%d:%d", sourceFingerprint, ls, le, rs, re)
cached, ok := reusable[key]
reusedByIndex[i] = ok && copyReusableVideo(ctx, cached, "left", leftOut) && copyReusableVideo(ctx, cached, "right", rightOut)
if !reusedByIndex[i] {
leftCuts = append(leftCuts, annotationFrameSlice{ls, le, leftOut})
rightCuts = append(rightCuts, annotationFrameSlice{rs, re, rightOut})
}
}
if len(leftCuts) > 0 {
if err := stageVideos(); err != nil {
return nil, err
}
progress("slicing", 0)
var leftErr, rightErr error
var wg sync.WaitGroup
wg.Add(2)
go func() { defer wg.Done(); leftErr = writeFrameAccurateSlices(ctx, leftPath, leftCuts) }()
go func() { defer wg.Done(); rightErr = writeFrameAccurateSlices(ctx, rightPath, rightCuts) }()
wg.Wait()
if leftErr != nil {
return nil, leftErr
}
if rightErr != nil {
return nil, rightErr
}
}
keptDuration, discardedDuration := 0.0, 0.0
for index, slice := range normalizedSlices {
progress("slicing", index)
@ -804,32 +851,9 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
}
leftOutput, rightOutput := filepath.Join(directory, "left.mp4"), filepath.Join(directory, "right.mp4")
mediaKey := fmt.Sprintf("%s:h264-ultrafast-crf23-v1:%d:%d:%d:%d", sourceFingerprint, leftStart, leftEnd, rightStart, rightEnd)
cached, found := reusable[mediaKey]
reused := found && copyReusableVideo(ctx, cached, "left", leftOutput) && copyReusableVideo(ctx, cached, "right", rightOutput)
reused := reusedByIndex[index]
if reused {
reusedSlices++
} else {
if err := stageVideos(); err != nil {
return nil, err
}
var leftSliceErr, rightSliceErr error
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
leftSliceErr = writeFrameAccurateSlice(ctx, leftPath, leftOutput, leftStart, leftEnd)
}()
go func() {
defer wg.Done()
rightSliceErr = writeFrameAccurateSlice(ctx, rightPath, rightOutput, rightStart, rightEnd)
}()
wg.Wait()
if leftSliceErr != nil {
return nil, leftSliceErr
}
if rightSliceErr != nil {
return nil, rightSliceErr
}
}
leftCount, rightCount := leftEnd-leftStart, rightEnd-rightStart
if !reused {
@ -951,24 +975,21 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
return nil, fmt.Errorf("源文件在切片期间发生变化,请重新确认切片")
}
progress("publishing", len(normalizedSlices))
if err := copyAnnotationTree(ctx, root, publishedRoot); err != nil {
if err := server.assertAnnotationLease(ctx); err != nil {
return nil, err
}
manifestSHA, manifestBytes, err = server.publishAnnotationDirectory(ctx, task.OrganizationID, root, publishedRoot, revisionManifest)
if err != nil {
return nil, err
}
root, manifestPath = publishedRoot, filepath.Join(publishedRoot, "revision-manifest.json")
log.Printf("[api] materialization_media task=%s revision=%d reused_slices=%d slices=%d", task.ID, revision, reusedSlices, len(normalizedSlices))
artifact := domain.ExecutionArtifact{ID: artifactID, RunID: task.RunID, NodeID: task.NodeID, PortID: "materializedSliceDirectory", PortName: "materializedSliceDirectory", Name: fmt.Sprintf("revision-%06d.directory.index", revision), Type: domain.PortTypeFile, Size: fmt.Sprintf("%d files", len(manifestItems)*7+1), SizeBytes: manifestBytes, Path: root, Metadata: &domain.ArtifactMetadata{SHA256: manifestSHA, MimeType: "application/json"}, Data: map[string]any{"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": root, "manifestPath": manifestPath, "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "normalizedSlices": normalizedSlices, "normalizedSegments": normalizedSegments}}
run.Artifacts = append(run.Artifacts, artifact)
artifact.Data["boundaryContract"] = "hardware-frame-v1"
artifact.Data["sourceFingerprint"] = sourceFingerprint
artifact.Data["reusedSliceCount"] = reusedSlices
artifact.Data["deliverySelection"] = deliverySelection
for index := range run.NodeExecutions {
if run.NodeExecutions[index].NodeID == task.NodeID {
run.NodeExecutions[index].Artifacts = append(run.NodeExecutions[index].Artifacts, artifact)
}
}
run.UpdatedAt = nowISO()
if err := server.store.UpsertRun(ctx, *run); err != nil {
if err := server.storeAnnotationMaterializationArtifact(ctx, task, run, artifact); err != nil {
return nil, err
}
return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "deliverySelection": deliverySelection, "runId": task.RunID, "artifactId": artifact.ID, "manifestSha256": manifestSHA}, Slices: normalizedSlices, Segments: normalizedSegments, CoverageStartSecond: commonMediaStart, CoverageEndSecond: commonMediaEnd}, nil

View File

@ -76,6 +76,7 @@ type server struct {
materializationSlotsOnce sync.Once
materializationSlots chan struct{}
materializationProgress sync.Map
externalAnnotationProcessor bool
}
type runJobTerminator interface {
@ -1259,6 +1260,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
operatorRunnerImage: appConfig.OperatorRunnerImage,
workerAPIBaseURL: appConfig.WorkerAPIBaseURL,
globalBOSEndpoint: appConfig.GlobalBOSEndpoint,
externalAnnotationProcessor: os.Getenv("ANNOTATION_PROCESSOR_MODE") == "external",
apiToken: appConfig.APIToken,
requireSessionAuth: appConfig.RequireSessionAuth,
sessionCookieSecure: appConfig.SessionCookieSecure,

View File

@ -12,6 +12,9 @@ const sliceLeaseDuration = domain.SliceProcessingLeaseDuration
// Persisted leases survive API restarts. A dead process must not leave the
// workbench spinning forever; retry remains explicit to avoid an OOM loop.
func (server *server) recoverInterruptedSlice(ctx context.Context, item *domain.HumanItemRecord, now time.Time) error {
if durableAnnotationActive(item) {
return nil
}
if !domain.ExpireSliceProcessing(item, now) {
return nil
}

View File

@ -0,0 +1,69 @@
package domain
import (
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/bsonrw"
)
// ProcessingJob is a private durable outbox embedded in the authoritative
// HumanItem CAS. Creating a submission and queuing it are one database write.
type AnnotationProcessingJob struct {
ID string `bson:"id" json:"id"`
Kind string `bson:"kind" json:"kind"`
TaskID string `bson:"taskId" json:"taskId"`
ActorID string `bson:"actorId" json:"actorId"`
Input map[string]any `bson:"input,omitempty" json:"-"`
SourceFingerprint string `bson:"sourceFingerprint,omitempty" json:"-"`
Revision int `bson:"revision,omitempty" json:"revision,omitempty"`
Status string `bson:"status" json:"status"`
Owner string `bson:"owner,omitempty" json:"-"`
LeaseUntil string `bson:"leaseUntil,omitempty" json:"-"`
Attempt int `bson:"attempt" json:"attempt"`
Stage string `bson:"stage,omitempty" json:"stage,omitempty"`
CompletedSlices int `bson:"completedSlices" json:"completedSlices"`
TotalSlices int `bson:"totalSlices" json:"totalSlices"`
Error string `bson:"error,omitempty" json:"error,omitempty"`
CreatedAt string `bson:"createdAt" json:"createdAt"`
UpdatedAt string `bson:"updatedAt" json:"updatedAt"`
}
func (j *AnnotationProcessingJob) UnmarshalBSON(data []byte) error {
type plain AnnotationProcessingJob
decoder, e := bson.NewDecoder(bsonrw.NewBSONDocumentReader(data))
if e != nil {
return e
}
decoder.DefaultDocumentM()
var out plain
if e = decoder.Decode(&out); e != nil {
return e
}
*j = AnnotationProcessingJob(out)
return nil
}
type AnnotationTimelineIndex struct {
ID string `bson:"id"`
OrganizationID string `bson:"organizationId"`
ProjectID string `bson:"projectId"`
WorkflowID string `bson:"workflowId"`
RunID string `bson:"runId"`
Body []byte `bson:"body"`
SHA256 string `bson:"sha256"`
CreatedAt string `bson:"createdAt"`
}
type AnnotationIndexJob struct {
ID string `bson:"id"`
OrganizationID string `bson:"organizationId"`
ProjectID string `bson:"projectId"`
WorkflowID string `bson:"workflowId"`
RunID string `bson:"runId"`
Status string `bson:"status"`
Owner string `bson:"owner,omitempty"`
LeaseUntil string `bson:"leaseUntil,omitempty"`
Attempt int `bson:"attempt"`
NextAttemptAt string `bson:"nextAttemptAt"`
Error string `bson:"error,omitempty"`
CreatedAt string `bson:"createdAt"`
}

View File

@ -133,6 +133,7 @@ type HumanBatchRecord struct {
}
type HumanItemRecord struct {
ProcessingJob *AnnotationProcessingJob `json:"-" bson:"processingJob,omitempty"`
ProcessingScope string `json:"processingScope,omitempty" bson:"processingScope,omitempty"`
ProcessingProfile string `json:"processingProfile,omitempty" bson:"processingProfile,omitempty"`
ID string `json:"id" bson:"id"`

View File

@ -0,0 +1,113 @@
package store
import (
"context"
"droboticflow/internal/domain"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
)
func preserveAnnotationArtifacts(incoming *domain.RunRecord, previous domain.RunRecord) {
for i := range incoming.NodeExecutions {
n := &incoming.NodeExecutions[i]
seen := map[string]bool{}
for _, a := range n.Artifacts {
seen[a.ID] = true
}
for _, old := range previous.NodeExecutions {
if old.NodeID != n.NodeID {
continue
}
for _, a := range old.Artifacts {
if a.PortID == "materializedSliceDirectory" && !seen[a.ID] {
n.Artifacts = append(n.Artifacts, a)
seen[a.ID] = true
}
}
}
}
seen := map[string]bool{}
for _, a := range incoming.Artifacts {
seen[a.ID] = true
}
for _, n := range incoming.NodeExecutions {
for _, a := range n.Artifacts {
if a.PortID == "materializedSliceDirectory" && !seen[a.ID] {
incoming.Artifacts = append(incoming.Artifacts, a)
seen[a.ID] = true
}
}
}
}
func annotationSafeRunUpdate(stored domain.RunRecord, unset []string) (mongo.Pipeline, error) {
raw, e := bson.Marshal(stored)
if e != nil {
return nil, e
}
var fields bson.M
if e = bson.Unmarshal(raw, &fields); e != nil {
return nil, e
}
set := bson.M{}
for key, value := range fields {
set[key] = bson.M{"$literal": value}
}
if len(stored.NodeExecutions) > 0 {
oldNodes := bson.M{"$filter": bson.M{"input": bson.M{"$ifNull": bson.A{"$nodeExecutions", bson.A{}}}, "as": "old", "cond": bson.M{"$eq": bson.A{"$$old.nodeId", "$$new.nodeId"}}}}
oldArtifacts := bson.M{"$reduce": bson.M{"input": oldNodes, "initialValue": bson.A{}, "in": bson.M{"$concatArrays": bson.A{"$$value", bson.M{"$ifNull": bson.A{"$$this.artifacts", bson.A{}}}}}}}
newArtifacts := bson.M{"$ifNull": bson.A{"$$new.artifacts", bson.A{}}}
newIDs := bson.M{"$map": bson.M{"input": newArtifacts, "as": "a", "in": "$$a.id"}}
keep := bson.M{"$filter": bson.M{"input": oldArtifacts, "as": "a", "cond": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$a.portId", "materializedSliceDirectory"}}, bson.M{"$not": bson.A{bson.M{"$in": bson.A{"$$a.id", newIDs}}}}}}}}
set["nodeExecutions"] = bson.M{"$map": bson.M{"input": bson.M{"$literal": stored.NodeExecutions}, "as": "new", "in": bson.M{"$mergeObjects": bson.A{"$$new", bson.M{"artifacts": bson.M{"$concatArrays": bson.A{newArtifacts, keep}}}}}}}
}
p := mongo.Pipeline{bson.D{{Key: "$set", Value: set}}}
if len(unset) > 0 {
p = append(p, bson.D{{Key: "$unset", Value: unset}})
}
return p, nil
}
// Field-level publication cannot overwrite another Capture or worker's state.
func (s *MemoryStore) AppendAnnotationArtifact(_ context.Context, org, node string, a domain.ExecutionArtifact) error {
s.mu.Lock()
defer s.mu.Unlock()
r, ok := s.runs[a.RunID]
if !ok || r.OrganizationID != org {
return fmt.Errorf("annotation run removed")
}
for _, old := range r.Artifacts {
if old.ID == a.ID {
return nil
}
}
found := false
for i := range r.NodeExecutions {
if r.NodeExecutions[i].NodeID == node {
r.NodeExecutions[i].Artifacts = append(r.NodeExecutions[i].Artifacts, a)
found = true
}
}
if !found {
return fmt.Errorf("annotation node removed")
}
r.Artifacts = append(r.Artifacts, a)
r.UpdatedAt = nowISO()
s.runs[r.ID] = clone(r)
return nil
}
func (s *MongoStore) AppendAnnotationArtifact(ctx context.Context, org, node string, a domain.ExecutionArtifact) error {
old := bson.M{"$ifNull": bson.A{"$$node.artifacts", bson.A{}}}
ids := bson.M{"$map": bson.M{"input": old, "as": "a", "in": "$$a.id"}}
items := bson.M{"$cond": bson.A{bson.M{"$in": bson.A{a.ID, ids}}, old, bson.M{"$concatArrays": bson.A{old, bson.M{"$literal": []domain.ExecutionArtifact{a}}}}}}
nodes := bson.M{"$map": bson.M{"input": "$nodeExecutions", "as": "node", "in": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{"$$node.nodeId", node}}, bson.M{"$mergeObjects": bson.A{"$$node", bson.M{"artifacts": items}}}, "$$node"}}}}
r, e := s.runs.UpdateOne(ctx, bson.M{"id": a.RunID, "organizationId": org, "nodeExecutions.nodeId": node}, mongo.Pipeline{bson.D{{Key: "$set", Value: bson.M{"nodeExecutions": nodes, "updatedAt": nowISO()}}}})
if e != nil {
return e
}
if r.MatchedCount != 1 {
return fmt.Errorf("annotation run/node removed")
}
return nil
}

View File

@ -205,6 +205,14 @@ func (store *MemoryStore) UpsertAnnotationTask(_ context.Context, item domain.An
}
item.UpdatedAt = now
store.annotationTasks[item.ID] = item
if item.SourceIsFullVideo && item.TemplateVersion >= 5 && item.ArtifactID != "" {
if store.annotationIndexJobs == nil {
store.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
}
if _, exists := store.annotationIndexJobs[item.ID]; !exists {
store.annotationIndexJobs[item.ID] = annotationIndexJob(item)
}
}
return nil
}
@ -505,7 +513,13 @@ func (store *MongoStore) UpsertAnnotationTask(ctx context.Context, item domain.A
item.CreatedAt = now
}
item.UpdatedAt = now
return mongoUpsert(ctx, store.annotationTasks, item.ID, item)
if err := mongoUpsert(ctx, store.annotationTasks, item.ID, item); err != nil {
return err
}
if item.SourceIsFullVideo && item.TemplateVersion >= 5 && item.ArtifactID != "" {
_ = store.EnqueueAnnotationIndex(ctx, item)
}
return nil
}
func (store *MongoStore) ListAnnotationRevisions(ctx context.Context, organizationID string, taskID string) ([]domain.AnnotationRevisionRecord, error) {

View File

@ -0,0 +1,188 @@
package store
import (
"context"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"sort"
)
type AnnotationProcessingStore interface {
AppendAnnotationArtifact(context.Context, string, string, domain.ExecutionArtifact) error
GetAnnotationTimelineIndex(context.Context, string) (*domain.AnnotationTimelineIndex, error)
PutAnnotationTimelineIndex(context.Context, domain.AnnotationTimelineIndex) error
EnqueueAnnotationIndex(context.Context, domain.AnnotationTaskRecord) error
ClaimAnnotationIndex(context.Context, string, string, string) (*domain.AnnotationIndexJob, error)
FinishAnnotationIndex(context.Context, domain.AnnotationIndexJob) (bool, error)
ClaimAnnotationProcessing(context.Context, string, string, string) (*domain.HumanItemRecord, error)
SaveAnnotationProcessing(context.Context, string, domain.AnnotationProcessingJob) (bool, error)
TouchAnnotationProcessing(context.Context, string, string, string, string, string, int, int) (bool, error)
}
func (s *MemoryStore) TouchAnnotationProcessing(_ context.Context, itemID, id, owner, until, stage string, completed, total int) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
h, ok := s.humanItems[itemID]
if !ok || h.ProcessingJob == nil || h.ProcessingJob.ID != id || h.ProcessingJob.Owner != owner || h.ProcessingJob.Status != "running" {
return false, nil
}
h.ProcessingJob.LeaseUntil = until
h.ProcessingJob.Stage = stage
h.ProcessingJob.CompletedSlices = completed
h.ProcessingJob.TotalSlices = total
h.ProcessingJob.UpdatedAt = nowISO()
s.humanItems[itemID] = h
return true, nil
}
func (s *MongoStore) TouchAnnotationProcessing(ctx context.Context, itemID, id, owner, until, stage string, completed, total int) (bool, error) {
r, e := s.humanItems.UpdateOne(ctx, bson.M{"id": itemID, "processingJob.id": id, "processingJob.owner": owner, "processingJob.status": "running"}, bson.M{"$set": bson.M{"processingJob.leaseUntil": until, "processingJob.stage": stage, "processingJob.completedSlices": completed, "processingJob.totalSlices": total, "processingJob.updatedAt": nowISO()}})
if e != nil {
return false, e
}
return r.MatchedCount == 1, nil
}
func (s *MemoryStore) GetAnnotationTimelineIndex(_ context.Context, id string) (*domain.AnnotationTimelineIndex, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.annotationTimelineIndexes[id]
if !ok {
return nil, nil
}
v.Body = append([]byte(nil), v.Body...)
return &v, nil
}
func (s *MemoryStore) PutAnnotationTimelineIndex(_ context.Context, v domain.AnnotationTimelineIndex) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.annotationTimelineIndexes == nil {
s.annotationTimelineIndexes = map[string]domain.AnnotationTimelineIndex{}
}
v.Body = append([]byte(nil), v.Body...)
s.annotationTimelineIndexes[v.ID] = v
return nil
}
func (s *MongoStore) GetAnnotationTimelineIndex(ctx context.Context, id string) (*domain.AnnotationTimelineIndex, error) {
return mongoGetByID[domain.AnnotationTimelineIndex](ctx, s.db.Collection("annotationTimelineIndexes"), id)
}
func (s *MongoStore) PutAnnotationTimelineIndex(ctx context.Context, v domain.AnnotationTimelineIndex) error {
return mongoUpsert(ctx, s.db.Collection("annotationTimelineIndexes"), v.ID, v)
}
func annotationIndexJob(t domain.AnnotationTaskRecord) domain.AnnotationIndexJob {
return domain.AnnotationIndexJob{ID: t.ID, OrganizationID: t.OrganizationID, ProjectID: t.ProjectID, WorkflowID: t.WorkflowID, RunID: t.RunID, Status: "queued", CreatedAt: nowISO(), NextAttemptAt: ""}
}
func (s *MemoryStore) EnqueueAnnotationIndex(_ context.Context, t domain.AnnotationTaskRecord) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.annotationIndexJobs == nil {
s.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
}
if _, ok := s.annotationIndexJobs[t.ID]; !ok {
s.annotationIndexJobs[t.ID] = annotationIndexJob(t)
}
return nil
}
func (s *MongoStore) EnqueueAnnotationIndex(ctx context.Context, t domain.AnnotationTaskRecord) error {
_, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": t.ID}, bson.M{"$setOnInsert": annotationIndexJob(t)}, options.Update().SetUpsert(true))
return e
}
func claimableJob(status, lease, next, now string) bool {
return status == "queued" && next <= now || status == "running" && lease < now
}
func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
s.mu.Lock()
defer s.mu.Unlock()
ids := []string{}
for id, j := range s.annotationIndexJobs {
if claimableJob(j.Status, j.LeaseUntil, j.NextAttemptAt, now) {
ids = append(ids, id)
}
}
sort.Strings(ids)
if len(ids) == 0 {
return nil, nil
}
j := s.annotationIndexJobs[ids[0]]
j.Status = "running"
j.Owner = owner
j.LeaseUntil = until
j.Attempt++
s.annotationIndexJobs[j.ID] = j
return &j, nil
}
func (s *MongoStore) ClaimAnnotationIndex(ctx context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
var j domain.AnnotationIndexJob
e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&j)
if e == mongo.ErrNoDocuments {
return nil, nil
}
return &j, e
}
func (s *MemoryStore) FinishAnnotationIndex(_ context.Context, j domain.AnnotationIndexJob) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
old, ok := s.annotationIndexJobs[j.ID]
if !ok || old.Owner != j.Owner || old.Status != "running" {
return false, nil
}
s.annotationIndexJobs[j.ID] = j
return true, nil
}
func (s *MongoStore) FinishAnnotationIndex(ctx context.Context, j domain.AnnotationIndexJob) (bool, error) {
r, e := s.db.Collection("annotationIndexJobs").ReplaceOne(ctx, bson.M{"id": j.ID, "owner": j.Owner, "status": "running"}, j)
if e != nil {
return false, e
}
return r.MatchedCount == 1, nil
}
func (s *MemoryStore) ClaimAnnotationProcessing(_ context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
s.mu.Lock()
defer s.mu.Unlock()
ids := []string{}
for id, h := range s.humanItems {
if j := h.ProcessingJob; j != nil && claimableJob(j.Status, j.LeaseUntil, "", now) {
ids = append(ids, id)
}
}
sort.Strings(ids)
if len(ids) == 0 {
return nil, nil
}
h := clone(s.humanItems[ids[0]])
h.ProcessingJob.Status = "running"
h.ProcessingJob.Owner = owner
h.ProcessingJob.LeaseUntil = until
h.ProcessingJob.Attempt++
s.humanItems[h.ID] = clone(h)
return &h, nil
}
func (s *MongoStore) ClaimAnnotationProcessing(ctx context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
var h domain.HumanItemRecord
e := s.humanItems.FindOneAndUpdate(ctx, bson.M{"$or": bson.A{bson.M{"processingJob.status": "queued"}, bson.M{"processingJob.status": "running", "processingJob.leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"processingJob.status": "running", "processingJob.owner": owner, "processingJob.leaseUntil": until}, "$inc": bson.M{"processingJob.attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "processingJob.createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&h)
if e == mongo.ErrNoDocuments {
return nil, nil
}
return &h, e
}
func (s *MemoryStore) SaveAnnotationProcessing(_ context.Context, itemID string, j domain.AnnotationProcessingJob) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
h, ok := s.humanItems[itemID]
if !ok || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner || h.ProcessingJob.Status != "running" {
return false, nil
}
h.ProcessingJob = &j
s.humanItems[itemID] = clone(h)
return true, nil
}
func (s *MongoStore) SaveAnnotationProcessing(ctx context.Context, itemID string, j domain.AnnotationProcessingJob) (bool, error) {
r, e := s.humanItems.UpdateOne(ctx, bson.M{"id": itemID, "processingJob.id": j.ID, "processingJob.owner": j.Owner, "processingJob.status": "running"}, bson.M{"$set": bson.M{"processingJob": j}})
if e != nil {
return false, e
}
return r.MatchedCount == 1, nil
}

View File

@ -0,0 +1,80 @@
package store
import (
"context"
"droboticflow/internal/domain"
"encoding/json"
"go.mongodb.org/mongo-driver/bson"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestAnnotationOutboxBSONAndExclusiveClaims(t *testing.T) {
ctx := context.Background()
s := NewMemoryStore()
j := domain.AnnotationProcessingJob{ID: "job", TaskID: "task", Status: "queued", Input: map[string]any{"sliceSegments": []map[string]any{{"id": "slice", "start": 0.0, "end": 1.0}}}}
encoded, e := bson.Marshal(j)
if e != nil {
t.Fatal(e)
}
var restored domain.AnnotationProcessingJob
if e = bson.Unmarshal(encoded, &restored); e != nil {
t.Fatal(e)
}
body, _ := json.Marshal(restored.Input)
if strings.Contains(string(body), "Key") || !strings.Contains(string(body), "sliceSegments") {
t.Fatal("BSON changed queued input", string(body))
}
h := domain.HumanItemRecord{ID: "item", OrganizationID: "org", Version: 3, ProcessingJob: &restored}
s.UpsertHumanItem(ctx, h)
var claims atomic.Int32
var wg sync.WaitGroup
now := time.Now().UTC()
for n := 0; n < 16; n++ {
wg.Add(1)
go func() {
defer wg.Done()
h, e := s.ClaimAnnotationProcessing(ctx, domain.NewID("worker"), now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if e != nil {
t.Error(e)
}
if h != nil {
claims.Add(1)
if len(h.ProcessingJob.Input) == 0 {
t.Error("private input lost in store clone")
}
}
}()
}
wg.Wait()
if claims.Load() != 1 {
t.Fatal("duplicate claims", claims.Load())
}
}
func TestWorkerSnapshotCannotErasePublishedAnnotationArtifacts(t *testing.T) {
s := NewMemoryStore()
ctx := context.Background()
r := domain.RunRecord{ID: "run", OrganizationID: "org", NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "human"}}}
s.UpsertRun(ctx, r)
a := domain.ExecutionArtifact{ID: "media", RunID: r.ID, NodeID: "human", PortID: "materializedSliceDirectory"}
if e := s.AppendAnnotationArtifact(ctx, "org", "human", a); e != nil {
t.Fatal(e)
}
r.Status = domain.RunStatusWaitingForHuman
s.UpsertRun(ctx, r)
fresh, _ := s.GetRun(ctx, r.ID)
if len(fresh.Artifacts) != 1 || fresh.Artifacts[0].ID != a.ID {
t.Fatal("stale worker erased media")
}
if e := s.AppendAnnotationArtifact(ctx, "org", "human", a); e != nil {
t.Fatal(e)
}
fresh, _ = s.GetRun(ctx, r.ID)
if len(fresh.Artifacts) != 1 {
t.Fatal("duplicate artifact")
}
}

View File

@ -210,6 +210,15 @@ func clone[T any](value T) T {
data, _ := json.Marshal(value)
var out T
_ = json.Unmarshal(data, &out)
// The durable processing outbox is intentionally absent from public JSON.
// Preserve it when cloning private in-memory records for store operations.
if item, ok := any(value).(domain.HumanItemRecord); ok && item.ProcessingJob != nil {
copied := any(out).(domain.HumanItemRecord)
job := *item.ProcessingJob
job.Input = clone(job.Input)
copied.ProcessingJob = &job
return any(copied).(T)
}
return out
}

View File

@ -169,6 +169,9 @@ func (store *MemoryStore) UpdateHumanItemVersioned(_ context.Context, item domai
if !ok || existing.Version != expectedVersion {
return false, nil
}
if j := item.ProcessingJob; j != nil && j.Owner != "" && (existing.ProcessingJob == nil || existing.ProcessingJob.ID != j.ID || existing.ProcessingJob.Owner != j.Owner) {
return false, nil
}
item = normalizeHumanItem(item)
store.humanItems[item.ID] = clone(item)
return true, nil
@ -467,7 +470,12 @@ func (store *MongoStore) UpsertHumanItem(ctx context.Context, item domain.HumanI
func (store *MongoStore) UpdateHumanItemVersioned(ctx context.Context, item domain.HumanItemRecord, expectedVersion int) (bool, error) {
item = normalizeHumanItem(item)
result, err := store.humanItems.ReplaceOne(ctx, bson.M{"id": item.ID, "version": expectedVersion}, item)
filter := bson.M{"id": item.ID, "version": expectedVersion}
if j := item.ProcessingJob; j != nil && j.Owner != "" {
filter["processingJob.id"] = j.ID
filter["processingJob.owner"] = j.Owner
}
result, err := store.humanItems.ReplaceOne(ctx, filter, item)
if err != nil {
return false, err
}

View File

@ -12,71 +12,73 @@ import (
)
type MemoryStore struct {
mu sync.Mutex
organizations map[string]domain.OrganizationRecord
projects map[string]domain.ProjectRecord
projectDeletions map[string]domain.ProjectDeletionTask
operators map[string]domain.OperatorRecord
builtinOperatorDeletions map[string]domain.BuiltinOperatorDeletionRecord
workflows map[string]domain.WorkflowRecord
workflowRuntimePolicies map[string]domain.WorkflowRuntimePolicyRecord
templates map[string]domain.TemplateRecord
workflowReleases map[string]domain.WorkflowReleaseRecord
projectWorkflowInstances map[string]domain.ProjectWorkflowInstanceRecord
runs map[string]domain.RunRecord
sourceStates map[string]domain.SourceStateRecord
captureRecords map[string]domain.CaptureRecord
collectionTasks map[string]domain.CollectionTaskRecord
datasetVersions map[string]domain.DatasetVersionRecord
dataMetricFacts map[string]domain.DataMetricFactRecord
auditTasks map[string]domain.AuditTaskRecord
samplingTasks map[string]domain.SamplingTaskRecord
annotationWorkgroups map[string]domain.AnnotationWorkgroupRecord
annotationMemberships map[string]domain.ProjectAnnotationMembershipRecord
suppliers map[string]domain.SupplierRecord
projectMembers map[string]domain.ProjectMemberRecord
annotationLabels map[string]domain.AnnotationLabelRecord
projectAnnotationLabels map[string]domain.ProjectAnnotationLabelsRecord
annotationTemplates map[string]domain.AnnotationTemplateRecord
annotationTasks map[string]domain.AnnotationTaskRecord
annotationRevisions map[string]domain.AnnotationRevisionRecord
sliceRevisions map[string]domain.SliceRevisionRecord
annotationReviews map[string]domain.AnnotationReviewRecord
annotationWorklogEvents map[string]domain.AnnotationWorklogEventRecord
organizationPlugins map[string]domain.OrganizationPluginRecord
humanBatches map[string]domain.HumanBatchRecord
humanItems map[string]domain.HumanItemRecord
humanClaimReceipts map[string]HumanClaimReceipt
humanStageAttempts map[string]domain.HumanStageAttemptRecord
humanEvents map[string]domain.HumanEventRecord
humanRejects map[string]domain.HumanRejectRecord
releasePartitions map[string]domain.ReleasePartitionRecord
deliveryPackages map[string]domain.DeliveryPackageRecord
users map[string]domain.UserRecord
sessions map[string]domain.SessionRecord
storageConnections map[string]domain.StorageConnectionRecord
uploadSessions map[string]domain.ObjectStorageUploadSessionRecord
mongoConnections map[string]domain.MongoConnectionRecord
objectSyncSpaces map[string]domain.ObjectSyncSpaceRecord
objectSyncDevices map[string]domain.ObjectSyncDeviceRecord
objectSyncAuthorizations map[string]domain.ObjectSyncDeviceAuthorizationRecord
objectSyncCredentials map[string]domain.ObjectSyncCredentialRecord
objectSyncSessions map[string]domain.ObjectSyncSessionRecord
objectSyncItems map[string]domain.ObjectSyncItemRecord
objectSyncLeases map[string]domain.ObjectSyncLeaseRecord
objectSyncHeads map[string]domain.ObjectSyncObjectHeadRecord
objectSyncVersions map[string]domain.ObjectSyncVersionRecord
objectSyncAuditEvents map[string]domain.ObjectSyncAuditEvent
objectSyncAuditSeals map[string]domain.ObjectSyncAuditSealRecord
objectSyncReleases map[string]domain.ObjectSyncReleaseRecord
objectSyncClientReleases map[string]domain.ObjectSyncClientReleaseRecord
objectSyncReconciles map[string]domain.ObjectSyncReconcileRecord
objectSyncIdempotency map[string]domain.ObjectSyncIdempotencyRecord
objectSyncRateLimits map[string]domain.ObjectSyncRateLimitRecord
databaseCollections map[string][]map[string]any
databaseIndexes map[string][]domain.DatabaseIndexSummary
settings *domain.SettingsRecord
artifactStorageSettings map[string]domain.ArtifactStorageSettings
annotationTimelineIndexes map[string]domain.AnnotationTimelineIndex
annotationIndexJobs map[string]domain.AnnotationIndexJob
mu sync.Mutex
organizations map[string]domain.OrganizationRecord
projects map[string]domain.ProjectRecord
projectDeletions map[string]domain.ProjectDeletionTask
operators map[string]domain.OperatorRecord
builtinOperatorDeletions map[string]domain.BuiltinOperatorDeletionRecord
workflows map[string]domain.WorkflowRecord
workflowRuntimePolicies map[string]domain.WorkflowRuntimePolicyRecord
templates map[string]domain.TemplateRecord
workflowReleases map[string]domain.WorkflowReleaseRecord
projectWorkflowInstances map[string]domain.ProjectWorkflowInstanceRecord
runs map[string]domain.RunRecord
sourceStates map[string]domain.SourceStateRecord
captureRecords map[string]domain.CaptureRecord
collectionTasks map[string]domain.CollectionTaskRecord
datasetVersions map[string]domain.DatasetVersionRecord
dataMetricFacts map[string]domain.DataMetricFactRecord
auditTasks map[string]domain.AuditTaskRecord
samplingTasks map[string]domain.SamplingTaskRecord
annotationWorkgroups map[string]domain.AnnotationWorkgroupRecord
annotationMemberships map[string]domain.ProjectAnnotationMembershipRecord
suppliers map[string]domain.SupplierRecord
projectMembers map[string]domain.ProjectMemberRecord
annotationLabels map[string]domain.AnnotationLabelRecord
projectAnnotationLabels map[string]domain.ProjectAnnotationLabelsRecord
annotationTemplates map[string]domain.AnnotationTemplateRecord
annotationTasks map[string]domain.AnnotationTaskRecord
annotationRevisions map[string]domain.AnnotationRevisionRecord
sliceRevisions map[string]domain.SliceRevisionRecord
annotationReviews map[string]domain.AnnotationReviewRecord
annotationWorklogEvents map[string]domain.AnnotationWorklogEventRecord
organizationPlugins map[string]domain.OrganizationPluginRecord
humanBatches map[string]domain.HumanBatchRecord
humanItems map[string]domain.HumanItemRecord
humanClaimReceipts map[string]HumanClaimReceipt
humanStageAttempts map[string]domain.HumanStageAttemptRecord
humanEvents map[string]domain.HumanEventRecord
humanRejects map[string]domain.HumanRejectRecord
releasePartitions map[string]domain.ReleasePartitionRecord
deliveryPackages map[string]domain.DeliveryPackageRecord
users map[string]domain.UserRecord
sessions map[string]domain.SessionRecord
storageConnections map[string]domain.StorageConnectionRecord
uploadSessions map[string]domain.ObjectStorageUploadSessionRecord
mongoConnections map[string]domain.MongoConnectionRecord
objectSyncSpaces map[string]domain.ObjectSyncSpaceRecord
objectSyncDevices map[string]domain.ObjectSyncDeviceRecord
objectSyncAuthorizations map[string]domain.ObjectSyncDeviceAuthorizationRecord
objectSyncCredentials map[string]domain.ObjectSyncCredentialRecord
objectSyncSessions map[string]domain.ObjectSyncSessionRecord
objectSyncItems map[string]domain.ObjectSyncItemRecord
objectSyncLeases map[string]domain.ObjectSyncLeaseRecord
objectSyncHeads map[string]domain.ObjectSyncObjectHeadRecord
objectSyncVersions map[string]domain.ObjectSyncVersionRecord
objectSyncAuditEvents map[string]domain.ObjectSyncAuditEvent
objectSyncAuditSeals map[string]domain.ObjectSyncAuditSealRecord
objectSyncReleases map[string]domain.ObjectSyncReleaseRecord
objectSyncClientReleases map[string]domain.ObjectSyncClientReleaseRecord
objectSyncReconciles map[string]domain.ObjectSyncReconcileRecord
objectSyncIdempotency map[string]domain.ObjectSyncIdempotencyRecord
objectSyncRateLimits map[string]domain.ObjectSyncRateLimitRecord
databaseCollections map[string][]map[string]any
databaseIndexes map[string][]domain.DatabaseIndexSummary
settings *domain.SettingsRecord
artifactStorageSettings map[string]domain.ArtifactStorageSettings
}
func NewMemoryStore() *MemoryStore {
@ -570,6 +572,16 @@ func (store *MemoryStore) DeleteProject(_ context.Context, id string) (bool, err
return false, nil
}
delete(store.projects, id)
for k, v := range store.annotationTimelineIndexes {
if v.ProjectID == id {
delete(store.annotationTimelineIndexes, k)
}
}
for k, v := range store.annotationIndexJobs {
if v.ProjectID == id {
delete(store.annotationIndexJobs, k)
}
}
workflowIDs := make([]string, 0)
for workflowID, workflow := range store.workflows {
@ -965,6 +977,16 @@ func (store *MemoryStore) deleteWorkflowLocked(id string) bool {
if _, ok := store.workflows[id]; !ok {
return false
}
for k, v := range store.annotationTimelineIndexes {
if v.WorkflowID == id {
delete(store.annotationTimelineIndexes, k)
}
}
for k, v := range store.annotationIndexJobs {
if v.WorkflowID == id {
delete(store.annotationIndexJobs, k)
}
}
annotationTaskIDs := map[string]struct{}{}
humanBatchIDs := map[string]struct{}{}
humanItemIDs := map[string]struct{}{}
@ -1824,6 +1846,9 @@ func (store *MemoryStore) UpsertRun(_ context.Context, run domain.RunRecord) err
}
}
computeRunDuration(&run)
if old, ok := store.runs[run.ID]; ok {
preserveAnnotationArtifacts(&run, old)
}
run.CaptureDirectories = domain.CaptureDirectoriesFromRun(run)
store.runs[run.ID] = clone(run)
return nil

View File

@ -447,6 +447,8 @@ func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, er
filter bson.M
}{
{store.annotationTasks, bson.M{"projectId": id}},
{store.db.Collection("annotationTimelineIndexes"), bson.M{"projectId": id}},
{store.db.Collection("annotationIndexJobs"), bson.M{"projectId": id}},
{store.humanRejects, bson.M{"projectId": id}},
{store.humanItems, bson.M{"projectId": id}},
{store.humanBatches, bson.M{"projectId": id}},
@ -777,6 +779,8 @@ func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, e
}{
{store.projectWorkflowInstances, bson.M{"workflowId": id}},
{store.annotationTasks, bson.M{"workflowId": id}},
{store.db.Collection("annotationTimelineIndexes"), bson.M{"workflowId": id}},
{store.db.Collection("annotationIndexJobs"), bson.M{"workflowId": id}},
{store.humanRejects, bson.M{"workflowId": id}},
{store.humanItems, bson.M{"workflowId": id}},
{store.humanBatches, bson.M{"workflowId": id}},
@ -1782,14 +1786,10 @@ func (store *MongoStore) UpsertRun(ctx context.Context, item domain.RunRecord) e
}
computeRunDuration(&item)
stored := compactRunForStorage(item)
update := bson.M{"$set": stored}
unsetFields := runStorageFieldsToUnset(item)
if len(unsetFields) > 0 {
unset := bson.M{}
for _, field := range unsetFields {
unset[field] = ""
}
update["$unset"] = unset
update, err := annotationSafeRunUpdate(stored, unsetFields)
if err != nil {
return err
}
_, err = store.runs.UpdateOne(ctx, bson.M{"id": item.ID}, update, options.Update().SetUpsert(allow))
return err

View File

@ -49,6 +49,16 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
return err
}
indexesByCollection := map[*mongo.Collection][]mongo.IndexModel{
store.db.Collection("annotationTimelineIndexes"): {
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
index("projectId", bson.D{{Key: "projectId", Value: 1}}),
index("workflowId", bson.D{{Key: "workflowId", Value: 1}}),
},
store.db.Collection("annotationIndexJobs"): {
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
index("status_nextAttemptAt", bson.D{{Key: "status", Value: 1}, {Key: "nextAttemptAt", Value: 1}}),
index("status_leaseUntil", bson.D{{Key: "status", Value: 1}, {Key: "leaseUntil", Value: 1}}),
},
store.db.Collection("humanClaimRequests"): {
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
index("organization_project", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}}),
@ -197,6 +207,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
index("organizationId_workflowId_status", bson.D{{Key: "organizationId", Value: 1}, {Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}),
},
store.humanItems: {
index("processingJob_status_lease", bson.D{{Key: "processingJob.status", Value: 1}, {Key: "processingJob.leaseUntil", Value: 1}, {Key: "processingJob.createdAt", Value: 1}}),
index("organizationId_legacyTaskId", bson.D{{Key: "organizationId", Value: 1}, {Key: "legacyTaskId", Value: 1}}),
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
uniqueIndex("organizationId_batchId_subjectItemKey_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "batchId", Value: 1}, {Key: "subject.itemKey", Value: 1}}),

View File

@ -14,6 +14,7 @@ import (
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"golang.org/x/sync/errgroup"
)
type deliveryPrefixLister interface {
@ -177,31 +178,58 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
}
return paths[i] < paths[j]
})
entries := []directoryManifestEntry{}
var total int64
for _, path := range paths {
entries := make([]directoryManifestEntry, len(paths))
publishFile := func(ctx context.Context, i int, path string) error {
relative, e := filepath.Rel(artifact.Path, path)
if e != nil {
return artifact, e
return e
}
relative = filepath.ToSlash(relative)
if strings.HasPrefix(relative, "../") || relative == "manifest.json" || relative == ".delivery-owner.json" {
return artifact, fmt.Errorf("无效交付成员 %s", relative)
return fmt.Errorf("无效交付成员 %s", relative)
}
stage := stageRoot + "/files/" + relative
meta, sha, e := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, path)
if e != nil {
return artifact, e
return e
}
key := target.Prefix + "/" + relative
if e = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); e != nil {
return artifact, e
return e
}
if e = publisher.Delete(ctx, *connection, target.Bucket, stage); e != nil {
return artifact, e
return e
}
entries = append(entries, directoryManifestEntry{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha})
total += meta.SizeBytes
entries[i] = directoryManifestEntry{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}
return nil
}
parallel := 1
switch publisher.(type) {
case objectsync.BOSPublisher, *objectsync.BOSPublisher:
parallel = 4
}
group, gctx := errgroup.WithContext(ctx)
group.SetLimit(parallel)
for i, path := range paths {
if filepath.Base(path) == "packages-manifest.json" {
continue
}
group.Go(func() error { return publishFile(gctx, i, path) })
}
if e := group.Wait(); e != nil {
return artifact, e
}
// A visible package manifest must not precede its media files.
for i, path := range paths {
if filepath.Base(path) == "packages-manifest.json" {
if e := publishFile(ctx, i, path); e != nil {
return artifact, e
}
}
}
var total int64
for _, entry := range entries {
total += entry.SizeBytes
}
if len(entries) == 0 {
return artifact, fmt.Errorf("交付目录为空")

View File

@ -0,0 +1,52 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: droboticflow-annotation-processor
namespace: __NAMESPACE__
spec:
replicas: 1
selector:
matchLabels: {app: droboticflow-annotation-processor}
template:
metadata:
labels: {app: droboticflow-annotation-processor}
spec:
automountServiceAccountToken: false
terminationGracePeriodSeconds: 30
imagePullSecrets:
- name: __K8S_IMAGE_PULL_SECRET_NAME__
containers:
- name: processor
image: __API_IMAGE__
command: [/usr/local/bin/droboticflow-api]
env:
- {name: ANNOTATION_PROCESSOR_MODE, value: processor}
- {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "2"}
- {name: ANNOTATION_INDEX_BACKFILL, value: "1"}
- {name: WORKSPACE_MOUNT_PATH, value: "__WORKSPACE_MOUNT_PATH__"}
- {name: GLOBAL_BOS_ENDPOINT, value: "__BOS_ENDPOINT__"}
- {name: ANNOTATION_WORKSPACE_CONNECTION_ID, value: "__ANNOTATION_WORKSPACE_CONNECTION_ID__"}
- {name: ANNOTATION_WORKSPACE_BUCKET, value: "__ANNOTATION_WORKSPACE_BUCKET__"}
- name: MONGO_DB
valueFrom:
configMapKeyRef: {name: droboticflow-config, key: MONGO_DB}
- name: MONGO_URI
valueFrom:
secretKeyRef: {name: droboticflow-mongo-secret, key: MONGO_URI}
ports:
- {name: health, containerPort: 3003}
readinessProbe:
httpGet: {path: /ready, port: health}
periodSeconds: 5
resources:
requests: {cpu: "1", memory: 512Mi}
limits: {cpu: "4", memory: 8Gi}
volumeMounts:
- {name: workspace, mountPath: "__WORKSPACE_MOUNT_PATH__"}
- {name: scratch, mountPath: /tmp}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: __WORKSPACE_PVC_NAME__}
- name: scratch
emptyDir: {sizeLimit: 20Gi}

View File

@ -130,6 +130,7 @@ data:
DROSYNC_COORDINATOR_PORT: "3003"
DROBOTICFLOW_OBJECT_SYNC_ENABLED: "__DROBOTICFLOW_OBJECT_SYNC_ENABLED__"
MONGO_DB: "__MONGO_DB__"
ANNOTATION_PROCESSOR_MODE: "__ANNOTATION_PROCESSOR_MODE__"
MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__"
MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__"
WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__"
@ -288,6 +289,9 @@ spec:
key: MONGO_URI
- name: DIST_DIR
value: /app/dist
- name: ANNOTATION_PROCESSOR_MODE
valueFrom:
configMapKeyRef: {name: droboticflow-config, key: ANNOTATION_PROCESSOR_MODE}
- name: K8S_NAMESPACE
valueFrom:
configMapKeyRef:

View File

@ -28,6 +28,11 @@ BOS_ENDPOINT="${BOS_ENDPOINT:-}"
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
ANNOTATION_PROCESSOR_ENABLED="${ANNOTATION_PROCESSOR_ENABLED:-false}"
ANNOTATION_PROCESSOR_MODE=inprocess
if [[ "$ANNOTATION_PROCESSOR_ENABLED" == "true" ]]; then ANNOTATION_PROCESSOR_MODE=external; fi
ANNOTATION_WORKSPACE_CONNECTION_ID="${ANNOTATION_WORKSPACE_CONNECTION_ID:-}"
ANNOTATION_WORKSPACE_BUCKET="${ANNOTATION_WORKSPACE_BUCKET:-}"
API_NODE_PORT="${API_NODE_PORT:-30080}"
API_SERVICE_TYPE="${API_SERVICE_TYPE:-NodePort}"
API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
@ -122,6 +127,10 @@ OPERATOR_COMMAND_ALLOWLIST_REGEX_BLOCK="$(render_block_scalar "$OPERATOR_COMMAND
OPERATOR_COMMAND_DENYLIST_REGEX_BLOCK="$(render_block_scalar "$OPERATOR_COMMAND_DENYLIST_REGEX")"
export \
ANNOTATION_PROCESSOR_ENABLED \
ANNOTATION_PROCESSOR_MODE \
ANNOTATION_WORKSPACE_CONNECTION_ID \
ANNOTATION_WORKSPACE_BUCKET \
TEMPLATE \
K8S_NAMESPACE \
MONGO_DB \
@ -185,7 +194,12 @@ import re
import sys
template = pathlib.Path(os.environ["TEMPLATE"]).read_text(encoding="utf-8")
if os.environ["ANNOTATION_PROCESSOR_ENABLED"] == "true":
template += pathlib.Path(os.environ["TEMPLATE"]).with_name("annotation-processor.template.yaml").read_text(encoding="utf-8")
replacements = {
"__ANNOTATION_PROCESSOR_MODE__": os.environ["ANNOTATION_PROCESSOR_MODE"],
"__ANNOTATION_WORKSPACE_CONNECTION_ID__": os.environ["ANNOTATION_WORKSPACE_CONNECTION_ID"],
"__ANNOTATION_WORKSPACE_BUCKET__": os.environ["ANNOTATION_WORKSPACE_BUCKET"],
"__NAMESPACE__": os.environ["K8S_NAMESPACE"],
"__MONGO_DB__": os.environ["MONGO_DB"],
"__MONGO_CONNECT_RETRIES__": os.environ["MONGO_CONNECT_RETRIES"],

View File

@ -662,7 +662,7 @@ export default function AnnotationOperations() {
projectId ? annotationOperationsApi.claimCount(projectId).catch(() => null) : Promise.resolve(null),
]);
const owned = mine.filter((i) => i.legacyTaskId);
const details = await Promise.all(owned.map((i) => annotationOperationsApi.getTask(i.legacyTaskId)));
const details = await Promise.all(owned.map((i) => annotationOperationsApi.getTaskSummary(i.legacyTaskId)));
if (requestID !== projectLoadRequestRef.current) return;
setProjectMembers(members);
setTemplates(nextTemplates);

View File

@ -87,6 +87,8 @@ export const annotationOperationsApi = {
) => (await apiRequest<{ items: AnnotationTaskRecord[] }>(`/api/plugins/annotation-qa/tasks${query(filters)}`)).items,
getTask: (id: string, signal?: AbortSignal) =>
apiRequest<AnnotationTaskDetail>(`/api/plugins/annotation-qa/tasks/${id}`, { signal }),
getTaskSummary: (id: string) =>
apiRequest<{ task: AnnotationTaskRecord }>(`/api/plugins/annotation-qa/tasks/${id}?view=summary`),
getTaskStatus: (id: string, signal?: AbortSignal) =>
apiRequest<{
taskId: string;