forked from longtao.wu/droboticflow
⚡ Persist frame indices and isolate resumable annotation processing
This commit is contained in:
parent
ffa43bd5b9
commit
cc0202d40d
@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@ -90,6 +91,43 @@ func main() {
|
||||
}()
|
||||
|
||||
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
|
||||
if os.Getenv("ANNOTATION_PROCESSOR_MODE") == "processor" {
|
||||
if err := dataStore.EnsureIndexes(ctx); err != nil {
|
||||
fmt.Printf("[annotation-processor] indexes: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
parallel := 2
|
||||
if value, err := strconv.Atoi(os.Getenv("ANNOTATION_PROCESSOR_CONCURRENCY")); err == nil {
|
||||
parallel = value
|
||||
}
|
||||
health := &http.Server{Addr: "0.0.0.0:3003", ReadHeaderTimeout: 5 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/ready" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if e := dataStore.Ready(r.Context()); e != nil {
|
||||
http.Error(w, "store unavailable", 503)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write([]byte("annotation processor ready"))
|
||||
})}
|
||||
go func() {
|
||||
if e := health.ListenAndServe(); e != nil && e != http.ErrServerClosed {
|
||||
stop()
|
||||
}
|
||||
}()
|
||||
fmt.Println("[annotation-processor] durable worker and index precompute started")
|
||||
err := api.RunAnnotationProcessor(ctx, dataStore, cfg, parallel, os.Getenv("ANNOTATION_INDEX_BACKFILL") == "1")
|
||||
shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = health.Shutdown(shutdown)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
fmt.Printf("[annotation-processor] failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := dataStore.RunMigrations(ctx); err != nil {
|
||||
fmt.Printf("[api] failed to run mongo migrations: %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
30
docs/annotation-processing.md
Normal file
30
docs/annotation-processing.md
Normal file
@ -0,0 +1,30 @@
|
||||
# Durable annotation processing
|
||||
|
||||
## Compatibility boundary
|
||||
|
||||
No change to dictionary selection, role/supplier ownership, semantic blanks, explicit invalid intervals, discard evidence, acceptance rules, exact frame boundaries or delivery directory layout. Materialization manifests remain v2/v3. Existing generation files remain readable. New media objects are independently owned by each generation; no symlinks or references into a generation that retention can delete.
|
||||
|
||||
## Performance paths
|
||||
|
||||
1. New full-video annotation tasks enqueue index preparation. The independent processor builds a source-version-keyed timeline and stores a compressed, checksummed document in `annotationTimelineIndexes`. API restarts and memory-cache eviction no longer require rebuilding an existing index. Source ETag/version changes invalidate it. Legacy/open tasks can be backfilled without creating Runs or changing assignments.
|
||||
2. When only names/descriptions/disposition change and all frame windows still match, the metadata-only path validates the source version and prior manifest, copies verified media/PTS/IMU objects server-side in BOS, and rewrites only small annotation/synchronization manifests locally. It does not reread original IMU/PTS or decode video. A native-filesystem installation retains checked file copies as a compatibility fallback. Old BOS generations without pinned object ETags receive normal verification once.
|
||||
3. New/changed slices share input decoding in groups of up to eight outputs. Every output still uses half-open original-frame trims and decoded frame-count checks. Tests compare decoded frame hashes against the single-output implementation.
|
||||
4. Generation publication uses at most four concurrent BOS transfers and publishes its manifest last. Final delivery also uses bounded BOS concurrency, retaining conditional ownership checks, digest verification, retry semantics and manifest-last ordering.
|
||||
5. The API validates a submission once and atomically stores the durable outbox in the HumanItem CAS. It returns processing, not success. The processor rechecks current actor/project/role/source ownership and uses an expiring, fenced claim. A stale worker cannot publish after takeover. Worker Run snapshots preserve API-published materializations.
|
||||
6. Task-pool cards use an authorized task-summary view instead of loading each task's full history. Processing polls use the existing lightweight status view. Histories are loaded for editing, not every progress tick.
|
||||
|
||||
## Operation
|
||||
|
||||
- Default `ANNOTATION_PROCESSOR_MODE=inprocess` preserves existing deployments until explicitly switched.
|
||||
- The API uses `ANNOTATION_PROCESSOR_MODE=external` after the independent processor is healthy.
|
||||
- The processor runs the same API image/binary with `ANNOTATION_PROCESSOR_MODE=processor`. It has no public task-mutation endpoint; `/ready` on port 3003 is for health checks only. It does not need a Kubernetes service-account token.
|
||||
- `ANNOTATION_PROCESSOR_CONCURRENCY=2` is the default (allowed 1–4 per process). Multiple replicas claim independently from the database. Local `/tmp` needs bounded scratch capacity; the supplied deployment uses a 20 GiB emptyDir.
|
||||
- `ANNOTATION_INDEX_BACKFILL=1` queues existing pending full-video annotation tasks at startup. Index work has a separate bounded loop. Precomputation may take time for a backlog; it is not a claim that every newly discovered file is immediately ready.
|
||||
- Processing leases last two minutes and renew every ten seconds. Interrupted work is requeued on graceful shutdown, or reclaimed after lease expiry. Repeated interruptions stop after three attempts with the saved workspace intact. Legacy API-only reservations can be adopted without changing the visible business version.
|
||||
- BOS workspace mapping uses `ANNOTATION_WORKSPACE_CONNECTION_ID` / `ANNOTATION_WORKSPACE_BUCKET`, with the prior project-deletion workspace mapping as a fallback. Native local files use filesystem versions.
|
||||
- Manifest rendering: set `ANNOTATION_PROCESSOR_ENABLED=true` plus the workspace mapping to render `k8s/annotation-processor.template.yaml` and switch API mode. Keep the processor on the same code/image release as the API when deploying future changes. Roll out the processor before enabling external mode; inspect existing legacy in-flight submissions before changing modes.
|
||||
- Project/workflow deletion removes index/cache bookkeeping with the associated scope. Media retention still removes only superseded private generation paths, never original inputs.
|
||||
|
||||
## Verification
|
||||
|
||||
Focused tests cover durable outbox BSON/JSON privacy, concurrent exclusive claims, expired-owner fencing, process replacement, source changes, persistent index recovery/corruption, proactive indexing, semantic-empty submission, metadata-only generation, decoded frame equivalence and delivery ownership/retry behavior. Live load tests must use a separate database/processor and private Run prefixes; production source objects are read-only. Report the tested concurrency and actual video duration rather than extrapolating unlimited capacity.
|
||||
24
internal/api/annotation_artifact.go
Normal file
24
internal/api/annotation_artifact.go
Normal file
@ -0,0 +1,24 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func (s *server) storeAnnotationMaterializationArtifact(ctx context.Context, t domain.AnnotationTaskRecord, run *domain.RunRecord, a domain.ExecutionArtifact) error {
|
||||
if e := s.assertAnnotationLease(ctx); e != nil {
|
||||
return e
|
||||
}
|
||||
if db, ok := s.store.(store.AnnotationProcessingStore); ok {
|
||||
return db.AppendAnnotationArtifact(ctx, t.OrganizationID, t.NodeID, a)
|
||||
}
|
||||
run.Artifacts = append(run.Artifacts, a)
|
||||
for i := range run.NodeExecutions {
|
||||
if run.NodeExecutions[i].NodeID == t.NodeID {
|
||||
run.NodeExecutions[i].Artifacts = append(run.NodeExecutions[i].Artifacts, a)
|
||||
}
|
||||
}
|
||||
run.UpdatedAt = nowISO()
|
||||
return s.store.UpsertRun(ctx, *run)
|
||||
}
|
||||
43
internal/api/annotation_batch_decode.go
Normal file
43
internal/api/annotation_batch_decode.go
Normal file
@ -0,0 +1,43 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type annotationFrameSlice struct {
|
||||
Start, End int
|
||||
Output string
|
||||
}
|
||||
|
||||
// Each input is decoded once per bounded group, not once per output slice.
|
||||
// Half-open frame trims preserve exact original frame selection; no fast seek
|
||||
// or keyframe-only stream copy is used.
|
||||
func writeFrameAccurateSlices(ctx context.Context, source string, slices []annotationFrameSlice) error {
|
||||
for offset := 0; offset < len(slices); offset += 8 {
|
||||
group := slices[offset:min(offset+8, len(slices))]
|
||||
filters := []string{}
|
||||
split := fmt.Sprintf("[0:v]split=%d", len(group))
|
||||
for i := range group {
|
||||
split += fmt.Sprintf("[in%d]", i)
|
||||
}
|
||||
filters = append(filters, split)
|
||||
for i, s := range group {
|
||||
if s.Start < 0 || s.End <= s.Start {
|
||||
return fmt.Errorf("invalid frame slice")
|
||||
}
|
||||
filters = append(filters, fmt.Sprintf("[in%d]trim=start_frame=%d:end_frame=%d,setpts=PTS-STARTPTS[out%d]", i, s.Start, s.End, i))
|
||||
}
|
||||
args := []string{"-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-threads", "2", "-i", source, "-filter_complex_threads", "1", "-filter_complex", strings.Join(filters, ";")}
|
||||
for i, s := range group {
|
||||
args = append(args, "-map", fmt.Sprintf("[out%d]", i), "-an", "-c:v", "libx264", "-threads", "2", "-preset", "ultrafast", "-crf", "23", "-pix_fmt", "yuv420p", s.Output)
|
||||
}
|
||||
out, e := exec.CommandContext(ctx, "ffmpeg", args...).CombinedOutput()
|
||||
if e != nil {
|
||||
return fmt.Errorf("batch frame slicing: %w: %s", e, out)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
41
internal/api/annotation_batch_decode_test.go
Normal file
41
internal/api/annotation_batch_decode_test.go
Normal file
@ -0,0 +1,41 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBatchDecoderMatchesSingleDecoderFrameContent(t *testing.T) {
|
||||
f, root := annotationProcessorFixture(t)
|
||||
_ = f
|
||||
source := filepath.Join(root, "left.mp4")
|
||||
cuts := []annotationFrameSlice{{0, 9, filepath.Join(root, "batch-a.mp4")}, {9, 24, filepath.Join(root, "batch-b.mp4")}, {24, 32, filepath.Join(root, "batch-c.mp4")}}
|
||||
if e := writeFrameAccurateSlices(context.Background(), source, cuts); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
frames := func(path string) string {
|
||||
out, e := exec.Command("ffmpeg", "-v", "error", "-i", path, "-f", "framemd5", "-").Output()
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
lines := []string{}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
if line != "" && !strings.HasPrefix(line, "#") {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
for i, c := range cuts {
|
||||
single := filepath.Join(root, []string{"single-a.mp4", "single-b.mp4", "single-c.mp4"}[i])
|
||||
if e := writeFrameAccurateSlice(context.Background(), source, single, c.Start, c.End); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if frames(single) != frames(c.Output) {
|
||||
t.Fatalf("batch changed decoded frame content for %d", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -74,8 +74,12 @@ func (s *server) retainCurrentAnnotationData(ctx context.Context, task domain.An
|
||||
if pathErr != nil && !os.IsNotExist(pathErr) {
|
||||
return pathErr
|
||||
}
|
||||
if pathErr == nil && resolved != dir {
|
||||
return fmt.Errorf("refusing symlinked annotation directory")
|
||||
if pathErr == nil {
|
||||
rootReal, rootErr := filepath.EvalSymlinks(s.workspaceMountPath)
|
||||
relative, relErr := filepath.Rel(s.workspaceMountPath, dir)
|
||||
if rootErr != nil || relErr != nil || resolved != filepath.Join(rootReal, relative) {
|
||||
return fmt.Errorf("refusing symlinked annotation directory")
|
||||
}
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return err
|
||||
|
||||
282
internal/api/annotation_metadata_revision.go
Normal file
282
internal/api/annotation_metadata_revision.go
Normal file
@ -0,0 +1,282 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func readVerifiedAnnotationJSON(path, sha string) (map[string]any, error) {
|
||||
b, e := os.ReadFile(path)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
sum := sha256.Sum256(b)
|
||||
if sha == "" || hex.EncodeToString(sum[:]) != sha {
|
||||
return nil, fmt.Errorf("annotation manifest checksum mismatch")
|
||||
}
|
||||
var m map[string]any
|
||||
e = json.Unmarshal(b, &m)
|
||||
return m, e
|
||||
}
|
||||
|
||||
// No source IMU/PTS reads, video decoding or application-side media transfer
|
||||
// on the BOS fast path. New generations own independent server-side copies.
|
||||
func (s *server) tryAnnotationMetadataRevision(ctx context.Context, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, run *domain.RunRecord, payload combinedAnnotationSubmissionInput, revision int, fingerprint string) (*humanSliceMaterialization, error) {
|
||||
prefix := fmt.Sprintf("%s-%s-%s-materialized-revision-", task.RunID, task.NodeID, task.ID)
|
||||
var prior *domain.ExecutionArtifact
|
||||
highest := 0
|
||||
slices := append([]domain.SliceSegment(nil), payload.SliceSegments...)
|
||||
sort.Slice(slices, func(i, j int) bool { return slices[i].Start < slices[j].Start })
|
||||
for i := range run.Artifacts {
|
||||
a := &run.Artifacts[i]
|
||||
n, e := strconv.Atoi(strings.TrimPrefix(a.ID, prefix))
|
||||
if e != nil || !strings.HasPrefix(a.ID, prefix) || n >= revision || n <= highest || stringValue(a.Data["sourceFingerprint"]) != fingerprint {
|
||||
continue
|
||||
}
|
||||
old := decodeMaterializationValue[domain.SliceSegment](a.Data["normalizedSlices"])
|
||||
if len(old) != len(slices) || len(old) == 0 {
|
||||
continue
|
||||
}
|
||||
same := true
|
||||
for j := range old {
|
||||
if math.Abs(old[j].Start-slices[j].Start) > 1e-9 || math.Abs(old[j].End-slices[j].End) > 1e-9 {
|
||||
same = false
|
||||
}
|
||||
}
|
||||
if same {
|
||||
prior = a
|
||||
highest = n
|
||||
}
|
||||
}
|
||||
if prior == nil || prior.Metadata == nil {
|
||||
return nil, nil
|
||||
}
|
||||
base := filepath.Join(s.workspaceMountPath, "system", "runs", task.RunID, "human-materializations", item.ID, task.ID)
|
||||
if prior.Path != filepath.Join(base, fmt.Sprintf("revision-%06d", highest)) {
|
||||
return nil, nil
|
||||
}
|
||||
manifest, e := readVerifiedAnnotationJSON(filepath.Join(prior.Path, "revision-manifest.json"), prior.Metadata.SHA256)
|
||||
if e != nil {
|
||||
return nil, nil
|
||||
}
|
||||
entries := listValue(manifest["items"])
|
||||
if len(entries) != len(slices) {
|
||||
return nil, nil
|
||||
}
|
||||
timeline, e := s.annotationFrameTimelineForTask(ctx, task, "left")
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if stringValue(timeline["sourceFingerprint"]) != fingerprint {
|
||||
return nil, fmt.Errorf("source changed during annotation update")
|
||||
}
|
||||
grid := annotationBoundaryGrid{Seconds: timeline["boundariesSeconds"].([]float64), Timestamps: timeline["boundaryTimestampUs"].([]int64)}
|
||||
segments := append([]domain.AnnotationSegment{}, payload.Segments...)
|
||||
for i := range segments {
|
||||
a, e := grid.resolve(segments[i].Start)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
b, e := grid.resolve(segments[i].End)
|
||||
if e != nil || b <= a {
|
||||
return nil, fmt.Errorf("invalid annotation boundary")
|
||||
}
|
||||
segments[i].Start, segments[i].End = grid.Seconds[a], grid.Seconds[b]
|
||||
segments[i].StartTimestampUS, segments[i].EndTimestampUS = grid.Timestamps[a], grid.Timestamps[b]
|
||||
}
|
||||
if issues := validateDescriptionIntervals(slices, segments, task.TemplateVersion < 6); len(issues) > 0 {
|
||||
return nil, fmt.Errorf("%s", issues[0].Message)
|
||||
}
|
||||
stage, e := os.MkdirTemp("", "annotation-metadata-")
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer os.RemoveAll(stage)
|
||||
destination := filepath.Join(base, fmt.Sprintf("revision-%06d", revision))
|
||||
kept, discarded := 0, 0
|
||||
keptDuration, discardedDuration := 0.0, 0.0
|
||||
for i, raw := range entries {
|
||||
entry := mapValue(raw)
|
||||
entries[i] = entry
|
||||
slice := &slices[i]
|
||||
oldDir := stringValue(entry["directory"])
|
||||
if oldDir != filepath.ToSlash(filepath.Join("slices", fmt.Sprintf("slice-%06d", i+1))) {
|
||||
return nil, nil
|
||||
}
|
||||
files := mapValue(entry["files"])
|
||||
entry["files"] = files
|
||||
syncFile := mapValue(files["synchronization"])
|
||||
synchronized, e := readVerifiedAnnotationJSON(filepath.Join(prior.Path, oldDir, "synchronized-slice.json"), stringValue(syncFile["sha256"]))
|
||||
if e != nil {
|
||||
return nil, nil
|
||||
}
|
||||
slice.StartTimestampUS = int64Value(entry["startTimestampUs"])
|
||||
slice.EndTimestampUS = int64Value(entry["endTimestampUs"])
|
||||
if slice.IsDiscarded() {
|
||||
discarded++
|
||||
discardedDuration += slice.End - slice.Start
|
||||
} else {
|
||||
kept++
|
||||
keptDuration += slice.End - slice.Start
|
||||
}
|
||||
entry["sourceSliceId"], entry["sliceName"], entry["deliveryDisposition"] = slice.ID, slice.Name, sliceDisposition(*slice)
|
||||
synchronized["sliceName"], synchronized["deliveryDisposition"] = slice.Name, sliceDisposition(*slice)
|
||||
delete(entry, "discardEvidence")
|
||||
delete(synchronized, "discardEvidence")
|
||||
if slice.IsDiscarded() {
|
||||
evidence := map[string]any{"reason": slice.DiscardReason, "at": slice.DiscardedAt, "previousAnnotations": slice.DiscardedAnnotations}
|
||||
entry["discardEvidence"] = evidence
|
||||
synchronized["discardEvidence"] = evidence
|
||||
}
|
||||
actions := []map[string]any{}
|
||||
for _, segment := range segments {
|
||||
if slice.IsDiscarded() || segment.End <= slice.Start || segment.Start >= slice.End {
|
||||
continue
|
||||
}
|
||||
actions = append(actions, map[string]any{"id": segment.ID, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": (segment.Start - slice.Start) * 1000, "clipEndMs": (segment.End - slice.Start) * 1000, "startTimestampUs": segment.StartTimestampUS, "endTimestampUs": segment.EndTimestampUS})
|
||||
}
|
||||
annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": entry["sliceId"], "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions}
|
||||
if e = os.MkdirAll(filepath.Join(stage, oldDir), 0755); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
for name, value := range map[string]map[string]any{"annotation": annotation, "synchronization": synchronized} {
|
||||
filename := "annotation.json"
|
||||
if name == "synchronization" {
|
||||
filename = "synchronized-slice.json"
|
||||
}
|
||||
sha, size, e := writeJSONFile(filepath.Join(stage, oldDir, filename), value)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
files[name] = map[string]any{"name": filename, "sha256": sha, "bytes": size}
|
||||
}
|
||||
}
|
||||
// Verify all old metadata before copying any members. A legacy generation
|
||||
// without BOS digest/ETag metadata safely falls back to normal generation.
|
||||
var copiedVersions sync.Map
|
||||
preflight, pctx := errgroup.WithContext(ctx)
|
||||
preflight.SetLimit(4)
|
||||
for _, raw := range entries {
|
||||
entry := mapValue(raw)
|
||||
dir := stringValue(entry["directory"])
|
||||
files := mapValue(entry["files"])
|
||||
for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
|
||||
file := mapValue(files[kind])
|
||||
name := stringValue(file["name"])
|
||||
if name == "" || filepath.Base(name) != name {
|
||||
return nil, nil
|
||||
}
|
||||
preflight.Go(func() error {
|
||||
return s.annotationMediaCopyReady(pctx, task.OrganizationID, filepath.Join(prior.Path, dir, name), file)
|
||||
})
|
||||
}
|
||||
}
|
||||
if e = preflight.Wait(); e != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
group, gctx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(4)
|
||||
for _, raw := range entries {
|
||||
entry := mapValue(raw)
|
||||
dir := stringValue(entry["directory"])
|
||||
files := mapValue(entry["files"])
|
||||
for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
|
||||
file := mapValue(files[kind])
|
||||
name := stringValue(file["name"])
|
||||
if name == "" || filepath.Base(name) != name {
|
||||
return nil, nil
|
||||
}
|
||||
group.Go(func() error {
|
||||
target := filepath.Join(destination, dir, name)
|
||||
meta, e := s.copyAnnotationObject(gctx, task.OrganizationID, filepath.Join(prior.Path, dir, name), target, file)
|
||||
if e == nil {
|
||||
copiedVersions.Store(target, meta.ETag)
|
||||
}
|
||||
return e
|
||||
})
|
||||
}
|
||||
}
|
||||
if e = group.Wait(); e != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
log.Printf("[api] metadata reuse fallback task=%s: %v", task.ID, e)
|
||||
return nil, nil
|
||||
}
|
||||
// New copy ETags may differ for multipart/server-side copies. Pin them in
|
||||
// the next manifest, without downloading media or parsing the original IMU.
|
||||
for i, raw := range entries {
|
||||
entry := mapValue(raw)
|
||||
entries[i] = entry
|
||||
files := mapValue(entry["files"])
|
||||
entry["files"] = files
|
||||
dir := stringValue(entry["directory"])
|
||||
for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
|
||||
file := mapValue(files[kind])
|
||||
files[kind] = file
|
||||
if tag, ok := copiedVersions.Load(filepath.Join(destination, dir, stringValue(file["name"]))); ok && tag.(string) != "" {
|
||||
file["etag"] = tag.(string)
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{"annotation", "synchronization"} {
|
||||
file := mapValue(files[kind])
|
||||
files[kind] = file
|
||||
name := stringValue(file["name"])
|
||||
meta, e := s.publishAnnotationFile(ctx, task.OrganizationID, filepath.Join(stage, dir, name), filepath.Join(destination, dir, name), stringValue(file["sha256"]))
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if meta.ETag != "" {
|
||||
file["etag"] = meta.ETag
|
||||
}
|
||||
}
|
||||
}
|
||||
selection := map[string]any{"keptSliceCount": kept, "discardedSliceCount": discarded, "keptDurationSeconds": keptDuration, "discardedDurationSeconds": discardedDuration}
|
||||
manifest["items"], manifest["revision"], manifest["schemaVersion"] = entries, revision, "droboticflow.slice-materialization/v2"
|
||||
delete(manifest, "deliverySelection")
|
||||
if discarded > 0 {
|
||||
manifest["schemaVersion"] = "droboticflow.slice-materialization/v3"
|
||||
manifest["deliverySelection"] = selection
|
||||
}
|
||||
sha, size, e := writeJSONFile(filepath.Join(stage, "revision-manifest.json"), manifest)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if e = s.assertAnnotationLease(ctx); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if _, e = s.publishAnnotationFile(ctx, task.OrganizationID, filepath.Join(stage, "revision-manifest.json"), filepath.Join(destination, "revision-manifest.json"), sha); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
artifact := *prior
|
||||
artifact.ID = prefix + fmt.Sprintf("%06d", revision)
|
||||
artifact.Name = fmt.Sprintf("revision-%06d.directory.index", revision)
|
||||
artifact.Path = destination
|
||||
artifact.SizeBytes = size
|
||||
artifact.Metadata = &domain.ArtifactMetadata{SHA256: sha, MimeType: "application/json"}
|
||||
artifact.Data = mapValue(prior.Data)
|
||||
artifact.Data["rootPath"], artifact.Data["manifestPath"], artifact.Data["revision"] = destination, filepath.Join(destination, "revision-manifest.json"), revision
|
||||
artifact.Data["normalizedSlices"], artifact.Data["normalizedSegments"], artifact.Data["deliverySelection"], artifact.Data["reusedSliceCount"], artifact.Data["metadataOnly"] = slices, segments, selection, len(slices), true
|
||||
if e = s.storeAnnotationMaterializationArtifact(ctx, task, run, artifact); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
coverage := mapValue(manifest["coverage"])
|
||||
log.Printf("[api] metadata-only revision task=%s revision=%d slices=%d", task.ID, revision, len(slices))
|
||||
return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(slices), "coverage": coverage, "deliverySelection": selection, "runId": task.RunID, "artifactId": artifact.ID, "manifestSha256": sha, "metadataOnly": true}, Slices: slices, Segments: segments, CoverageStartSecond: floatValue(coverage["startSecond"]), CoverageEndSecond: floatValue(coverage["endSecond"])}, nil
|
||||
}
|
||||
@ -1,13 +1,11 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
@ -629,6 +627,10 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques
|
||||
}
|
||||
// Polling must not load histories, enrich media, recover state, or rebuild
|
||||
// timelines. A transition/interrupted job asks the client to fetch detail once.
|
||||
if request.URL.Query().Get("view") == "summary" {
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"task": item})
|
||||
return
|
||||
}
|
||||
if request.URL.Query().Get("view") == "status" {
|
||||
status := map[string]any{"taskId": item.ID, "taskStatus": item.Status}
|
||||
if progress, ok := server.materializationProgress.Load(item.OrganizationID + ":" + item.ID); ok {
|
||||
@ -650,6 +652,15 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques
|
||||
status["needsRefresh"] = true
|
||||
}
|
||||
}
|
||||
if j := linkedHumanItem.ProcessingJob; j != nil {
|
||||
status["progress"] = map[string]any{"stage": j.Stage, "completedSlices": j.CompletedSlices, "totalSlices": j.TotalSlices, "startedAt": j.CreatedAt, "attempt": j.Attempt}
|
||||
if durableAnnotationActive(linkedHumanItem) {
|
||||
status["needsRefresh"] = false
|
||||
}
|
||||
if j.Status == "failed" {
|
||||
status["error"] = j.Error
|
||||
}
|
||||
}
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, status)
|
||||
return
|
||||
@ -666,7 +677,7 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques
|
||||
if linkedHumanItem.PendingCombinedSubmissionID != "" {
|
||||
jobKey := linkedHumanItem.OrganizationID + ":" + linkedHumanItem.PendingCombinedSubmissionID
|
||||
_, jobActive := server.combinedSubmissionJobs.Load(jobKey)
|
||||
if !jobActive {
|
||||
if !jobActive && !durableAnnotationActive(linkedHumanItem) {
|
||||
revisions, _ := server.store.ListAnnotationRevisions(request.Context(), item.OrganizationID, item.ID)
|
||||
submissionRevisionFound := false
|
||||
submissionCreatedAt := ""
|
||||
@ -884,7 +895,7 @@ func (server *server) buildAnnotationFrameTimeline(ctx context.Context, task dom
|
||||
for i, f := range grid.RightFrames {
|
||||
rightSeconds[i] = float64(f) * rightDuration / float64(len(rightRows))
|
||||
}
|
||||
return map[string]any{"camera": camera, "frameCount": len(rows), "originTimestampUs": grid.Origin, "coverageEndTimestampUs": end, "boundariesSeconds": grid.Seconds, "rightBoundariesSeconds": rightSeconds, "boundaryTimestampUs": grid.Timestamps, "leftFrameIndices": grid.LeftFrames, "rightFrameIndices": grid.RightFrames, "rangeStartSecond": grid.Seconds[0], "rangeEndSecond": grid.Seconds[len(grid.Seconds)-1], "boundaryContract": "hardware-frame-v1", "ptsToMediaScale": grid.Scale}, nil
|
||||
return map[string]any{"camera": camera, "frameCount": len(rows), "leftFrameCount": lc, "rightFrameCount": rc, "originTimestampUs": grid.Origin, "coverageEndTimestampUs": end, "boundariesSeconds": grid.Seconds, "rightBoundariesSeconds": rightSeconds, "boundaryTimestampUs": grid.Timestamps, "leftFrameIndices": grid.LeftFrames, "rightFrameIndices": grid.RightFrames, "rangeStartSecond": grid.Seconds[0], "rangeEndSecond": grid.Seconds[len(grid.Seconds)-1], "boundaryContract": "hardware-frame-v1", "ptsToMediaScale": grid.Scale}, nil
|
||||
}
|
||||
|
||||
func (server *server) handleAnnotationTaskFrameTimelineGet(writer http.ResponseWriter, request *http.Request) {
|
||||
@ -1300,7 +1311,7 @@ func validateDraftStatuses(statuses map[string]domain.SliceAnnotationStatus, sli
|
||||
// reserveFanyuV5CombinedSubmission wins a CAS before either revision write.
|
||||
// A retry with the same payload token can resume; a different same-version
|
||||
// submission cannot overwrite partial or completed revision records.
|
||||
func (server *server) reserveFanyuV5CombinedSubmission(ctx context.Context, item domain.HumanItemRecord, expectedVersion int, submissionID string, payloadSHA string) (*domain.HumanItemRecord, bool, error) {
|
||||
func (server *server) reserveFanyuV5CombinedSubmission(ctx context.Context, item domain.HumanItemRecord, expectedVersion int, submissionID string, payloadSHA string, jobs ...*domain.AnnotationProcessingJob) (*domain.HumanItemRecord, bool, error) {
|
||||
if item.PendingCombinedSubmissionID != "" {
|
||||
// Older clients could keep confirming descriptions after a submission
|
||||
// reserved its version. That writer can no longer finish its CAS. Only
|
||||
@ -1318,6 +1329,20 @@ func (server *server) reserveFanyuV5CombinedSubmission(ctx context.Context, item
|
||||
item.PendingCombinedPayloadSHA = ""
|
||||
} else {
|
||||
if item.PendingCombinedPayloadSHA == payloadSHA {
|
||||
if len(jobs) > 0 && jobs[0] != nil && (item.ProcessingJob == nil || item.ProcessingJob.ID != item.PendingCombinedSubmissionID) {
|
||||
adopted := item
|
||||
job := *jobs[0]
|
||||
job.ID = item.PendingCombinedSubmissionID
|
||||
adopted.ProcessingJob = &job
|
||||
ok, e := server.store.UpdateHumanItemVersioned(ctx, adopted, item.Version)
|
||||
if e != nil {
|
||||
return nil, false, e
|
||||
}
|
||||
if !ok {
|
||||
return nil, false, fmt.Errorf("combined submission adoption conflict")
|
||||
}
|
||||
return &adopted, true, nil
|
||||
}
|
||||
return &item, true, nil
|
||||
}
|
||||
return nil, false, fmt.Errorf("a different combined submission is already reserved")
|
||||
@ -1328,6 +1353,17 @@ func (server *server) reserveFanyuV5CombinedSubmission(ctx context.Context, item
|
||||
reserved.Version++
|
||||
reserved.PendingCombinedSubmissionID = submissionID
|
||||
reserved.PendingCombinedPayloadSHA = payloadSHA
|
||||
if len(jobs) > 0 && jobs[0] != nil {
|
||||
reserved.ProcessingJob = jobs[0]
|
||||
if jobs[0].Kind == "submit" {
|
||||
reserved.Subject.Payload = mapValue(item.Subject.Payload)
|
||||
if reserved.Subject.Payload == nil {
|
||||
reserved.Subject.Payload = map[string]any{}
|
||||
}
|
||||
in := jobs[0].Input
|
||||
reserved.Subject.Payload["annotationWorkspace"] = map[string]any{"sliceSegments": in["sliceSegments"], "operations": in["sliceOperations"], "segments": in["segments"], "userId": jobs[0].ActorID, "savedAt": nowISO()}
|
||||
}
|
||||
}
|
||||
updated, err := server.store.UpdateHumanItemVersioned(ctx, reserved, item.Version)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
@ -1583,7 +1619,11 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
|
||||
}
|
||||
done := make(chan struct{})
|
||||
heartbeatStopped := make(chan struct{})
|
||||
go func() { defer close(heartbeatStopped); server.keepSliceLease(ctx, cancel, itemID, requestID, done) }()
|
||||
if _, durable := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease); durable {
|
||||
close(heartbeatStopped)
|
||||
} else {
|
||||
go func() { defer close(heartbeatStopped); server.keepSliceLease(ctx, cancel, itemID, requestID, done) }()
|
||||
}
|
||||
slices := descriptionOnlySlices(payload.SliceSegments)
|
||||
sliceRevision := domain.SliceRevisionRecord{
|
||||
ID: "slice-confirmation-" + requestID, OrganizationID: task.OrganizationID, TaskID: task.ID,
|
||||
@ -1593,6 +1633,9 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
|
||||
materialization, processErr := server.materializeCombinedSliceSubmission(ctx, *task, *item, combinedAnnotationSubmissionInput{SliceSegments: slices, Segments: []domain.AnnotationSegment{}}, revision)
|
||||
close(done)
|
||||
<-heartbeatStopped
|
||||
if _, durable := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease); durable && (ctx.Err() != nil || server.assertAnnotationLease(ctx) != nil) {
|
||||
return
|
||||
}
|
||||
// Persist failure even when ffmpeg was cancelled by the deadline/lease.
|
||||
ctx, cleanup := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cleanup()
|
||||
@ -1600,6 +1643,9 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
|
||||
if currentErr != nil || current == nil {
|
||||
return
|
||||
}
|
||||
if lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease); ok && (current.ProcessingJob == nil || current.ProcessingJob.Owner != lease.Owner) {
|
||||
return
|
||||
}
|
||||
workflow := mapValue(current.Subject.Payload["sliceWorkflow"])
|
||||
if strings.TrimSpace(fmt.Sprint(workflow["requestId"])) != requestID || stringValue(workflow["status"]) != "processing" {
|
||||
return
|
||||
@ -1744,6 +1790,14 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter,
|
||||
"segments": []domain.AnnotationSegment{}, "statuses": pendingSliceStatuses(payload.SliceSegments),
|
||||
"userId": principal.User.ID, "savedAt": now,
|
||||
}
|
||||
if server.externalAnnotationProcessor {
|
||||
job, err := server.newAnnotationProcessingJob(request.Context(), *task, principal.User.ID, requestID, "slice", payload, revision)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusUnprocessableEntity, err)
|
||||
return
|
||||
}
|
||||
item.ProcessingJob = job
|
||||
}
|
||||
expectedVersion := item.Version
|
||||
item.Version++
|
||||
item.UpdatedAt = now
|
||||
@ -1757,7 +1811,9 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
_ = server.appendHumanEvent(request.Context(), *item, "slice_confirmation_started", principal.User.ID, map[string]any{"requestId": requestID, "revision": revision, "sliceCount": len(payload.SliceSegments)})
|
||||
go server.finishSliceConfirmation(context.WithoutCancel(request.Context()), task.ID, item.ID, requestID, payload, revision, principal.User.ID)
|
||||
if !server.externalAnnotationProcessor {
|
||||
go server.finishSliceConfirmation(context.WithoutCancel(request.Context()), task.ID, item.ID, requestID, payload, revision, principal.User.ID)
|
||||
}
|
||||
renderJSON(writer, http.StatusAccepted, map[string]any{"item": item, "sliceWorkflow": item.Subject.Payload["sliceWorkflow"]})
|
||||
}
|
||||
|
||||
@ -1832,6 +1888,11 @@ func (server *server) handleSliceResetCreate(writer http.ResponseWriter, request
|
||||
}
|
||||
delete(item.Subject.Payload, "annotationDraft")
|
||||
item.Subject.Payload["annotationWorkspace"] = resetDraft
|
||||
if item.ProcessingJob != nil {
|
||||
item.ProcessingJob.Status = "cancelled"
|
||||
item.ProcessingJob.Input = nil
|
||||
item.ProcessingJob.Owner = ""
|
||||
}
|
||||
item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "editing", "phase": "slice", "resetAt": now, "revision": intValue(mapValue(item.Subject.Payload["sliceWorkflow"])["revision"])}
|
||||
expectedVersion := item.Version
|
||||
item.Version++
|
||||
@ -1935,7 +1996,15 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
|
||||
return
|
||||
}
|
||||
submissionRef := fanyuV5CombinedSubmissionID(humanItem.ID, payload.Version)
|
||||
reservedItem, resumed, reserveErr := server.reserveFanyuV5CombinedSubmission(request.Context(), *humanItem, payload.Version, submissionRef, payloadSHA)
|
||||
var processingJob *domain.AnnotationProcessingJob
|
||||
if server.externalAnnotationProcessor && strings.Contains(strings.ToLower(request.Header.Get("Prefer")), "respond-async") {
|
||||
processingJob, err = server.newAnnotationProcessingJob(request.Context(), *task, principal.User.ID, submissionRef, "submit", payload, 0)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusUnprocessableEntity, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
reservedItem, resumed, reserveErr := server.reserveFanyuV5CombinedSubmission(request.Context(), *humanItem, payload.Version, submissionRef, payloadSHA, processingJob)
|
||||
if reserveErr != nil || reservedItem == nil {
|
||||
renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"})
|
||||
return
|
||||
@ -1944,6 +2013,33 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
|
||||
if resumed {
|
||||
submissionRef = humanItem.PendingCombinedSubmissionID
|
||||
}
|
||||
if strings.Contains(strings.ToLower(request.Header.Get("Prefer")), "respond-async") {
|
||||
if !server.externalAnnotationProcessor {
|
||||
jobKey := humanItem.OrganizationID + ":" + submissionRef
|
||||
if _, loaded := server.combinedSubmissionJobs.LoadOrStore(jobKey, struct{}{}); !loaded {
|
||||
backgroundRequest := request.Clone(context.WithoutCancel(request.Context()))
|
||||
go func() {
|
||||
defer server.combinedSubmissionJobs.Delete(jobKey)
|
||||
freshTask, te := server.store.GetAnnotationTask(backgroundRequest.Context(), task.ID)
|
||||
freshItem, he := server.store.GetHumanItem(backgroundRequest.Context(), humanItem.ID)
|
||||
if te != nil || he != nil || freshTask == nil || freshItem == nil || freshItem.PendingCombinedSubmissionID != submissionRef {
|
||||
return
|
||||
}
|
||||
server.completeCombinedAnnotationSubmission(&asynchronousAnnotationResponseWriter{}, backgroundRequest, freshTask, freshItem, payload, principal, submissionRef, resumed)
|
||||
}()
|
||||
}
|
||||
}
|
||||
renderJSON(writer, http.StatusAccepted, map[string]any{"item": humanItem, "submissionId": submissionRef, "status": "processing", "idempotent": resumed})
|
||||
return
|
||||
}
|
||||
server.completeCombinedAnnotationSubmission(writer, request, task, humanItem, payload, principal, submissionRef, resumed)
|
||||
}
|
||||
|
||||
func (server *server) completeCombinedAnnotationSubmission(writer http.ResponseWriter, request *http.Request, task *domain.AnnotationTaskRecord, humanItem *domain.HumanItemRecord, payload combinedAnnotationSubmissionInput, principal *authenticatedPrincipal, submissionRef string, resumed bool) {
|
||||
if err := server.assertAnnotationLease(request.Context()); err != nil {
|
||||
renderError(writer, http.StatusConflict, err)
|
||||
return
|
||||
}
|
||||
sliceRevisions, sliceErr := server.store.ListSliceRevisions(request.Context(), task.OrganizationID, task.ID)
|
||||
annotationRevisions, annotationErr := server.store.ListAnnotationRevisions(request.Context(), task.OrganizationID, task.ID)
|
||||
reviews, reviewErr := server.store.ListAnnotationReviews(request.Context(), task.OrganizationID, task.ID)
|
||||
@ -1983,35 +2079,11 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
|
||||
now := nowISO()
|
||||
sliceRevision := domain.SliceRevisionRecord{ID: sliceID, OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: revisionNumber, Segments: payload.SliceSegments, Operations: payload.SliceOperations, AuthorUserID: principal.User.ID, ChangeReason: submissionRef + ":" + strings.TrimSpace(payload.ChangeReason), DurationSeconds: payload.DurationSeconds, CreatedAt: now}
|
||||
annotationRevision := domain.AnnotationRevisionRecord{ID: annotationID, OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: revisionNumber, TemplateVersion: task.TemplateVersion, Segments: payload.Segments, ObjectHints: payload.ObjectHints, AuthorUserID: principal.User.ID, ChangeReason: submissionRef + ":" + strings.TrimSpace(payload.ChangeReason), IsRework: isRework, CountsAsOutput: !isRework, DurationSeconds: payload.DurationSeconds, CreatedAt: now}
|
||||
if strings.Contains(strings.ToLower(request.Header.Get("Prefer")), "respond-async") {
|
||||
jobKey := humanItem.OrganizationID + ":" + submissionRef
|
||||
if _, loaded := server.combinedSubmissionJobs.LoadOrStore(jobKey, struct{}{}); !loaded {
|
||||
payloadBytes, marshalErr := json.Marshal(payload)
|
||||
if marshalErr != nil {
|
||||
server.combinedSubmissionJobs.Delete(jobKey)
|
||||
renderError(writer, http.StatusInternalServerError, marshalErr)
|
||||
return
|
||||
}
|
||||
backgroundRequest := request.Clone(context.WithoutCancel(request.Context()))
|
||||
backgroundRequest.Header = request.Header.Clone()
|
||||
backgroundRequest.Header.Del("Prefer")
|
||||
backgroundRequest.Body = io.NopCloser(bytes.NewReader(payloadBytes))
|
||||
backgroundRequest.ContentLength = int64(len(payloadBytes))
|
||||
go func() {
|
||||
defer server.combinedSubmissionJobs.Delete(jobKey)
|
||||
backgroundWriter := &asynchronousAnnotationResponseWriter{}
|
||||
server.handleCombinedAnnotationSubmission(backgroundWriter, backgroundRequest)
|
||||
if backgroundWriter.status >= http.StatusBadRequest {
|
||||
fmt.Printf("[api] asynchronous combined submission %s failed status=%d body=%s\n", submissionRef, backgroundWriter.status, strings.TrimSpace(backgroundWriter.body.String()))
|
||||
}
|
||||
}()
|
||||
}
|
||||
renderJSON(writer, http.StatusAccepted, map[string]any{
|
||||
"item": humanItem, "submissionId": submissionRef, "status": "processing", "idempotent": resumed,
|
||||
})
|
||||
materialization, materializationErr := server.materializeCombinedSliceSubmission(request.Context(), *task, *humanItem, payload, revisionNumber)
|
||||
if request.Context().Err() != nil {
|
||||
renderError(writer, http.StatusConflict, request.Context().Err())
|
||||
return
|
||||
}
|
||||
materialization, materializationErr := server.materializeCombinedSliceSubmission(request.Context(), *task, *humanItem, payload, revisionNumber)
|
||||
if materializationErr != nil {
|
||||
previousVersion := humanItem.Version
|
||||
humanItem.PendingCombinedSubmissionID = ""
|
||||
@ -2042,6 +2114,10 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
|
||||
return
|
||||
}
|
||||
previousVersion := humanItem.Version
|
||||
if err := server.assertAnnotationLease(request.Context()); err != nil {
|
||||
renderError(writer, http.StatusConflict, err)
|
||||
return
|
||||
}
|
||||
humanItem.CurrentRevision = annotationRevision.Revision
|
||||
humanItem.CurrentStageID = "quality_review"
|
||||
humanItem.Status = domain.HumanItemReady
|
||||
|
||||
@ -220,19 +220,29 @@ func (s *server) annotationFrameTimelineForTask(ctx context.Context, task domain
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.annotationTimelines.get(ctx, key+":"+camera, func(ctx context.Context) (map[string]any, error) {
|
||||
value, err := s.buildAnnotationFrameTimeline(ctx, task, camera)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
after, err := s.annotationSourceFingerprint(ctx, task, *run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if after != key {
|
||||
return nil, fmt.Errorf("源文件在校验期间发生变化,请重试")
|
||||
}
|
||||
value["sourceFingerprint"] = key
|
||||
return value, nil
|
||||
value, err := s.annotationTimelines.get(ctx, key, func(ctx context.Context) (map[string]any, error) {
|
||||
return s.persistentTimeline(ctx, task, key, func(ctx context.Context) (map[string]any, error) {
|
||||
value, err := s.buildAnnotationFrameTimeline(ctx, task, "left")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
after, err := s.annotationSourceFingerprint(ctx, task, *run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if after != key {
|
||||
return nil, fmt.Errorf("源文件在校验期间发生变化,请重试")
|
||||
}
|
||||
value["sourceFingerprint"] = key
|
||||
return value, nil
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value["camera"] = camera
|
||||
if camera == "right" {
|
||||
value["frameCount"] = value["rightFrameCount"]
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
188
internal/api/annotation_performance_live_test.go
Normal file
188
internal/api/annotation_performance_live_test.go
Normal file
@ -0,0 +1,188 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Opt-in seeding only. Original production task/files remain read-only; every
|
||||
// write goes to a uniquely named sandbox database and private test Run prefixes.
|
||||
func TestSeedAnnotationPerformanceSandbox(t *testing.T) {
|
||||
if os.Getenv("FANYU_PERF_SEED") != "1" {
|
||||
t.Skip("explicit isolated performance fixture")
|
||||
}
|
||||
nonce := os.Getenv("FANYU_PERF_NONCE")
|
||||
if !regexp.MustCompile(`^2026[0-9a-z]{8,24}$`).MatchString(nonce) {
|
||||
t.Fatal("invalid nonce")
|
||||
}
|
||||
name := "annotation_perf_" + nonce
|
||||
if name == os.Getenv("MONGO_DB") {
|
||||
t.Fatal("sandbox must differ from source")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
client, e := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")))
|
||||
if e != nil {
|
||||
t.Fatal("mongo connect")
|
||||
}
|
||||
defer client.Disconnect(ctx)
|
||||
source := store.NewMongoStore(client.Database(os.Getenv("MONGO_DB")))
|
||||
target := store.NewMongoStore(client.Database(name))
|
||||
mode := os.Getenv("FANYU_PERF_MODE")
|
||||
if mode == "disable-users" {
|
||||
users, e := target.ListUsers(ctx)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, u := range users {
|
||||
if !regexp.MustCompile(`^perf-` + nonce + `-`).MatchString(u.ID) {
|
||||
continue
|
||||
}
|
||||
u.Status = domain.UserStatusDisabled
|
||||
if e = target.UpsertUser(ctx, u); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
fmt.Println("disabled only isolated performance accounts")
|
||||
return
|
||||
}
|
||||
collections, e := client.Database(name).ListCollectionNames(ctx, bson.M{})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(collections) > 0 {
|
||||
t.Fatal("sandbox already exists; use a fresh nonce")
|
||||
}
|
||||
password := os.Getenv("FANYU_PERF_PASSWORD")
|
||||
if len(password) < 24 {
|
||||
t.Fatal("ephemeral password required")
|
||||
}
|
||||
count := 5
|
||||
if v, e := strconv.Atoi(os.Getenv("FANYU_PERF_USERS")); e == nil {
|
||||
count = v
|
||||
}
|
||||
if count < 2 || count > 10 {
|
||||
t.Fatal("bounded concurrency 2..10 required")
|
||||
}
|
||||
original, e := source.GetAnnotationTask(ctx, "annotation-task-7d738b9fc2bd9e9e778308029a7d6a50")
|
||||
if e != nil || original == nil || original.OrganizationID != "org-29cd7a684faba19f630f8ad5890e18bb" || original.SourceDurationSeconds < 800 {
|
||||
t.Fatal("authorized long source task unavailable")
|
||||
}
|
||||
run, e := source.GetRun(ctx, original.RunID)
|
||||
if e != nil || run == nil {
|
||||
t.Fatal("source run unavailable")
|
||||
}
|
||||
var input domain.NodeExecutionRecord
|
||||
for _, n := range run.NodeExecutions {
|
||||
if n.NodeID == original.NodeID {
|
||||
input = n
|
||||
}
|
||||
}
|
||||
org, e := source.GetOrganization(ctx, original.OrganizationID)
|
||||
if e != nil || org == nil {
|
||||
t.Fatal("source organization")
|
||||
}
|
||||
connection, e := source.GetStorageConnection(ctx, "storage-d0a49fe4b9881c324f0640ab62359603")
|
||||
if e != nil || connection == nil || connection.OrganizationID != org.ID {
|
||||
t.Fatal("workspace connection scope")
|
||||
}
|
||||
if e = target.UpsertOrganization(ctx, *org); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = target.UpsertStorageConnection(ctx, *connection); e != nil {
|
||||
t.Fatal("copy scoped storage configuration")
|
||||
}
|
||||
if e = target.EnsureIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
projectID := "project-perf-" + nonce
|
||||
workflowID := "workflow-perf-" + nonce
|
||||
supplierID := "supplier-perf-" + nonce
|
||||
must := func(e error) {
|
||||
t.Helper()
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
must(target.UpsertProject(ctx, domain.ProjectRecord{ID: projectID, OrganizationID: org.ID, Name: "长视频并发隔离验收", Description: "Isolated performance fixture " + nonce}))
|
||||
must(target.UpsertSupplier(ctx, domain.SupplierRecord{ID: supplierID, OrganizationID: org.ID, Name: "性能测试专用供应商", Status: "active"}))
|
||||
must(target.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: workflowID, OrganizationID: org.ID, ProjectID: projectID, Name: "隔离人工流程"}))
|
||||
templateID := "template-perf-" + nonce
|
||||
must(target.UpsertAnnotationTemplate(ctx, domain.AnnotationTemplateRecord{ID: templateID, OrganizationID: org.ID, ProjectID: projectID, Version: 6, Status: domain.AnnotationTemplatePublished, Labels: domain.ConvenienceStore22SemanticLabels()}))
|
||||
labelID := "slice-perf-" + nonce
|
||||
must(target.UpsertAnnotationLabel(ctx, domain.AnnotationLabelRecord{ID: labelID, OrganizationID: org.ID, Kind: "slice", Code: labelID, Name: "压力切片", Status: domain.AnnotationLabelActive}))
|
||||
must(target.UpsertProjectSliceLabels(ctx, domain.ProjectAnnotationLabelsRecord{ID: "labels-" + nonce, OrganizationID: org.ID, ProjectID: projectID, SliceLabelIDs: []string{labelID}}))
|
||||
users, tasks := []map[string]string{}, []string{}
|
||||
for i := 0; i < count; i++ {
|
||||
suffix := fmt.Sprintf("%02d", i+1)
|
||||
uid := "perf-" + nonce + "-ann-" + suffix
|
||||
email := uid + "@example.invalid"
|
||||
hash, e := hashPassword(password)
|
||||
must(e)
|
||||
must(target.UpsertUser(ctx, domain.UserRecord{ID: uid, OrganizationID: org.ID, Name: "压力标注员" + suffix, Email: email, PasswordHash: hash, Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive, SupplierID: supplierID}))
|
||||
must(target.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + uid, OrganizationID: org.ID, ProjectID: projectID, UserID: uid, SupplierID: supplierID, Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}}))
|
||||
runID := "run-perf-" + nonce + "-" + suffix
|
||||
taskID := "task-perf-" + nonce + "-" + suffix
|
||||
batchID := "batch-perf-" + nonce + "-" + suffix
|
||||
itemID := "item-perf-" + nonce + "-" + suffix
|
||||
bindings := []domain.NodeInputBinding{}
|
||||
artifacts := []domain.ExecutionArtifact{}
|
||||
leftID, rightID := "", ""
|
||||
for _, port := range []string{"fullLeftVideos", "fullRightVideos", "leftTimestamps", "rightTimestamps", "imu"} {
|
||||
a := artifactsForTarget(input, port)
|
||||
if len(a) != 1 {
|
||||
t.Fatalf("expected one source per port %s, got %d", port, len(a))
|
||||
}
|
||||
v := a[0]
|
||||
v.ID = runID + "-" + port
|
||||
v.RunID = runID
|
||||
v.NodeID = "source"
|
||||
bindings = append(bindings, domain.NodeInputBinding{TargetPortID: port, SourceNodeID: "source", SourcePortID: port, Artifacts: []domain.ExecutionArtifact{v}})
|
||||
artifacts = append(artifacts, v)
|
||||
if port == "fullLeftVideos" {
|
||||
leftID = v.ID
|
||||
}
|
||||
if port == "fullRightVideos" {
|
||||
rightID = v.ID
|
||||
}
|
||||
}
|
||||
must(target.UpsertRun(ctx, domain.RunRecord{ID: runID, OrganizationID: org.ID, ProjectID: projectID, WorkflowID: workflowID, ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.RunStatusWaitingForHuman, Mode: domain.RunModeProduction, Artifacts: artifacts, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "source", Status: domain.NodeExecutionStatusCompleted, Artifacts: artifacts}, {NodeID: "human", Status: domain.NodeExecutionStatusWaitingForHuman, InputBindings: bindings, InputArtifacts: artifacts}}}))
|
||||
must(target.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, OrganizationID: org.ID, PluginID: "annotation-qa", ProjectID: projectID, SupplierID: supplierID, WorkflowID: workflowID, RunID: runID, GateNodeID: "human", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages(), RequirePrePackageAcceptance: true, ReleasePolicy: domain.HumanReleaseAllAccepted, ItemCount: 1}))
|
||||
must(target.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: itemID, OrganizationID: org.ID, PluginID: "annotation-qa", ProjectID: projectID, SupplierID: supplierID, BatchID: batchID, WorkflowID: workflowID, RunID: runID, GateNodeID: "human", LegacyTaskID: taskID, ProcessingProfile: domain.ProcessingProfileRawHuman, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Version: 1, Subject: domain.HumanSubjectRef{ItemKey: "pressure-" + suffix, Payload: map[string]any{"captureDirectory": "真实长视频压力-" + suffix, "sliceWorkflow": map[string]any{"status": "editing", "phase": "slice"}}}}))
|
||||
copy := *original
|
||||
copy.ID = taskID
|
||||
copy.ProjectID = projectID
|
||||
copy.WorkflowID = workflowID
|
||||
copy.RunID = runID
|
||||
copy.NodeID = "human"
|
||||
copy.ArtifactID = leftID
|
||||
copy.RightArtifactID = rightID
|
||||
copy.SourceVideo = "/api/runs/" + runID + "/artifacts/" + leftID + "/content"
|
||||
copy.RightSourceVideo = "/api/runs/" + runID + "/artifacts/" + rightID + "/content"
|
||||
copy.TemplateID = templateID
|
||||
copy.TemplateVersion = 6
|
||||
copy.SupplierID = supplierID
|
||||
copy.AssignedUserID = ""
|
||||
copy.Status = domain.AnnotationTaskStatusPendingAnnotation
|
||||
copy.CurrentRevision = 0
|
||||
copy.ProcessingProfile = domain.ProcessingProfileRawHuman
|
||||
must(target.UpsertAnnotationTask(ctx, copy))
|
||||
users = append(users, map[string]string{"id": uid, "email": email})
|
||||
tasks = append(tasks, taskID)
|
||||
}
|
||||
_, e = client.Database(name).Collection("performanceScope").InsertOne(ctx, bson.M{"id": nonce, "sourceTaskId": original.ID, "projectId": projectID})
|
||||
must(e)
|
||||
out, _ := json.Marshal(map[string]any{"database": name, "nonce": nonce, "projectId": projectID, "users": users, "tasks": tasks, "sourceTaskId": original.ID, "sourceDurationSeconds": original.SourceDurationSeconds})
|
||||
fmt.Println("PERFORMANCE_FIXTURE " + string(out))
|
||||
}
|
||||
352
internal/api/annotation_processor.go
Normal file
352
internal/api/annotation_processor.go
Normal file
@ -0,0 +1,352 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type annotationJobLease struct{ ItemID, JobID, Owner string }
|
||||
type annotationJobLeaseKey struct{}
|
||||
|
||||
func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.AnnotationTaskRecord, actor, id, kind string, input any, revision int) (*domain.AnnotationProcessingJob, error) {
|
||||
run, e := s.store.GetRun(ctx, t.RunID)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if run == nil {
|
||||
return nil, fmt.Errorf("annotation source run unavailable")
|
||||
}
|
||||
fingerprint, e := s.annotationSourceFingerprint(ctx, t, *run)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
b, e := json.Marshal(input)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
var values map[string]any
|
||||
if e = json.Unmarshal(b, &values); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil
|
||||
}
|
||||
|
||||
func durableAnnotationActive(item *domain.HumanItemRecord) bool {
|
||||
if item == nil || item.ProcessingJob == nil {
|
||||
return false
|
||||
}
|
||||
j := item.ProcessingJob
|
||||
return (j.Status == "queued" || j.Status == "running") && ((j.Kind == "submit" && j.ID == item.PendingCombinedSubmissionID) || (j.Kind == "slice" && j.ID == stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["requestId"])))
|
||||
}
|
||||
|
||||
func (s *server) assertAnnotationLease(ctx context.Context) error {
|
||||
lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease)
|
||||
if !ok {
|
||||
return ctx.Err()
|
||||
}
|
||||
h, e := s.store.GetHumanItem(ctx, lease.ItemID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != lease.JobID || h.ProcessingJob.Owner != lease.Owner || h.ProcessingJob.Status != "running" {
|
||||
return fmt.Errorf("annotation processing lease lost")
|
||||
}
|
||||
if h.Status == domain.HumanItemCancelled {
|
||||
return fmt.Errorf("annotation task cancelled")
|
||||
}
|
||||
project, e := s.store.GetProject(ctx, h.ProjectID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if project == nil || project.DeletionTaskID != "" {
|
||||
return fmt.Errorf("annotation project removed or deleting")
|
||||
}
|
||||
expires, e := time.Parse(time.RFC3339Nano, h.ProcessingJob.LeaseUntil)
|
||||
if e != nil || time.Now().After(expires) {
|
||||
return fmt.Errorf("annotation processing lease expired")
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// RunAnnotationProcessor has no public mutation endpoints. Authenticated API
|
||||
// requests create fenced outboxes; this independent process performs the work.
|
||||
func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppConfig, parallel int, backfill bool) error {
|
||||
queue, ok := db.(store.AnnotationProcessingStore)
|
||||
if !ok {
|
||||
return fmt.Errorf("annotation processing store unavailable")
|
||||
}
|
||||
if parallel < 1 || parallel > 4 {
|
||||
return fmt.Errorf("annotation processor concurrency must be 1..4")
|
||||
}
|
||||
s := &server{store: db, workspaceMountPath: cfg.WorkspaceMountPath, globalBOSEndpoint: cfg.GlobalBOSEndpoint, externalAnnotationProcessor: true}
|
||||
s.materializationSlotsOnce.Do(func() { s.materializationSlots = make(chan struct{}, parallel) })
|
||||
if backfill {
|
||||
orgs, e := db.ListOrganizations(ctx)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, org := range orgs {
|
||||
tasks, e := db.ListAnnotationTasks(ctx, org.ID, "")
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if t.SourceIsFullVideo && t.TemplateVersion >= 5 && t.Status == domain.AnnotationTaskStatusPendingAnnotation {
|
||||
if e = queue.EnqueueAnnotationIndex(ctx, t); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for n := 0; n < parallel; n++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for ctx.Err() == nil {
|
||||
now := time.Now().UTC()
|
||||
item, e := queue.ClaimAnnotationProcessing(ctx, domain.NewID("annotation-lease"), now.Format(time.RFC3339Nano), now.Add(2*time.Minute).Format(time.RFC3339Nano))
|
||||
if e != nil {
|
||||
log.Printf("[annotation-processor] claim: %v", e)
|
||||
} else if item != nil {
|
||||
s.executeAnnotationJob(ctx, *item)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for ctx.Err() == nil {
|
||||
now := time.Now().UTC()
|
||||
j, e := queue.ClaimAnnotationIndex(ctx, domain.NewID("index-lease"), now.Format(time.RFC3339Nano), now.Add(20*time.Minute).Format(time.RFC3339Nano))
|
||||
if e != nil {
|
||||
log.Printf("[annotation-processor] index claim: %v", e)
|
||||
} else if j != nil {
|
||||
indexCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
task, readErr := db.GetAnnotationTask(indexCtx, j.ID)
|
||||
if readErr == nil && task != nil {
|
||||
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
|
||||
} else if task == nil {
|
||||
readErr = fmt.Errorf("annotation task no longer exists")
|
||||
}
|
||||
cancel()
|
||||
j.Status = "completed"
|
||||
j.Error = ""
|
||||
if readErr != nil {
|
||||
j.Error = readErr.Error()
|
||||
j.Status = "queued"
|
||||
j.NextAttemptAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)
|
||||
if j.Attempt >= 5 {
|
||||
j.Status = "failed"
|
||||
}
|
||||
}
|
||||
_, _ = queue.FinishAnnotationIndex(context.WithoutCancel(ctx), *j)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
}()
|
||||
wg.Wait()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (s *server) executeAnnotationJob(parent context.Context, claimed domain.HumanItemRecord) {
|
||||
queue := s.store.(store.AnnotationProcessingStore)
|
||||
j := *claimed.ProcessingJob
|
||||
ctx, cancel := context.WithTimeout(parent, 24*time.Hour)
|
||||
defer cancel()
|
||||
ctx = context.WithValue(ctx, annotationJobLeaseKey{}, annotationJobLease{claimed.ID, j.ID, j.Owner})
|
||||
done, stopped := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(stopped)
|
||||
tick := time.NewTicker(10 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
stage, completed, total := "validating", 0, j.TotalSlices
|
||||
if raw, ok := s.materializationProgress.Load(claimed.OrganizationID + ":" + j.TaskID); ok {
|
||||
p := mapValue(raw)
|
||||
stage = stringValue(p["stage"])
|
||||
completed = intValue(p["completedSlices"])
|
||||
total = intValue(p["totalSlices"])
|
||||
}
|
||||
ok, e := queue.TouchAnnotationProcessing(ctx, claimed.ID, j.ID, j.Owner, time.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), stage, completed, total)
|
||||
if e != nil || !ok {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
e := s.processAnnotationJob(ctx, claimed)
|
||||
close(done)
|
||||
<-stopped
|
||||
finishCtx, stop := context.WithTimeout(context.WithoutCancel(parent), 15*time.Second)
|
||||
defer stop()
|
||||
if parent.Err() != nil || ctx.Err() != nil {
|
||||
j.Status = "queued"
|
||||
j.Stage = "queued"
|
||||
j.Error = "processing interrupted; resuming"
|
||||
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
|
||||
return
|
||||
}
|
||||
j.Status = "completed"
|
||||
j.Stage = "completed"
|
||||
j.Input = nil
|
||||
j.CompletedSlices = j.TotalSlices
|
||||
j.UpdatedAt = nowISO()
|
||||
if e != nil {
|
||||
j.Status = "failed"
|
||||
j.Stage = "failed"
|
||||
j.Error = e.Error()
|
||||
log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e)
|
||||
}
|
||||
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
|
||||
if e != nil {
|
||||
s.failAnnotationJob(finishCtx, claimed.ID, j)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *server) processAnnotationJob(ctx context.Context, h domain.HumanItemRecord) error {
|
||||
j := h.ProcessingJob
|
||||
if j.Attempt > 3 {
|
||||
return fmt.Errorf("后台处理连续中断,已保留编辑内容,请重试")
|
||||
}
|
||||
t, e := s.store.GetAnnotationTask(ctx, j.TaskID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if t == nil || t.OrganizationID != h.OrganizationID || t.RunID != h.RunID {
|
||||
return fmt.Errorf("annotation job task scope changed")
|
||||
}
|
||||
// Repair a crash after authoritative handoff but before legacy task publication.
|
||||
if j.Kind == "submit" && h.PendingCombinedSubmissionID == "" && h.CurrentStageID == "quality_review" && h.CurrentRevision > 0 {
|
||||
t.CurrentRevision = h.CurrentRevision
|
||||
t.Status = domain.AnnotationTaskStatusPendingQuality
|
||||
t.AssignedUserID = ""
|
||||
return s.store.UpsertAnnotationTask(ctx, *t)
|
||||
}
|
||||
actor, e := s.store.GetUser(ctx, j.ActorID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if actor == nil || actor.Status != domain.UserStatusActive || actor.OrganizationID != h.OrganizationID || h.AssignedUserID != actor.ID || h.CurrentStageID != "annotation_combined" {
|
||||
return fmt.Errorf("annotation processing ownership or user status changed")
|
||||
}
|
||||
org, e := s.store.GetOrganization(ctx, h.OrganizationID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if org == nil || org.Status != domain.OrganizationStatusActive {
|
||||
return fmt.Errorf("annotation organization unavailable")
|
||||
}
|
||||
principal := &authenticatedPrincipal{User: *actor, Organization: *org}
|
||||
r, _ := http.NewRequestWithContext(context.WithValue(ctx, authContextKey{}, principal), http.MethodPost, "http://annotation-processor/internal", nil)
|
||||
w := &asynchronousAnnotationResponseWriter{}
|
||||
if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *t); !ok {
|
||||
return fmt.Errorf("annotation job no longer authorized")
|
||||
}
|
||||
if !annotationManager(principal) && !s.canAccessCurrentHumanStage(ctx, r, h) {
|
||||
return fmt.Errorf("annotation job role changed")
|
||||
}
|
||||
run, e := s.store.GetRun(ctx, t.RunID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if run == nil {
|
||||
return fmt.Errorf("annotation run unavailable")
|
||||
}
|
||||
key, e := s.annotationSourceFingerprint(ctx, *t, *run)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if key != j.SourceFingerprint {
|
||||
return fmt.Errorf("源文件已改变,已保留编辑内容,请重新确认")
|
||||
}
|
||||
b, e := json.Marshal(normalizeStructuredValue(j.Input))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
switch j.Kind {
|
||||
case "slice":
|
||||
var p sliceConfirmationInput
|
||||
if e = json.Unmarshal(b, &p); e != nil {
|
||||
return e
|
||||
}
|
||||
s.finishSliceConfirmation(r.Context(), t.ID, h.ID, j.ID, p, j.Revision, j.ActorID)
|
||||
fresh, e := s.store.GetHumanItem(ctx, h.ID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if fresh == nil {
|
||||
return fmt.Errorf("annotation task removed")
|
||||
}
|
||||
flow := mapValue(fresh.Subject.Payload["sliceWorkflow"])
|
||||
if stringValue(flow["status"]) != "completed" {
|
||||
return fmt.Errorf("%s", firstNonEmptyText(stringValue(flow["error"]), "slice processing did not complete"))
|
||||
}
|
||||
case "submit":
|
||||
if h.PendingCombinedSubmissionID != j.ID {
|
||||
return fmt.Errorf("annotation submission superseded")
|
||||
}
|
||||
var p combinedAnnotationSubmissionInput
|
||||
if e = json.Unmarshal(b, &p); e != nil {
|
||||
return e
|
||||
}
|
||||
s.completeCombinedAnnotationSubmission(w, r, t, &h, p, principal, j.ID, j.Attempt > 1)
|
||||
if w.status >= 400 {
|
||||
return fmt.Errorf("annotation submission failed: %s", w.body.String())
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown annotation job kind")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *server) failAnnotationJob(ctx context.Context, id string, j domain.AnnotationProcessingJob) {
|
||||
h, e := s.store.GetHumanItem(ctx, id)
|
||||
if e != nil || h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner {
|
||||
return
|
||||
}
|
||||
version := h.Version
|
||||
if j.Kind == "submit" && h.PendingCombinedSubmissionID == j.ID {
|
||||
h.PendingCombinedSubmissionID = ""
|
||||
h.PendingCombinedPayloadSHA = ""
|
||||
}
|
||||
if j.Kind == "slice" {
|
||||
w := mapValue(h.Subject.Payload["sliceWorkflow"])
|
||||
if stringValue(w["requestId"]) == j.ID {
|
||||
w["status"] = "failed"
|
||||
w["error"] = j.Error
|
||||
h.Subject.Payload["sliceWorkflow"] = w
|
||||
}
|
||||
}
|
||||
h.Version++
|
||||
h.UpdatedAt = nowISO()
|
||||
_, _ = s.store.UpdateHumanItemVersioned(ctx, *h, version)
|
||||
}
|
||||
141
internal/api/annotation_processor_test.go
Normal file
141
internal/api/annotation_processor_test.go
Normal file
@ -0,0 +1,141 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func annotationProcessorFixture(t *testing.T) (combinedDraftFixture, string) {
|
||||
t.Helper()
|
||||
if _, e := exec.LookPath("ffmpeg"); e != nil {
|
||||
t.Skip("ffmpeg required")
|
||||
}
|
||||
f := newCombinedDraftFixture(t)
|
||||
if e := f.store.UpsertOrganization(f.ctx, domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Name: "Processor test", Status: domain.OrganizationStatusActive}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
root := t.TempDir()
|
||||
seedSliceDictionary(t, f, "测试切片")
|
||||
task := f.task
|
||||
task.NodeID = "human"
|
||||
task.ArtifactID = "left-video"
|
||||
task.RightArtifactID = "right-video"
|
||||
task.ClipEnd = 4
|
||||
task.SourceIsFullVideo = true
|
||||
task.SourceDurationSeconds = 4
|
||||
task.CoverageStartSecond = 0
|
||||
task.CoverageEndSecond = 4
|
||||
task.CoverageMode = "full_common_range"
|
||||
pts, imu := "timestamp_us\n", "timestamp_us,x,y,z,type\n"
|
||||
for i := 0; i < 32; i++ {
|
||||
stamp := 1000000 + i*125000
|
||||
pts += fmt.Sprintf("%d\n", stamp)
|
||||
imu += fmt.Sprintf("%d,1,2,3,accel\n%d,4,5,6,gyro\n", stamp, stamp)
|
||||
}
|
||||
bindings := []domain.NodeInputBinding{}
|
||||
for _, side := range []string{"left", "right"} {
|
||||
video := filepath.Join(root, side+".mp4")
|
||||
if out, e := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=64x48:r=8:d=4", "-c:v", "libx264", "-pix_fmt", "yuv420p", video).CombinedOutput(); e != nil {
|
||||
t.Fatalf("%v: %s", e, out)
|
||||
}
|
||||
p := filepath.Join(root, side+"_pts.csv")
|
||||
os.WriteFile(p, []byte(pts), 0600)
|
||||
videoPort := "fullLeftVideos"
|
||||
if side == "right" {
|
||||
videoPort = "fullRightVideos"
|
||||
}
|
||||
bindings = append(bindings, domain.NodeInputBinding{TargetPortID: videoPort, Artifacts: []domain.ExecutionArtifact{{ID: side + "-video", Name: side + ".mp4", Path: video, Data: map[string]any{"collectionId": "capture"}}}}, domain.NodeInputBinding{TargetPortID: side + "Timestamps", Artifacts: []domain.ExecutionArtifact{{ID: side + "-pts", Name: side + "_pts.csv", Path: p, Data: map[string]any{"collectionId": "capture"}}}})
|
||||
}
|
||||
p := filepath.Join(root, "imu.csv")
|
||||
os.WriteFile(p, []byte(imu), 0600)
|
||||
bindings = append(bindings, domain.NodeInputBinding{TargetPortID: "imu", Artifacts: []domain.ExecutionArtifact{{ID: "imu", Name: "imu.csv", Path: p, Data: map[string]any{"collectionId": "capture"}}}})
|
||||
f.store.UpsertRun(f.ctx, domain.RunRecord{ID: task.RunID, OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, WorkflowID: task.WorkflowID, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: task.NodeID, InputBindings: bindings}}})
|
||||
f.store.UpsertAnnotationTask(f.ctx, task)
|
||||
f.task = task
|
||||
f.handler = NewHandlerWithConfig(f.store, nil, config.AppConfig{WorkspaceMountPath: root})
|
||||
return f, root
|
||||
}
|
||||
|
||||
func TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
||||
f, root := annotationProcessorFixture(t)
|
||||
slices := []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 0, End: 2}, {ID: "b", Name: "测试切片", Start: 2, End: 4}}
|
||||
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": f.item.Version, "sliceSegments": slices})
|
||||
if r.Code != 202 {
|
||||
t.Fatal(r.Code, r.Body.String())
|
||||
}
|
||||
h, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
if h.ProcessingJob == nil || h.ProcessingJob.Status != "queued" || len(h.ProcessingJob.Input) == 0 {
|
||||
t.Fatal("missing atomic outbox")
|
||||
}
|
||||
if b, _ := json.Marshal(h); stringContains(string(b), "processingJob") || stringContains(string(b), "sourceFingerprint") {
|
||||
t.Fatal("private outbox leaked")
|
||||
}
|
||||
// A new API instance must not clear a durable job when it has no local map.
|
||||
f.handler = NewHandlerWithConfig(f.store, nil, config.AppConfig{WorkspaceMountPath: root})
|
||||
r = f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"?view=status", f.annotator, nil)
|
||||
if r.Code != 200 {
|
||||
t.Fatal(r.Code, r.Body.String())
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "dead-worker", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano))
|
||||
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "replacement-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 {
|
||||
t.Fatal("lease not recovered")
|
||||
}
|
||||
if ok, _ := f.store.UpdateHumanItemVersioned(f.ctx, *old, old.Version); ok {
|
||||
t.Fatal("expired worker can publish")
|
||||
}
|
||||
p := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
|
||||
p.executeAnnotationJob(context.Background(), *claimed)
|
||||
h, _ = f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
if h.ProcessingJob.Status != "completed" || len(h.ProcessingJob.Input) != 0 || stringValue(mapValue(h.Subject.Payload["sliceWorkflow"])["status"]) != "completed" {
|
||||
t.Fatalf("slice job failed: %+v", h.ProcessingJob)
|
||||
}
|
||||
baseHandler := f.handler
|
||||
f.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r.Header.Set("Prefer", "respond-async")
|
||||
baseHandler.ServeHTTP(w, r)
|
||||
})
|
||||
r = f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/combined-submissions", f.annotator, map[string]any{"version": h.Version, "sliceSegments": slices, "segments": []any{}, "requireSliceCompletion": true})
|
||||
if r.Code != 202 {
|
||||
t.Fatal(r.Code, r.Body.String())
|
||||
}
|
||||
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
||||
if before.PendingCombinedSubmissionID == "" || before.ProcessingJob.Kind != "submit" {
|
||||
t.Fatal("submission not atomically queued")
|
||||
}
|
||||
job, _ := f.store.ClaimAnnotationProcessing(f.ctx, "submit-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if job == nil {
|
||||
t.Fatal("submission not claimable")
|
||||
}
|
||||
p.executeAnnotationJob(context.Background(), *job)
|
||||
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
||||
if after.CurrentStageID != "quality_review" || after.ProcessingJob.Status != "completed" || after.PendingCombinedSubmissionID != "" {
|
||||
t.Fatalf("submission failed %+v", after.ProcessingJob)
|
||||
}
|
||||
run, _ := f.store.GetRun(f.ctx, f.task.RunID)
|
||||
found := false
|
||||
for _, a := range run.Artifacts {
|
||||
if boolValue(a.Data["metadataOnly"]) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("unchanged media was rebuilt")
|
||||
}
|
||||
}
|
||||
|
||||
func stringContains(s, part string) bool {
|
||||
return len(part) > 0 && len(s) >= len(part) && strings.Contains(s, part)
|
||||
}
|
||||
197
internal/api/annotation_storage.go
Normal file
197
internal/api/annotation_storage.go
Normal file
@ -0,0 +1,197 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/objectsync"
|
||||
"fmt"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func annotationObjectSHA(meta objectsync.ObjectMetadata) string {
|
||||
for k, v := range meta.UserMeta {
|
||||
if strings.EqualFold(k, "sha256") {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *server) annotationMediaCopyReady(ctx context.Context, org, path string, file map[string]any) error {
|
||||
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, path)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
if stringValue(file["etag"]) == "" {
|
||||
return fmt.Errorf("legacy media needs initial verification")
|
||||
}
|
||||
meta, e := (objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}).Head(ctx, *c, bucket, key)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if !meta.Exists || meta.ETag != stringValue(file["etag"]) || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != stringValue(file["sha256"]) {
|
||||
return fmt.Errorf("cached media version changed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *server) annotationWorkspaceObject(ctx context.Context, org, path string) (*domain.StorageConnectionRecord, string, string, error) {
|
||||
id, bucket := os.Getenv("ANNOTATION_WORKSPACE_CONNECTION_ID"), os.Getenv("ANNOTATION_WORKSPACE_BUCKET")
|
||||
if id == "" && bucket == "" {
|
||||
id, bucket = os.Getenv("PROJECT_DELETION_WORKSPACE_CONNECTION_ID"), os.Getenv("PROJECT_DELETION_WORKSPACE_BUCKET")
|
||||
}
|
||||
if id == "" && bucket == "" {
|
||||
return nil, "", "", nil
|
||||
}
|
||||
if id == "" || bucket == "" {
|
||||
return nil, "", "", fmt.Errorf("annotation workspace mapping incomplete")
|
||||
}
|
||||
c, e := s.store.GetStorageConnection(ctx, id)
|
||||
if e != nil {
|
||||
return nil, "", "", e
|
||||
}
|
||||
if c == nil || c.OrganizationID != org || c.Provider != domain.StorageProviderBOS {
|
||||
return nil, "", "", fmt.Errorf("annotation workspace scope mismatch")
|
||||
}
|
||||
rel, e := filepath.Rel(s.workspaceMountPath, path)
|
||||
if e != nil || rel == "." || strings.HasPrefix(rel, "..") {
|
||||
return nil, "", "", fmt.Errorf("annotation output outside workspace")
|
||||
}
|
||||
return c, bucket, filepath.ToSlash(rel), nil
|
||||
}
|
||||
|
||||
func (s *server) publishAnnotationFile(ctx context.Context, org, source, target, sha string) (objectsync.ObjectMetadata, error) {
|
||||
if e := s.assertAnnotationLease(ctx); e != nil {
|
||||
return objectsync.ObjectMetadata{}, e
|
||||
}
|
||||
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, target)
|
||||
if e != nil {
|
||||
return objectsync.ObjectMetadata{}, e
|
||||
}
|
||||
if c == nil {
|
||||
e := copyAnnotationFile(ctx, source, target, sha)
|
||||
return objectsync.ObjectMetadata{}, e
|
||||
}
|
||||
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
|
||||
info, e := os.Stat(source)
|
||||
if e != nil {
|
||||
return objectsync.ObjectMetadata{}, e
|
||||
}
|
||||
meta, e := p.Head(ctx, *c, bucket, key)
|
||||
if e != nil {
|
||||
return meta, e
|
||||
}
|
||||
if meta.Exists {
|
||||
if meta.SizeBytes == info.Size() && annotationObjectSHA(meta) == sha {
|
||||
return meta, nil
|
||||
}
|
||||
return meta, fmt.Errorf("annotation output conflict: %s", filepath.Base(target))
|
||||
}
|
||||
f, e := os.Open(source)
|
||||
if e != nil {
|
||||
return meta, e
|
||||
}
|
||||
defer f.Close()
|
||||
meta, e = p.PutInternal(ctx, *c, bucket, key, contextFileReader{ctx, f}, info.Size(), "application/octet-stream", map[string]string{"sha256": sha})
|
||||
if e != nil {
|
||||
actual, he := p.Head(ctx, *c, bucket, key)
|
||||
if he == nil && actual.Exists && actual.SizeBytes == info.Size() && annotationObjectSHA(actual) == sha {
|
||||
return actual, nil
|
||||
}
|
||||
}
|
||||
return meta, e
|
||||
}
|
||||
|
||||
// The bytes stay in BOS. Source ETag and digest metadata fence the copy, while
|
||||
// the fresh generation keeps independent objects for existing retention rules.
|
||||
func (s *server) copyAnnotationObject(ctx context.Context, org, source, target string, file map[string]any) (objectsync.ObjectMetadata, error) {
|
||||
sha := stringValue(file["sha256"])
|
||||
if sha == "" {
|
||||
return objectsync.ObjectMetadata{}, fmt.Errorf("missing verified media digest")
|
||||
}
|
||||
c, bucket, key, e := s.annotationWorkspaceObject(ctx, org, source)
|
||||
if e != nil {
|
||||
return objectsync.ObjectMetadata{}, e
|
||||
}
|
||||
if c == nil {
|
||||
e := copyAnnotationFile(ctx, source, target, sha)
|
||||
return objectsync.ObjectMetadata{}, e
|
||||
}
|
||||
_, toBucket, toKey, e := s.annotationWorkspaceObject(ctx, org, target)
|
||||
if e != nil {
|
||||
return objectsync.ObjectMetadata{}, e
|
||||
}
|
||||
if bucket != toBucket {
|
||||
return objectsync.ObjectMetadata{}, fmt.Errorf("cross-workspace media copy")
|
||||
}
|
||||
p := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
|
||||
meta, e := p.Head(ctx, *c, bucket, key)
|
||||
if e != nil {
|
||||
return meta, e
|
||||
}
|
||||
// Old generations without authoritative object metadata take the verified
|
||||
// normal materialization path once, rather than trusting an unpinned copy.
|
||||
if !meta.Exists || meta.SizeBytes != int64Value(file["bytes"]) || annotationObjectSHA(meta) != sha || stringValue(file["etag"]) == "" || meta.ETag != stringValue(file["etag"]) {
|
||||
return meta, fmt.Errorf("cached media version unavailable")
|
||||
}
|
||||
old, e := p.Head(ctx, *c, bucket, toKey)
|
||||
if e != nil {
|
||||
return old, e
|
||||
}
|
||||
if old.Exists {
|
||||
if old.SizeBytes == meta.SizeBytes && annotationObjectSHA(old) == sha {
|
||||
return old, nil
|
||||
}
|
||||
return old, fmt.Errorf("annotation media destination conflict")
|
||||
}
|
||||
if e = s.assertAnnotationLease(ctx); e != nil {
|
||||
return old, e
|
||||
}
|
||||
return p.PublishConditional(ctx, *c, bucket, key, toKey, meta.SizeBytes, meta.ETag, "", "*", map[string]string{"sha256": sha})
|
||||
}
|
||||
|
||||
func (s *server) publishAnnotationDirectory(ctx context.Context, org, source, target string, manifest map[string]any) (string, int64, error) {
|
||||
// Publish all media/sidecars before the generation manifest. Limit network
|
||||
// concurrency independently of encoder concurrency.
|
||||
group, gctx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(4)
|
||||
items := listValue(manifest["items"])
|
||||
manifest["items"] = items
|
||||
for i, raw := range items {
|
||||
item := mapValue(raw)
|
||||
items[i] = item
|
||||
dir := stringValue(item["directory"])
|
||||
files := mapValue(item["files"])
|
||||
item["files"] = files
|
||||
for kind, rawFile := range files {
|
||||
file := mapValue(rawFile)
|
||||
files[kind] = file
|
||||
name := stringValue(file["name"])
|
||||
if name == "" || filepath.Base(name) != name || filepath.IsAbs(dir) || strings.Contains(dir, "..") {
|
||||
return "", 0, fmt.Errorf("invalid annotation member path")
|
||||
}
|
||||
group.Go(func() error {
|
||||
meta, e := s.publishAnnotationFile(gctx, org, filepath.Join(source, dir, name), filepath.Join(target, dir, name), stringValue(file["sha256"]))
|
||||
if e == nil && meta.ETag != "" {
|
||||
file["etag"] = meta.ETag
|
||||
}
|
||||
return e
|
||||
})
|
||||
}
|
||||
}
|
||||
if e := group.Wait(); e != nil {
|
||||
return "", 0, e
|
||||
}
|
||||
path := filepath.Join(source, "revision-manifest.json")
|
||||
sha, size, e := writeJSONFile(path, manifest)
|
||||
if e != nil {
|
||||
return "", 0, e
|
||||
}
|
||||
_, e = s.publishAnnotationFile(ctx, org, path, filepath.Join(target, "revision-manifest.json"), sha)
|
||||
return sha, size, e
|
||||
}
|
||||
104
internal/api/annotation_timeline_index.go
Normal file
104
internal/api/annotation_timeline_index.go
Normal file
@ -0,0 +1,104 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
)
|
||||
|
||||
type persistedAnnotationTimeline struct {
|
||||
Camera string `json:"camera"`
|
||||
FrameCount int `json:"frameCount"`
|
||||
LeftFrameCount int `json:"leftFrameCount"`
|
||||
RightFrameCount int `json:"rightFrameCount"`
|
||||
Origin int64 `json:"originTimestampUs"`
|
||||
End int64 `json:"coverageEndTimestampUs"`
|
||||
Seconds []float64 `json:"boundariesSeconds"`
|
||||
RightSeconds []float64 `json:"rightBoundariesSeconds"`
|
||||
Stamps []int64 `json:"boundaryTimestampUs"`
|
||||
Left []int `json:"leftFrameIndices"`
|
||||
Right []int `json:"rightFrameIndices"`
|
||||
StartSecond float64 `json:"rangeStartSecond"`
|
||||
EndSecond float64 `json:"rangeEndSecond"`
|
||||
Contract string `json:"boundaryContract"`
|
||||
Scale float64 `json:"ptsToMediaScale"`
|
||||
Fingerprint string `json:"sourceFingerprint"`
|
||||
}
|
||||
|
||||
func decodePersistentTimeline(record *domain.AnnotationTimelineIndex, fingerprint string) (map[string]any, error) {
|
||||
if record == nil {
|
||||
return nil, nil
|
||||
}
|
||||
sum := sha256.Sum256(record.Body)
|
||||
if hex.EncodeToString(sum[:]) != record.SHA256 {
|
||||
return nil, fmt.Errorf("timeline index checksum mismatch")
|
||||
}
|
||||
z, e := gzip.NewReader(bytes.NewReader(record.Body))
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer z.Close()
|
||||
b, e := io.ReadAll(io.LimitReader(z, 8<<20+1))
|
||||
if e != nil || len(b) > 8<<20 {
|
||||
return nil, fmt.Errorf("invalid timeline index size")
|
||||
}
|
||||
var t persistedAnnotationTimeline
|
||||
if e = json.Unmarshal(b, &t); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
n := len(t.Seconds)
|
||||
if t.Contract != "hardware-frame-v1" || t.Fingerprint != fingerprint || n < 2 || len(t.Stamps) != n || len(t.RightSeconds) != n || len(t.Left) != n || len(t.Right) != n || t.LeftFrameCount < 1 || t.RightFrameCount < 1 {
|
||||
return nil, fmt.Errorf("invalid timeline index contract")
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if math.IsNaN(t.Seconds[i]) || math.IsInf(t.Seconds[i], 0) || i > 0 && (t.Stamps[i] <= t.Stamps[i-1] || t.Seconds[i] <= t.Seconds[i-1]) {
|
||||
return nil, fmt.Errorf("invalid timeline index ordering")
|
||||
}
|
||||
}
|
||||
return map[string]any{"camera": "left", "frameCount": t.LeftFrameCount, "leftFrameCount": t.LeftFrameCount, "rightFrameCount": t.RightFrameCount, "originTimestampUs": t.Origin, "coverageEndTimestampUs": t.End, "boundariesSeconds": t.Seconds, "rightBoundariesSeconds": t.RightSeconds, "boundaryTimestampUs": t.Stamps, "leftFrameIndices": t.Left, "rightFrameIndices": t.Right, "rangeStartSecond": t.StartSecond, "rangeEndSecond": t.EndSecond, "boundaryContract": t.Contract, "ptsToMediaScale": t.Scale, "sourceFingerprint": t.Fingerprint}, nil
|
||||
}
|
||||
|
||||
func (s *server) persistentTimeline(ctx context.Context, t domain.AnnotationTaskRecord, key string, build func(context.Context) (map[string]any, error)) (map[string]any, error) {
|
||||
db, ok := s.store.(store.AnnotationProcessingStore)
|
||||
if !ok {
|
||||
return build(ctx)
|
||||
}
|
||||
id := "annotation-timeline-v1:" + t.OrganizationID + ":" + key
|
||||
record, e := db.GetAnnotationTimelineIndex(ctx, id)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if cached, e := decodePersistentTimeline(record, key); e == nil && cached != nil {
|
||||
return cached, nil
|
||||
}
|
||||
v, e := build(ctx)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
b, e := json.Marshal(v)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if len(b) > 8<<20 {
|
||||
return v, nil
|
||||
}
|
||||
var compressed bytes.Buffer
|
||||
z := gzip.NewWriter(&compressed)
|
||||
if _, e = z.Write(b); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if e = z.Close(); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
sum := sha256.Sum256(compressed.Bytes())
|
||||
e = db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: id, OrganizationID: t.OrganizationID, ProjectID: t.ProjectID, WorkflowID: t.WorkflowID, RunID: t.RunID, Body: compressed.Bytes(), SHA256: hex.EncodeToString(sum[:]), CreatedAt: nowISO()})
|
||||
return v, e
|
||||
}
|
||||
82
internal/api/annotation_timeline_index_test.go
Normal file
82
internal/api/annotation_timeline_index_test.go
Normal file
@ -0,0 +1,82 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPersistentTimelineSurvivesProcessCacheAndRebuildsCorruption(t *testing.T) {
|
||||
db := store.NewMemoryStore()
|
||||
task := domain.AnnotationTaskRecord{ID: "task", OrganizationID: "org", ProjectID: "project", RunID: "run"}
|
||||
count := 0
|
||||
v := map[string]any{"camera": "left", "frameCount": 2, "leftFrameCount": 2, "rightFrameCount": 2, "originTimestampUs": int64(100), "coverageEndTimestampUs": int64(300), "boundariesSeconds": []float64{0, 0.1, 0.2}, "rightBoundariesSeconds": []float64{0, 0.1, 0.2}, "boundaryTimestampUs": []int64{100, 200, 300}, "leftFrameIndices": []int{0, 1, 2}, "rightFrameIndices": []int{0, 1, 2}, "rangeStartSecond": 0.0, "rangeEndSecond": 0.2, "boundaryContract": "hardware-frame-v1", "ptsToMediaScale": 1.0, "sourceFingerprint": "source"}
|
||||
build := func(context.Context) (map[string]any, error) { count++; return cloneTimeline(v), nil }
|
||||
s := &server{store: db}
|
||||
if _, e := s.persistentTimeline(context.Background(), task, "source", build); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s = &server{store: db}
|
||||
got, e := s.persistentTimeline(context.Background(), task, "source", build)
|
||||
if e != nil || count != 1 {
|
||||
t.Fatal("persistent cache missed", count, e)
|
||||
}
|
||||
if got["boundaryTimestampUs"].([]int64)[1] != 200 {
|
||||
t.Fatal("index changed integer timestamps")
|
||||
}
|
||||
id := "annotation-timeline-v1:org:source"
|
||||
bad, _ := db.GetAnnotationTimelineIndex(context.Background(), id)
|
||||
bad.Body[0] ^= 0xff
|
||||
db.PutAnnotationTimelineIndex(context.Background(), *bad)
|
||||
if _, e = s.persistentTimeline(context.Background(), task, "source", build); e != nil || count != 2 {
|
||||
t.Fatal("corrupt cache not rebuilt", e)
|
||||
}
|
||||
other := task
|
||||
other.OrganizationID = "other"
|
||||
if _, e = s.persistentTimeline(context.Background(), other, "source", build); e != nil || count != 3 {
|
||||
t.Fatal("organization cache isolation", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskCreationPrecomputesPersistentTimeline(t *testing.T) {
|
||||
f, root := annotationProcessorFixture(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: root}, 1, false)
|
||||
}()
|
||||
s := &server{store: f.store, workspaceMountPath: root}
|
||||
run, _ := f.store.GetRun(ctx, f.task.RunID)
|
||||
key, e := s.annotationSourceFingerprint(ctx, f.task, *run)
|
||||
if e != nil {
|
||||
cancel()
|
||||
t.Fatal(e)
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
found := false
|
||||
for time.Now().Before(deadline) {
|
||||
index, e := f.store.GetAnnotationTimelineIndex(ctx, "annotation-timeline-v1:"+f.task.OrganizationID+":"+key)
|
||||
if e == nil && index != nil {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
if !found {
|
||||
t.Fatal("new task not proactively indexed")
|
||||
}
|
||||
restarted := &server{store: f.store, workspaceMountPath: root}
|
||||
left, e := restarted.annotationFrameTimelineForTask(context.Background(), f.task, "left")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
right, e := restarted.annotationFrameTimelineForTask(context.Background(), f.task, "right")
|
||||
if e != nil || left["frameCount"] != right["frameCount"] {
|
||||
t.Fatal("stereo index", e)
|
||||
}
|
||||
}
|
||||
@ -503,6 +503,9 @@ func absInt64(value int64) int64 {
|
||||
}
|
||||
|
||||
func (server *server) materializeCombinedSliceSubmission(ctx context.Context, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, payload combinedAnnotationSubmissionInput, revision int) (*humanSliceMaterialization, error) {
|
||||
if err := server.assertAnnotationLease(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
started := time.Now()
|
||||
ctx, cancel := context.WithTimeout(ctx, 24*time.Hour)
|
||||
defer cancel()
|
||||
@ -561,6 +564,11 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
|
||||
return nil, err
|
||||
}
|
||||
reusable := reusableAnnotationMedia(*run, task, item, server.workspaceMountPath, sourceFingerprint, revision)
|
||||
if metadata, e := server.tryAnnotationMetadataRevision(ctx, task, item, run, payload, revision, sourceFingerprint); e != nil {
|
||||
return nil, e
|
||||
} else if metadata != nil {
|
||||
return metadata, nil
|
||||
}
|
||||
stage, err := os.MkdirTemp("", "annotation-materialize-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -775,6 +783,45 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
|
||||
manifestItems := make([]map[string]any, 0, len(slices))
|
||||
keptSliceCount, discardedSliceCount := 0, 0
|
||||
reusedSlices := 0
|
||||
reusedByIndex := map[int]bool{}
|
||||
leftCuts, rightCuts := []annotationFrameSlice{}, []annotationFrameSlice{}
|
||||
for i, slice := range normalizedSlices {
|
||||
a, b := mediaSecondToTimestamp(slice.Start), mediaSecondToTimestamp(slice.End)
|
||||
ls := sort.Search(len(leftPTS), func(j int) bool { return leftPTS[j].TimestampUS >= a })
|
||||
le := sort.Search(len(leftPTS), func(j int) bool { return leftPTS[j].TimestampUS >= b })
|
||||
rs := sort.Search(len(rightPTS), func(j int) bool { return rightPTS[j].TimestampUS >= a })
|
||||
re := sort.Search(len(rightPTS), func(j int) bool { return rightPTS[j].TimestampUS >= b })
|
||||
dir := filepath.Join(root, "slices", fmt.Sprintf("slice-%06d", i+1))
|
||||
if e := os.MkdirAll(dir, 0755); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
leftOut, rightOut := filepath.Join(dir, "left.mp4"), filepath.Join(dir, "right.mp4")
|
||||
key := fmt.Sprintf("%s:h264-ultrafast-crf23-v1:%d:%d:%d:%d", sourceFingerprint, ls, le, rs, re)
|
||||
cached, ok := reusable[key]
|
||||
reusedByIndex[i] = ok && copyReusableVideo(ctx, cached, "left", leftOut) && copyReusableVideo(ctx, cached, "right", rightOut)
|
||||
if !reusedByIndex[i] {
|
||||
leftCuts = append(leftCuts, annotationFrameSlice{ls, le, leftOut})
|
||||
rightCuts = append(rightCuts, annotationFrameSlice{rs, re, rightOut})
|
||||
}
|
||||
}
|
||||
if len(leftCuts) > 0 {
|
||||
if err := stageVideos(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
progress("slicing", 0)
|
||||
var leftErr, rightErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); leftErr = writeFrameAccurateSlices(ctx, leftPath, leftCuts) }()
|
||||
go func() { defer wg.Done(); rightErr = writeFrameAccurateSlices(ctx, rightPath, rightCuts) }()
|
||||
wg.Wait()
|
||||
if leftErr != nil {
|
||||
return nil, leftErr
|
||||
}
|
||||
if rightErr != nil {
|
||||
return nil, rightErr
|
||||
}
|
||||
}
|
||||
keptDuration, discardedDuration := 0.0, 0.0
|
||||
for index, slice := range normalizedSlices {
|
||||
progress("slicing", index)
|
||||
@ -804,32 +851,9 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
|
||||
}
|
||||
leftOutput, rightOutput := filepath.Join(directory, "left.mp4"), filepath.Join(directory, "right.mp4")
|
||||
mediaKey := fmt.Sprintf("%s:h264-ultrafast-crf23-v1:%d:%d:%d:%d", sourceFingerprint, leftStart, leftEnd, rightStart, rightEnd)
|
||||
cached, found := reusable[mediaKey]
|
||||
reused := found && copyReusableVideo(ctx, cached, "left", leftOutput) && copyReusableVideo(ctx, cached, "right", rightOutput)
|
||||
reused := reusedByIndex[index]
|
||||
if reused {
|
||||
reusedSlices++
|
||||
} else {
|
||||
if err := stageVideos(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var leftSliceErr, rightSliceErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
leftSliceErr = writeFrameAccurateSlice(ctx, leftPath, leftOutput, leftStart, leftEnd)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
rightSliceErr = writeFrameAccurateSlice(ctx, rightPath, rightOutput, rightStart, rightEnd)
|
||||
}()
|
||||
wg.Wait()
|
||||
if leftSliceErr != nil {
|
||||
return nil, leftSliceErr
|
||||
}
|
||||
if rightSliceErr != nil {
|
||||
return nil, rightSliceErr
|
||||
}
|
||||
}
|
||||
leftCount, rightCount := leftEnd-leftStart, rightEnd-rightStart
|
||||
if !reused {
|
||||
@ -951,24 +975,21 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
|
||||
return nil, fmt.Errorf("源文件在切片期间发生变化,请重新确认切片")
|
||||
}
|
||||
progress("publishing", len(normalizedSlices))
|
||||
if err := copyAnnotationTree(ctx, root, publishedRoot); err != nil {
|
||||
if err := server.assertAnnotationLease(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manifestSHA, manifestBytes, err = server.publishAnnotationDirectory(ctx, task.OrganizationID, root, publishedRoot, revisionManifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, manifestPath = publishedRoot, filepath.Join(publishedRoot, "revision-manifest.json")
|
||||
log.Printf("[api] materialization_media task=%s revision=%d reused_slices=%d slices=%d", task.ID, revision, reusedSlices, len(normalizedSlices))
|
||||
artifact := domain.ExecutionArtifact{ID: artifactID, RunID: task.RunID, NodeID: task.NodeID, PortID: "materializedSliceDirectory", PortName: "materializedSliceDirectory", Name: fmt.Sprintf("revision-%06d.directory.index", revision), Type: domain.PortTypeFile, Size: fmt.Sprintf("%d files", len(manifestItems)*7+1), SizeBytes: manifestBytes, Path: root, Metadata: &domain.ArtifactMetadata{SHA256: manifestSHA, MimeType: "application/json"}, Data: map[string]any{"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": root, "manifestPath": manifestPath, "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "normalizedSlices": normalizedSlices, "normalizedSegments": normalizedSegments}}
|
||||
run.Artifacts = append(run.Artifacts, artifact)
|
||||
artifact.Data["boundaryContract"] = "hardware-frame-v1"
|
||||
artifact.Data["sourceFingerprint"] = sourceFingerprint
|
||||
artifact.Data["reusedSliceCount"] = reusedSlices
|
||||
artifact.Data["deliverySelection"] = deliverySelection
|
||||
for index := range run.NodeExecutions {
|
||||
if run.NodeExecutions[index].NodeID == task.NodeID {
|
||||
run.NodeExecutions[index].Artifacts = append(run.NodeExecutions[index].Artifacts, artifact)
|
||||
}
|
||||
}
|
||||
run.UpdatedAt = nowISO()
|
||||
if err := server.store.UpsertRun(ctx, *run); err != nil {
|
||||
if err := server.storeAnnotationMaterializationArtifact(ctx, task, run, artifact); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "deliverySelection": deliverySelection, "runId": task.RunID, "artifactId": artifact.ID, "manifestSha256": manifestSHA}, Slices: normalizedSlices, Segments: normalizedSegments, CoverageStartSecond: commonMediaStart, CoverageEndSecond: commonMediaEnd}, nil
|
||||
|
||||
@ -76,6 +76,7 @@ type server struct {
|
||||
materializationSlotsOnce sync.Once
|
||||
materializationSlots chan struct{}
|
||||
materializationProgress sync.Map
|
||||
externalAnnotationProcessor bool
|
||||
}
|
||||
|
||||
type runJobTerminator interface {
|
||||
@ -1259,6 +1260,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
|
||||
operatorRunnerImage: appConfig.OperatorRunnerImage,
|
||||
workerAPIBaseURL: appConfig.WorkerAPIBaseURL,
|
||||
globalBOSEndpoint: appConfig.GlobalBOSEndpoint,
|
||||
externalAnnotationProcessor: os.Getenv("ANNOTATION_PROCESSOR_MODE") == "external",
|
||||
apiToken: appConfig.APIToken,
|
||||
requireSessionAuth: appConfig.RequireSessionAuth,
|
||||
sessionCookieSecure: appConfig.SessionCookieSecure,
|
||||
|
||||
@ -12,6 +12,9 @@ const sliceLeaseDuration = domain.SliceProcessingLeaseDuration
|
||||
// Persisted leases survive API restarts. A dead process must not leave the
|
||||
// workbench spinning forever; retry remains explicit to avoid an OOM loop.
|
||||
func (server *server) recoverInterruptedSlice(ctx context.Context, item *domain.HumanItemRecord, now time.Time) error {
|
||||
if durableAnnotationActive(item) {
|
||||
return nil
|
||||
}
|
||||
if !domain.ExpireSliceProcessing(item, now) {
|
||||
return nil
|
||||
}
|
||||
|
||||
69
internal/domain/annotation_processing.go
Normal file
69
internal/domain/annotation_processing.go
Normal file
@ -0,0 +1,69 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/bsonrw"
|
||||
)
|
||||
|
||||
// ProcessingJob is a private durable outbox embedded in the authoritative
|
||||
// HumanItem CAS. Creating a submission and queuing it are one database write.
|
||||
type AnnotationProcessingJob struct {
|
||||
ID string `bson:"id" json:"id"`
|
||||
Kind string `bson:"kind" json:"kind"`
|
||||
TaskID string `bson:"taskId" json:"taskId"`
|
||||
ActorID string `bson:"actorId" json:"actorId"`
|
||||
Input map[string]any `bson:"input,omitempty" json:"-"`
|
||||
SourceFingerprint string `bson:"sourceFingerprint,omitempty" json:"-"`
|
||||
Revision int `bson:"revision,omitempty" json:"revision,omitempty"`
|
||||
Status string `bson:"status" json:"status"`
|
||||
Owner string `bson:"owner,omitempty" json:"-"`
|
||||
LeaseUntil string `bson:"leaseUntil,omitempty" json:"-"`
|
||||
Attempt int `bson:"attempt" json:"attempt"`
|
||||
Stage string `bson:"stage,omitempty" json:"stage,omitempty"`
|
||||
CompletedSlices int `bson:"completedSlices" json:"completedSlices"`
|
||||
TotalSlices int `bson:"totalSlices" json:"totalSlices"`
|
||||
Error string `bson:"error,omitempty" json:"error,omitempty"`
|
||||
CreatedAt string `bson:"createdAt" json:"createdAt"`
|
||||
UpdatedAt string `bson:"updatedAt" json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (j *AnnotationProcessingJob) UnmarshalBSON(data []byte) error {
|
||||
type plain AnnotationProcessingJob
|
||||
decoder, e := bson.NewDecoder(bsonrw.NewBSONDocumentReader(data))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
decoder.DefaultDocumentM()
|
||||
var out plain
|
||||
if e = decoder.Decode(&out); e != nil {
|
||||
return e
|
||||
}
|
||||
*j = AnnotationProcessingJob(out)
|
||||
return nil
|
||||
}
|
||||
|
||||
type AnnotationTimelineIndex struct {
|
||||
ID string `bson:"id"`
|
||||
OrganizationID string `bson:"organizationId"`
|
||||
ProjectID string `bson:"projectId"`
|
||||
WorkflowID string `bson:"workflowId"`
|
||||
RunID string `bson:"runId"`
|
||||
Body []byte `bson:"body"`
|
||||
SHA256 string `bson:"sha256"`
|
||||
CreatedAt string `bson:"createdAt"`
|
||||
}
|
||||
|
||||
type AnnotationIndexJob struct {
|
||||
ID string `bson:"id"`
|
||||
OrganizationID string `bson:"organizationId"`
|
||||
ProjectID string `bson:"projectId"`
|
||||
WorkflowID string `bson:"workflowId"`
|
||||
RunID string `bson:"runId"`
|
||||
Status string `bson:"status"`
|
||||
Owner string `bson:"owner,omitempty"`
|
||||
LeaseUntil string `bson:"leaseUntil,omitempty"`
|
||||
Attempt int `bson:"attempt"`
|
||||
NextAttemptAt string `bson:"nextAttemptAt"`
|
||||
Error string `bson:"error,omitempty"`
|
||||
CreatedAt string `bson:"createdAt"`
|
||||
}
|
||||
@ -133,6 +133,7 @@ type HumanBatchRecord struct {
|
||||
}
|
||||
|
||||
type HumanItemRecord struct {
|
||||
ProcessingJob *AnnotationProcessingJob `json:"-" bson:"processingJob,omitempty"`
|
||||
ProcessingScope string `json:"processingScope,omitempty" bson:"processingScope,omitempty"`
|
||||
ProcessingProfile string `json:"processingProfile,omitempty" bson:"processingProfile,omitempty"`
|
||||
ID string `json:"id" bson:"id"`
|
||||
|
||||
113
internal/store/annotation_artifact.go
Normal file
113
internal/store/annotation_artifact.go
Normal file
@ -0,0 +1,113 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
func preserveAnnotationArtifacts(incoming *domain.RunRecord, previous domain.RunRecord) {
|
||||
for i := range incoming.NodeExecutions {
|
||||
n := &incoming.NodeExecutions[i]
|
||||
seen := map[string]bool{}
|
||||
for _, a := range n.Artifacts {
|
||||
seen[a.ID] = true
|
||||
}
|
||||
for _, old := range previous.NodeExecutions {
|
||||
if old.NodeID != n.NodeID {
|
||||
continue
|
||||
}
|
||||
for _, a := range old.Artifacts {
|
||||
if a.PortID == "materializedSliceDirectory" && !seen[a.ID] {
|
||||
n.Artifacts = append(n.Artifacts, a)
|
||||
seen[a.ID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range incoming.Artifacts {
|
||||
seen[a.ID] = true
|
||||
}
|
||||
for _, n := range incoming.NodeExecutions {
|
||||
for _, a := range n.Artifacts {
|
||||
if a.PortID == "materializedSliceDirectory" && !seen[a.ID] {
|
||||
incoming.Artifacts = append(incoming.Artifacts, a)
|
||||
seen[a.ID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func annotationSafeRunUpdate(stored domain.RunRecord, unset []string) (mongo.Pipeline, error) {
|
||||
raw, e := bson.Marshal(stored)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
var fields bson.M
|
||||
if e = bson.Unmarshal(raw, &fields); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
set := bson.M{}
|
||||
for key, value := range fields {
|
||||
set[key] = bson.M{"$literal": value}
|
||||
}
|
||||
if len(stored.NodeExecutions) > 0 {
|
||||
oldNodes := bson.M{"$filter": bson.M{"input": bson.M{"$ifNull": bson.A{"$nodeExecutions", bson.A{}}}, "as": "old", "cond": bson.M{"$eq": bson.A{"$$old.nodeId", "$$new.nodeId"}}}}
|
||||
oldArtifacts := bson.M{"$reduce": bson.M{"input": oldNodes, "initialValue": bson.A{}, "in": bson.M{"$concatArrays": bson.A{"$$value", bson.M{"$ifNull": bson.A{"$$this.artifacts", bson.A{}}}}}}}
|
||||
newArtifacts := bson.M{"$ifNull": bson.A{"$$new.artifacts", bson.A{}}}
|
||||
newIDs := bson.M{"$map": bson.M{"input": newArtifacts, "as": "a", "in": "$$a.id"}}
|
||||
keep := bson.M{"$filter": bson.M{"input": oldArtifacts, "as": "a", "cond": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$a.portId", "materializedSliceDirectory"}}, bson.M{"$not": bson.A{bson.M{"$in": bson.A{"$$a.id", newIDs}}}}}}}}
|
||||
set["nodeExecutions"] = bson.M{"$map": bson.M{"input": bson.M{"$literal": stored.NodeExecutions}, "as": "new", "in": bson.M{"$mergeObjects": bson.A{"$$new", bson.M{"artifacts": bson.M{"$concatArrays": bson.A{newArtifacts, keep}}}}}}}
|
||||
}
|
||||
p := mongo.Pipeline{bson.D{{Key: "$set", Value: set}}}
|
||||
if len(unset) > 0 {
|
||||
p = append(p, bson.D{{Key: "$unset", Value: unset}})
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Field-level publication cannot overwrite another Capture or worker's state.
|
||||
func (s *MemoryStore) AppendAnnotationArtifact(_ context.Context, org, node string, a domain.ExecutionArtifact) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
r, ok := s.runs[a.RunID]
|
||||
if !ok || r.OrganizationID != org {
|
||||
return fmt.Errorf("annotation run removed")
|
||||
}
|
||||
for _, old := range r.Artifacts {
|
||||
if old.ID == a.ID {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for i := range r.NodeExecutions {
|
||||
if r.NodeExecutions[i].NodeID == node {
|
||||
r.NodeExecutions[i].Artifacts = append(r.NodeExecutions[i].Artifacts, a)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("annotation node removed")
|
||||
}
|
||||
r.Artifacts = append(r.Artifacts, a)
|
||||
r.UpdatedAt = nowISO()
|
||||
s.runs[r.ID] = clone(r)
|
||||
return nil
|
||||
}
|
||||
func (s *MongoStore) AppendAnnotationArtifact(ctx context.Context, org, node string, a domain.ExecutionArtifact) error {
|
||||
old := bson.M{"$ifNull": bson.A{"$$node.artifacts", bson.A{}}}
|
||||
ids := bson.M{"$map": bson.M{"input": old, "as": "a", "in": "$$a.id"}}
|
||||
items := bson.M{"$cond": bson.A{bson.M{"$in": bson.A{a.ID, ids}}, old, bson.M{"$concatArrays": bson.A{old, bson.M{"$literal": []domain.ExecutionArtifact{a}}}}}}
|
||||
nodes := bson.M{"$map": bson.M{"input": "$nodeExecutions", "as": "node", "in": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{"$$node.nodeId", node}}, bson.M{"$mergeObjects": bson.A{"$$node", bson.M{"artifacts": items}}}, "$$node"}}}}
|
||||
r, e := s.runs.UpdateOne(ctx, bson.M{"id": a.RunID, "organizationId": org, "nodeExecutions.nodeId": node}, mongo.Pipeline{bson.D{{Key: "$set", Value: bson.M{"nodeExecutions": nodes, "updatedAt": nowISO()}}}})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if r.MatchedCount != 1 {
|
||||
return fmt.Errorf("annotation run/node removed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -205,6 +205,14 @@ func (store *MemoryStore) UpsertAnnotationTask(_ context.Context, item domain.An
|
||||
}
|
||||
item.UpdatedAt = now
|
||||
store.annotationTasks[item.ID] = item
|
||||
if item.SourceIsFullVideo && item.TemplateVersion >= 5 && item.ArtifactID != "" {
|
||||
if store.annotationIndexJobs == nil {
|
||||
store.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
|
||||
}
|
||||
if _, exists := store.annotationIndexJobs[item.ID]; !exists {
|
||||
store.annotationIndexJobs[item.ID] = annotationIndexJob(item)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -505,7 +513,13 @@ func (store *MongoStore) UpsertAnnotationTask(ctx context.Context, item domain.A
|
||||
item.CreatedAt = now
|
||||
}
|
||||
item.UpdatedAt = now
|
||||
return mongoUpsert(ctx, store.annotationTasks, item.ID, item)
|
||||
if err := mongoUpsert(ctx, store.annotationTasks, item.ID, item); err != nil {
|
||||
return err
|
||||
}
|
||||
if item.SourceIsFullVideo && item.TemplateVersion >= 5 && item.ArtifactID != "" {
|
||||
_ = store.EnqueueAnnotationIndex(ctx, item)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListAnnotationRevisions(ctx context.Context, organizationID string, taskID string) ([]domain.AnnotationRevisionRecord, error) {
|
||||
|
||||
188
internal/store/annotation_processing.go
Normal file
188
internal/store/annotation_processing.go
Normal file
@ -0,0 +1,188 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type AnnotationProcessingStore interface {
|
||||
AppendAnnotationArtifact(context.Context, string, string, domain.ExecutionArtifact) error
|
||||
GetAnnotationTimelineIndex(context.Context, string) (*domain.AnnotationTimelineIndex, error)
|
||||
PutAnnotationTimelineIndex(context.Context, domain.AnnotationTimelineIndex) error
|
||||
EnqueueAnnotationIndex(context.Context, domain.AnnotationTaskRecord) error
|
||||
ClaimAnnotationIndex(context.Context, string, string, string) (*domain.AnnotationIndexJob, error)
|
||||
FinishAnnotationIndex(context.Context, domain.AnnotationIndexJob) (bool, error)
|
||||
ClaimAnnotationProcessing(context.Context, string, string, string) (*domain.HumanItemRecord, error)
|
||||
SaveAnnotationProcessing(context.Context, string, domain.AnnotationProcessingJob) (bool, error)
|
||||
TouchAnnotationProcessing(context.Context, string, string, string, string, string, int, int) (bool, error)
|
||||
}
|
||||
|
||||
func (s *MemoryStore) TouchAnnotationProcessing(_ context.Context, itemID, id, owner, until, stage string, completed, total int) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
h, ok := s.humanItems[itemID]
|
||||
if !ok || h.ProcessingJob == nil || h.ProcessingJob.ID != id || h.ProcessingJob.Owner != owner || h.ProcessingJob.Status != "running" {
|
||||
return false, nil
|
||||
}
|
||||
h.ProcessingJob.LeaseUntil = until
|
||||
h.ProcessingJob.Stage = stage
|
||||
h.ProcessingJob.CompletedSlices = completed
|
||||
h.ProcessingJob.TotalSlices = total
|
||||
h.ProcessingJob.UpdatedAt = nowISO()
|
||||
s.humanItems[itemID] = h
|
||||
return true, nil
|
||||
}
|
||||
func (s *MongoStore) TouchAnnotationProcessing(ctx context.Context, itemID, id, owner, until, stage string, completed, total int) (bool, error) {
|
||||
r, e := s.humanItems.UpdateOne(ctx, bson.M{"id": itemID, "processingJob.id": id, "processingJob.owner": owner, "processingJob.status": "running"}, bson.M{"$set": bson.M{"processingJob.leaseUntil": until, "processingJob.stage": stage, "processingJob.completedSlices": completed, "processingJob.totalSlices": total, "processingJob.updatedAt": nowISO()}})
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
return r.MatchedCount == 1, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) GetAnnotationTimelineIndex(_ context.Context, id string) (*domain.AnnotationTimelineIndex, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.annotationTimelineIndexes[id]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
v.Body = append([]byte(nil), v.Body...)
|
||||
return &v, nil
|
||||
}
|
||||
func (s *MemoryStore) PutAnnotationTimelineIndex(_ context.Context, v domain.AnnotationTimelineIndex) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.annotationTimelineIndexes == nil {
|
||||
s.annotationTimelineIndexes = map[string]domain.AnnotationTimelineIndex{}
|
||||
}
|
||||
v.Body = append([]byte(nil), v.Body...)
|
||||
s.annotationTimelineIndexes[v.ID] = v
|
||||
return nil
|
||||
}
|
||||
func (s *MongoStore) GetAnnotationTimelineIndex(ctx context.Context, id string) (*domain.AnnotationTimelineIndex, error) {
|
||||
return mongoGetByID[domain.AnnotationTimelineIndex](ctx, s.db.Collection("annotationTimelineIndexes"), id)
|
||||
}
|
||||
func (s *MongoStore) PutAnnotationTimelineIndex(ctx context.Context, v domain.AnnotationTimelineIndex) error {
|
||||
return mongoUpsert(ctx, s.db.Collection("annotationTimelineIndexes"), v.ID, v)
|
||||
}
|
||||
|
||||
func annotationIndexJob(t domain.AnnotationTaskRecord) domain.AnnotationIndexJob {
|
||||
return domain.AnnotationIndexJob{ID: t.ID, OrganizationID: t.OrganizationID, ProjectID: t.ProjectID, WorkflowID: t.WorkflowID, RunID: t.RunID, Status: "queued", CreatedAt: nowISO(), NextAttemptAt: ""}
|
||||
}
|
||||
func (s *MemoryStore) EnqueueAnnotationIndex(_ context.Context, t domain.AnnotationTaskRecord) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.annotationIndexJobs == nil {
|
||||
s.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
|
||||
}
|
||||
if _, ok := s.annotationIndexJobs[t.ID]; !ok {
|
||||
s.annotationIndexJobs[t.ID] = annotationIndexJob(t)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *MongoStore) EnqueueAnnotationIndex(ctx context.Context, t domain.AnnotationTaskRecord) error {
|
||||
_, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": t.ID}, bson.M{"$setOnInsert": annotationIndexJob(t)}, options.Update().SetUpsert(true))
|
||||
return e
|
||||
}
|
||||
func claimableJob(status, lease, next, now string) bool {
|
||||
return status == "queued" && next <= now || status == "running" && lease < now
|
||||
}
|
||||
func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ids := []string{}
|
||||
for id, j := range s.annotationIndexJobs {
|
||||
if claimableJob(j.Status, j.LeaseUntil, j.NextAttemptAt, now) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
j := s.annotationIndexJobs[ids[0]]
|
||||
j.Status = "running"
|
||||
j.Owner = owner
|
||||
j.LeaseUntil = until
|
||||
j.Attempt++
|
||||
s.annotationIndexJobs[j.ID] = j
|
||||
return &j, nil
|
||||
}
|
||||
func (s *MongoStore) ClaimAnnotationIndex(ctx context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
|
||||
var j domain.AnnotationIndexJob
|
||||
e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&j)
|
||||
if e == mongo.ErrNoDocuments {
|
||||
return nil, nil
|
||||
}
|
||||
return &j, e
|
||||
}
|
||||
func (s *MemoryStore) FinishAnnotationIndex(_ context.Context, j domain.AnnotationIndexJob) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
old, ok := s.annotationIndexJobs[j.ID]
|
||||
if !ok || old.Owner != j.Owner || old.Status != "running" {
|
||||
return false, nil
|
||||
}
|
||||
s.annotationIndexJobs[j.ID] = j
|
||||
return true, nil
|
||||
}
|
||||
func (s *MongoStore) FinishAnnotationIndex(ctx context.Context, j domain.AnnotationIndexJob) (bool, error) {
|
||||
r, e := s.db.Collection("annotationIndexJobs").ReplaceOne(ctx, bson.M{"id": j.ID, "owner": j.Owner, "status": "running"}, j)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
return r.MatchedCount == 1, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) ClaimAnnotationProcessing(_ context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ids := []string{}
|
||||
for id, h := range s.humanItems {
|
||||
if j := h.ProcessingJob; j != nil && claimableJob(j.Status, j.LeaseUntil, "", now) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
h := clone(s.humanItems[ids[0]])
|
||||
h.ProcessingJob.Status = "running"
|
||||
h.ProcessingJob.Owner = owner
|
||||
h.ProcessingJob.LeaseUntil = until
|
||||
h.ProcessingJob.Attempt++
|
||||
s.humanItems[h.ID] = clone(h)
|
||||
return &h, nil
|
||||
}
|
||||
func (s *MongoStore) ClaimAnnotationProcessing(ctx context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
|
||||
var h domain.HumanItemRecord
|
||||
e := s.humanItems.FindOneAndUpdate(ctx, bson.M{"$or": bson.A{bson.M{"processingJob.status": "queued"}, bson.M{"processingJob.status": "running", "processingJob.leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"processingJob.status": "running", "processingJob.owner": owner, "processingJob.leaseUntil": until}, "$inc": bson.M{"processingJob.attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "processingJob.createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&h)
|
||||
if e == mongo.ErrNoDocuments {
|
||||
return nil, nil
|
||||
}
|
||||
return &h, e
|
||||
}
|
||||
func (s *MemoryStore) SaveAnnotationProcessing(_ context.Context, itemID string, j domain.AnnotationProcessingJob) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
h, ok := s.humanItems[itemID]
|
||||
if !ok || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner || h.ProcessingJob.Status != "running" {
|
||||
return false, nil
|
||||
}
|
||||
h.ProcessingJob = &j
|
||||
s.humanItems[itemID] = clone(h)
|
||||
return true, nil
|
||||
}
|
||||
func (s *MongoStore) SaveAnnotationProcessing(ctx context.Context, itemID string, j domain.AnnotationProcessingJob) (bool, error) {
|
||||
r, e := s.humanItems.UpdateOne(ctx, bson.M{"id": itemID, "processingJob.id": j.ID, "processingJob.owner": j.Owner, "processingJob.status": "running"}, bson.M{"$set": bson.M{"processingJob": j}})
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
return r.MatchedCount == 1, nil
|
||||
}
|
||||
80
internal/store/annotation_processing_test.go
Normal file
80
internal/store/annotation_processing_test.go
Normal file
@ -0,0 +1,80 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAnnotationOutboxBSONAndExclusiveClaims(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewMemoryStore()
|
||||
j := domain.AnnotationProcessingJob{ID: "job", TaskID: "task", Status: "queued", Input: map[string]any{"sliceSegments": []map[string]any{{"id": "slice", "start": 0.0, "end": 1.0}}}}
|
||||
encoded, e := bson.Marshal(j)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var restored domain.AnnotationProcessingJob
|
||||
if e = bson.Unmarshal(encoded, &restored); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
body, _ := json.Marshal(restored.Input)
|
||||
if strings.Contains(string(body), "Key") || !strings.Contains(string(body), "sliceSegments") {
|
||||
t.Fatal("BSON changed queued input", string(body))
|
||||
}
|
||||
h := domain.HumanItemRecord{ID: "item", OrganizationID: "org", Version: 3, ProcessingJob: &restored}
|
||||
s.UpsertHumanItem(ctx, h)
|
||||
var claims atomic.Int32
|
||||
var wg sync.WaitGroup
|
||||
now := time.Now().UTC()
|
||||
for n := 0; n < 16; n++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
h, e := s.ClaimAnnotationProcessing(ctx, domain.NewID("worker"), now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
if h != nil {
|
||||
claims.Add(1)
|
||||
if len(h.ProcessingJob.Input) == 0 {
|
||||
t.Error("private input lost in store clone")
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if claims.Load() != 1 {
|
||||
t.Fatal("duplicate claims", claims.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerSnapshotCannotErasePublishedAnnotationArtifacts(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
r := domain.RunRecord{ID: "run", OrganizationID: "org", NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "human"}}}
|
||||
s.UpsertRun(ctx, r)
|
||||
a := domain.ExecutionArtifact{ID: "media", RunID: r.ID, NodeID: "human", PortID: "materializedSliceDirectory"}
|
||||
if e := s.AppendAnnotationArtifact(ctx, "org", "human", a); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
r.Status = domain.RunStatusWaitingForHuman
|
||||
s.UpsertRun(ctx, r)
|
||||
fresh, _ := s.GetRun(ctx, r.ID)
|
||||
if len(fresh.Artifacts) != 1 || fresh.Artifacts[0].ID != a.ID {
|
||||
t.Fatal("stale worker erased media")
|
||||
}
|
||||
if e := s.AppendAnnotationArtifact(ctx, "org", "human", a); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
fresh, _ = s.GetRun(ctx, r.ID)
|
||||
if len(fresh.Artifacts) != 1 {
|
||||
t.Fatal("duplicate artifact")
|
||||
}
|
||||
}
|
||||
@ -210,6 +210,15 @@ func clone[T any](value T) T {
|
||||
data, _ := json.Marshal(value)
|
||||
var out T
|
||||
_ = json.Unmarshal(data, &out)
|
||||
// The durable processing outbox is intentionally absent from public JSON.
|
||||
// Preserve it when cloning private in-memory records for store operations.
|
||||
if item, ok := any(value).(domain.HumanItemRecord); ok && item.ProcessingJob != nil {
|
||||
copied := any(out).(domain.HumanItemRecord)
|
||||
job := *item.ProcessingJob
|
||||
job.Input = clone(job.Input)
|
||||
copied.ProcessingJob = &job
|
||||
return any(copied).(T)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@ -169,6 +169,9 @@ func (store *MemoryStore) UpdateHumanItemVersioned(_ context.Context, item domai
|
||||
if !ok || existing.Version != expectedVersion {
|
||||
return false, nil
|
||||
}
|
||||
if j := item.ProcessingJob; j != nil && j.Owner != "" && (existing.ProcessingJob == nil || existing.ProcessingJob.ID != j.ID || existing.ProcessingJob.Owner != j.Owner) {
|
||||
return false, nil
|
||||
}
|
||||
item = normalizeHumanItem(item)
|
||||
store.humanItems[item.ID] = clone(item)
|
||||
return true, nil
|
||||
@ -467,7 +470,12 @@ func (store *MongoStore) UpsertHumanItem(ctx context.Context, item domain.HumanI
|
||||
|
||||
func (store *MongoStore) UpdateHumanItemVersioned(ctx context.Context, item domain.HumanItemRecord, expectedVersion int) (bool, error) {
|
||||
item = normalizeHumanItem(item)
|
||||
result, err := store.humanItems.ReplaceOne(ctx, bson.M{"id": item.ID, "version": expectedVersion}, item)
|
||||
filter := bson.M{"id": item.ID, "version": expectedVersion}
|
||||
if j := item.ProcessingJob; j != nil && j.Owner != "" {
|
||||
filter["processingJob.id"] = j.ID
|
||||
filter["processingJob.owner"] = j.Owner
|
||||
}
|
||||
result, err := store.humanItems.ReplaceOne(ctx, filter, item)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
@ -12,71 +12,73 @@ import (
|
||||
)
|
||||
|
||||
type MemoryStore struct {
|
||||
mu sync.Mutex
|
||||
organizations map[string]domain.OrganizationRecord
|
||||
projects map[string]domain.ProjectRecord
|
||||
projectDeletions map[string]domain.ProjectDeletionTask
|
||||
operators map[string]domain.OperatorRecord
|
||||
builtinOperatorDeletions map[string]domain.BuiltinOperatorDeletionRecord
|
||||
workflows map[string]domain.WorkflowRecord
|
||||
workflowRuntimePolicies map[string]domain.WorkflowRuntimePolicyRecord
|
||||
templates map[string]domain.TemplateRecord
|
||||
workflowReleases map[string]domain.WorkflowReleaseRecord
|
||||
projectWorkflowInstances map[string]domain.ProjectWorkflowInstanceRecord
|
||||
runs map[string]domain.RunRecord
|
||||
sourceStates map[string]domain.SourceStateRecord
|
||||
captureRecords map[string]domain.CaptureRecord
|
||||
collectionTasks map[string]domain.CollectionTaskRecord
|
||||
datasetVersions map[string]domain.DatasetVersionRecord
|
||||
dataMetricFacts map[string]domain.DataMetricFactRecord
|
||||
auditTasks map[string]domain.AuditTaskRecord
|
||||
samplingTasks map[string]domain.SamplingTaskRecord
|
||||
annotationWorkgroups map[string]domain.AnnotationWorkgroupRecord
|
||||
annotationMemberships map[string]domain.ProjectAnnotationMembershipRecord
|
||||
suppliers map[string]domain.SupplierRecord
|
||||
projectMembers map[string]domain.ProjectMemberRecord
|
||||
annotationLabels map[string]domain.AnnotationLabelRecord
|
||||
projectAnnotationLabels map[string]domain.ProjectAnnotationLabelsRecord
|
||||
annotationTemplates map[string]domain.AnnotationTemplateRecord
|
||||
annotationTasks map[string]domain.AnnotationTaskRecord
|
||||
annotationRevisions map[string]domain.AnnotationRevisionRecord
|
||||
sliceRevisions map[string]domain.SliceRevisionRecord
|
||||
annotationReviews map[string]domain.AnnotationReviewRecord
|
||||
annotationWorklogEvents map[string]domain.AnnotationWorklogEventRecord
|
||||
organizationPlugins map[string]domain.OrganizationPluginRecord
|
||||
humanBatches map[string]domain.HumanBatchRecord
|
||||
humanItems map[string]domain.HumanItemRecord
|
||||
humanClaimReceipts map[string]HumanClaimReceipt
|
||||
humanStageAttempts map[string]domain.HumanStageAttemptRecord
|
||||
humanEvents map[string]domain.HumanEventRecord
|
||||
humanRejects map[string]domain.HumanRejectRecord
|
||||
releasePartitions map[string]domain.ReleasePartitionRecord
|
||||
deliveryPackages map[string]domain.DeliveryPackageRecord
|
||||
users map[string]domain.UserRecord
|
||||
sessions map[string]domain.SessionRecord
|
||||
storageConnections map[string]domain.StorageConnectionRecord
|
||||
uploadSessions map[string]domain.ObjectStorageUploadSessionRecord
|
||||
mongoConnections map[string]domain.MongoConnectionRecord
|
||||
objectSyncSpaces map[string]domain.ObjectSyncSpaceRecord
|
||||
objectSyncDevices map[string]domain.ObjectSyncDeviceRecord
|
||||
objectSyncAuthorizations map[string]domain.ObjectSyncDeviceAuthorizationRecord
|
||||
objectSyncCredentials map[string]domain.ObjectSyncCredentialRecord
|
||||
objectSyncSessions map[string]domain.ObjectSyncSessionRecord
|
||||
objectSyncItems map[string]domain.ObjectSyncItemRecord
|
||||
objectSyncLeases map[string]domain.ObjectSyncLeaseRecord
|
||||
objectSyncHeads map[string]domain.ObjectSyncObjectHeadRecord
|
||||
objectSyncVersions map[string]domain.ObjectSyncVersionRecord
|
||||
objectSyncAuditEvents map[string]domain.ObjectSyncAuditEvent
|
||||
objectSyncAuditSeals map[string]domain.ObjectSyncAuditSealRecord
|
||||
objectSyncReleases map[string]domain.ObjectSyncReleaseRecord
|
||||
objectSyncClientReleases map[string]domain.ObjectSyncClientReleaseRecord
|
||||
objectSyncReconciles map[string]domain.ObjectSyncReconcileRecord
|
||||
objectSyncIdempotency map[string]domain.ObjectSyncIdempotencyRecord
|
||||
objectSyncRateLimits map[string]domain.ObjectSyncRateLimitRecord
|
||||
databaseCollections map[string][]map[string]any
|
||||
databaseIndexes map[string][]domain.DatabaseIndexSummary
|
||||
settings *domain.SettingsRecord
|
||||
artifactStorageSettings map[string]domain.ArtifactStorageSettings
|
||||
annotationTimelineIndexes map[string]domain.AnnotationTimelineIndex
|
||||
annotationIndexJobs map[string]domain.AnnotationIndexJob
|
||||
mu sync.Mutex
|
||||
organizations map[string]domain.OrganizationRecord
|
||||
projects map[string]domain.ProjectRecord
|
||||
projectDeletions map[string]domain.ProjectDeletionTask
|
||||
operators map[string]domain.OperatorRecord
|
||||
builtinOperatorDeletions map[string]domain.BuiltinOperatorDeletionRecord
|
||||
workflows map[string]domain.WorkflowRecord
|
||||
workflowRuntimePolicies map[string]domain.WorkflowRuntimePolicyRecord
|
||||
templates map[string]domain.TemplateRecord
|
||||
workflowReleases map[string]domain.WorkflowReleaseRecord
|
||||
projectWorkflowInstances map[string]domain.ProjectWorkflowInstanceRecord
|
||||
runs map[string]domain.RunRecord
|
||||
sourceStates map[string]domain.SourceStateRecord
|
||||
captureRecords map[string]domain.CaptureRecord
|
||||
collectionTasks map[string]domain.CollectionTaskRecord
|
||||
datasetVersions map[string]domain.DatasetVersionRecord
|
||||
dataMetricFacts map[string]domain.DataMetricFactRecord
|
||||
auditTasks map[string]domain.AuditTaskRecord
|
||||
samplingTasks map[string]domain.SamplingTaskRecord
|
||||
annotationWorkgroups map[string]domain.AnnotationWorkgroupRecord
|
||||
annotationMemberships map[string]domain.ProjectAnnotationMembershipRecord
|
||||
suppliers map[string]domain.SupplierRecord
|
||||
projectMembers map[string]domain.ProjectMemberRecord
|
||||
annotationLabels map[string]domain.AnnotationLabelRecord
|
||||
projectAnnotationLabels map[string]domain.ProjectAnnotationLabelsRecord
|
||||
annotationTemplates map[string]domain.AnnotationTemplateRecord
|
||||
annotationTasks map[string]domain.AnnotationTaskRecord
|
||||
annotationRevisions map[string]domain.AnnotationRevisionRecord
|
||||
sliceRevisions map[string]domain.SliceRevisionRecord
|
||||
annotationReviews map[string]domain.AnnotationReviewRecord
|
||||
annotationWorklogEvents map[string]domain.AnnotationWorklogEventRecord
|
||||
organizationPlugins map[string]domain.OrganizationPluginRecord
|
||||
humanBatches map[string]domain.HumanBatchRecord
|
||||
humanItems map[string]domain.HumanItemRecord
|
||||
humanClaimReceipts map[string]HumanClaimReceipt
|
||||
humanStageAttempts map[string]domain.HumanStageAttemptRecord
|
||||
humanEvents map[string]domain.HumanEventRecord
|
||||
humanRejects map[string]domain.HumanRejectRecord
|
||||
releasePartitions map[string]domain.ReleasePartitionRecord
|
||||
deliveryPackages map[string]domain.DeliveryPackageRecord
|
||||
users map[string]domain.UserRecord
|
||||
sessions map[string]domain.SessionRecord
|
||||
storageConnections map[string]domain.StorageConnectionRecord
|
||||
uploadSessions map[string]domain.ObjectStorageUploadSessionRecord
|
||||
mongoConnections map[string]domain.MongoConnectionRecord
|
||||
objectSyncSpaces map[string]domain.ObjectSyncSpaceRecord
|
||||
objectSyncDevices map[string]domain.ObjectSyncDeviceRecord
|
||||
objectSyncAuthorizations map[string]domain.ObjectSyncDeviceAuthorizationRecord
|
||||
objectSyncCredentials map[string]domain.ObjectSyncCredentialRecord
|
||||
objectSyncSessions map[string]domain.ObjectSyncSessionRecord
|
||||
objectSyncItems map[string]domain.ObjectSyncItemRecord
|
||||
objectSyncLeases map[string]domain.ObjectSyncLeaseRecord
|
||||
objectSyncHeads map[string]domain.ObjectSyncObjectHeadRecord
|
||||
objectSyncVersions map[string]domain.ObjectSyncVersionRecord
|
||||
objectSyncAuditEvents map[string]domain.ObjectSyncAuditEvent
|
||||
objectSyncAuditSeals map[string]domain.ObjectSyncAuditSealRecord
|
||||
objectSyncReleases map[string]domain.ObjectSyncReleaseRecord
|
||||
objectSyncClientReleases map[string]domain.ObjectSyncClientReleaseRecord
|
||||
objectSyncReconciles map[string]domain.ObjectSyncReconcileRecord
|
||||
objectSyncIdempotency map[string]domain.ObjectSyncIdempotencyRecord
|
||||
objectSyncRateLimits map[string]domain.ObjectSyncRateLimitRecord
|
||||
databaseCollections map[string][]map[string]any
|
||||
databaseIndexes map[string][]domain.DatabaseIndexSummary
|
||||
settings *domain.SettingsRecord
|
||||
artifactStorageSettings map[string]domain.ArtifactStorageSettings
|
||||
}
|
||||
|
||||
func NewMemoryStore() *MemoryStore {
|
||||
@ -570,6 +572,16 @@ func (store *MemoryStore) DeleteProject(_ context.Context, id string) (bool, err
|
||||
return false, nil
|
||||
}
|
||||
delete(store.projects, id)
|
||||
for k, v := range store.annotationTimelineIndexes {
|
||||
if v.ProjectID == id {
|
||||
delete(store.annotationTimelineIndexes, k)
|
||||
}
|
||||
}
|
||||
for k, v := range store.annotationIndexJobs {
|
||||
if v.ProjectID == id {
|
||||
delete(store.annotationIndexJobs, k)
|
||||
}
|
||||
}
|
||||
|
||||
workflowIDs := make([]string, 0)
|
||||
for workflowID, workflow := range store.workflows {
|
||||
@ -965,6 +977,16 @@ func (store *MemoryStore) deleteWorkflowLocked(id string) bool {
|
||||
if _, ok := store.workflows[id]; !ok {
|
||||
return false
|
||||
}
|
||||
for k, v := range store.annotationTimelineIndexes {
|
||||
if v.WorkflowID == id {
|
||||
delete(store.annotationTimelineIndexes, k)
|
||||
}
|
||||
}
|
||||
for k, v := range store.annotationIndexJobs {
|
||||
if v.WorkflowID == id {
|
||||
delete(store.annotationIndexJobs, k)
|
||||
}
|
||||
}
|
||||
annotationTaskIDs := map[string]struct{}{}
|
||||
humanBatchIDs := map[string]struct{}{}
|
||||
humanItemIDs := map[string]struct{}{}
|
||||
@ -1824,6 +1846,9 @@ func (store *MemoryStore) UpsertRun(_ context.Context, run domain.RunRecord) err
|
||||
}
|
||||
}
|
||||
computeRunDuration(&run)
|
||||
if old, ok := store.runs[run.ID]; ok {
|
||||
preserveAnnotationArtifacts(&run, old)
|
||||
}
|
||||
run.CaptureDirectories = domain.CaptureDirectoriesFromRun(run)
|
||||
store.runs[run.ID] = clone(run)
|
||||
return nil
|
||||
|
||||
@ -447,6 +447,8 @@ func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, er
|
||||
filter bson.M
|
||||
}{
|
||||
{store.annotationTasks, bson.M{"projectId": id}},
|
||||
{store.db.Collection("annotationTimelineIndexes"), bson.M{"projectId": id}},
|
||||
{store.db.Collection("annotationIndexJobs"), bson.M{"projectId": id}},
|
||||
{store.humanRejects, bson.M{"projectId": id}},
|
||||
{store.humanItems, bson.M{"projectId": id}},
|
||||
{store.humanBatches, bson.M{"projectId": id}},
|
||||
@ -777,6 +779,8 @@ func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, e
|
||||
}{
|
||||
{store.projectWorkflowInstances, bson.M{"workflowId": id}},
|
||||
{store.annotationTasks, bson.M{"workflowId": id}},
|
||||
{store.db.Collection("annotationTimelineIndexes"), bson.M{"workflowId": id}},
|
||||
{store.db.Collection("annotationIndexJobs"), bson.M{"workflowId": id}},
|
||||
{store.humanRejects, bson.M{"workflowId": id}},
|
||||
{store.humanItems, bson.M{"workflowId": id}},
|
||||
{store.humanBatches, bson.M{"workflowId": id}},
|
||||
@ -1782,14 +1786,10 @@ func (store *MongoStore) UpsertRun(ctx context.Context, item domain.RunRecord) e
|
||||
}
|
||||
computeRunDuration(&item)
|
||||
stored := compactRunForStorage(item)
|
||||
update := bson.M{"$set": stored}
|
||||
unsetFields := runStorageFieldsToUnset(item)
|
||||
if len(unsetFields) > 0 {
|
||||
unset := bson.M{}
|
||||
for _, field := range unsetFields {
|
||||
unset[field] = ""
|
||||
}
|
||||
update["$unset"] = unset
|
||||
update, err := annotationSafeRunUpdate(stored, unsetFields)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = store.runs.UpdateOne(ctx, bson.M{"id": item.ID}, update, options.Update().SetUpsert(allow))
|
||||
return err
|
||||
|
||||
@ -49,6 +49,16 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
indexesByCollection := map[*mongo.Collection][]mongo.IndexModel{
|
||||
store.db.Collection("annotationTimelineIndexes"): {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("projectId", bson.D{{Key: "projectId", Value: 1}}),
|
||||
index("workflowId", bson.D{{Key: "workflowId", Value: 1}}),
|
||||
},
|
||||
store.db.Collection("annotationIndexJobs"): {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("status_nextAttemptAt", bson.D{{Key: "status", Value: 1}, {Key: "nextAttemptAt", Value: 1}}),
|
||||
index("status_leaseUntil", bson.D{{Key: "status", Value: 1}, {Key: "leaseUntil", Value: 1}}),
|
||||
},
|
||||
store.db.Collection("humanClaimRequests"): {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organization_project", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}}),
|
||||
@ -197,6 +207,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
|
||||
index("organizationId_workflowId_status", bson.D{{Key: "organizationId", Value: 1}, {Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}),
|
||||
},
|
||||
store.humanItems: {
|
||||
index("processingJob_status_lease", bson.D{{Key: "processingJob.status", Value: 1}, {Key: "processingJob.leaseUntil", Value: 1}, {Key: "processingJob.createdAt", Value: 1}}),
|
||||
index("organizationId_legacyTaskId", bson.D{{Key: "organizationId", Value: 1}, {Key: "legacyTaskId", Value: 1}}),
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
uniqueIndex("organizationId_batchId_subjectItemKey_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "batchId", Value: 1}, {Key: "subject.itemKey", Value: 1}}),
|
||||
|
||||
@ -14,6 +14,7 @@ import (
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/objectsync"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type deliveryPrefixLister interface {
|
||||
@ -177,31 +178,58 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
|
||||
}
|
||||
return paths[i] < paths[j]
|
||||
})
|
||||
entries := []directoryManifestEntry{}
|
||||
var total int64
|
||||
for _, path := range paths {
|
||||
entries := make([]directoryManifestEntry, len(paths))
|
||||
publishFile := func(ctx context.Context, i int, path string) error {
|
||||
relative, e := filepath.Rel(artifact.Path, path)
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
return e
|
||||
}
|
||||
relative = filepath.ToSlash(relative)
|
||||
if strings.HasPrefix(relative, "../") || relative == "manifest.json" || relative == ".delivery-owner.json" {
|
||||
return artifact, fmt.Errorf("无效交付成员 %s", relative)
|
||||
return fmt.Errorf("无效交付成员 %s", relative)
|
||||
}
|
||||
stage := stageRoot + "/files/" + relative
|
||||
meta, sha, e := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, path)
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
return e
|
||||
}
|
||||
key := target.Prefix + "/" + relative
|
||||
if e = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); e != nil {
|
||||
return artifact, e
|
||||
return e
|
||||
}
|
||||
if e = publisher.Delete(ctx, *connection, target.Bucket, stage); e != nil {
|
||||
return artifact, e
|
||||
return e
|
||||
}
|
||||
entries = append(entries, directoryManifestEntry{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha})
|
||||
total += meta.SizeBytes
|
||||
entries[i] = directoryManifestEntry{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}
|
||||
return nil
|
||||
}
|
||||
parallel := 1
|
||||
switch publisher.(type) {
|
||||
case objectsync.BOSPublisher, *objectsync.BOSPublisher:
|
||||
parallel = 4
|
||||
}
|
||||
group, gctx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(parallel)
|
||||
for i, path := range paths {
|
||||
if filepath.Base(path) == "packages-manifest.json" {
|
||||
continue
|
||||
}
|
||||
group.Go(func() error { return publishFile(gctx, i, path) })
|
||||
}
|
||||
if e := group.Wait(); e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
// A visible package manifest must not precede its media files.
|
||||
for i, path := range paths {
|
||||
if filepath.Base(path) == "packages-manifest.json" {
|
||||
if e := publishFile(ctx, i, path); e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
}
|
||||
}
|
||||
var total int64
|
||||
for _, entry := range entries {
|
||||
total += entry.SizeBytes
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return artifact, fmt.Errorf("交付目录为空")
|
||||
|
||||
52
k8s/annotation-processor.template.yaml
Normal file
52
k8s/annotation-processor.template.yaml
Normal file
@ -0,0 +1,52 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: droboticflow-annotation-processor
|
||||
namespace: __NAMESPACE__
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels: {app: droboticflow-annotation-processor}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: droboticflow-annotation-processor}
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 30
|
||||
imagePullSecrets:
|
||||
- name: __K8S_IMAGE_PULL_SECRET_NAME__
|
||||
containers:
|
||||
- name: processor
|
||||
image: __API_IMAGE__
|
||||
command: [/usr/local/bin/droboticflow-api]
|
||||
env:
|
||||
- {name: ANNOTATION_PROCESSOR_MODE, value: processor}
|
||||
- {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "2"}
|
||||
- {name: ANNOTATION_INDEX_BACKFILL, value: "1"}
|
||||
- {name: WORKSPACE_MOUNT_PATH, value: "__WORKSPACE_MOUNT_PATH__"}
|
||||
- {name: GLOBAL_BOS_ENDPOINT, value: "__BOS_ENDPOINT__"}
|
||||
- {name: ANNOTATION_WORKSPACE_CONNECTION_ID, value: "__ANNOTATION_WORKSPACE_CONNECTION_ID__"}
|
||||
- {name: ANNOTATION_WORKSPACE_BUCKET, value: "__ANNOTATION_WORKSPACE_BUCKET__"}
|
||||
- name: MONGO_DB
|
||||
valueFrom:
|
||||
configMapKeyRef: {name: droboticflow-config, key: MONGO_DB}
|
||||
- name: MONGO_URI
|
||||
valueFrom:
|
||||
secretKeyRef: {name: droboticflow-mongo-secret, key: MONGO_URI}
|
||||
ports:
|
||||
- {name: health, containerPort: 3003}
|
||||
readinessProbe:
|
||||
httpGet: {path: /ready, port: health}
|
||||
periodSeconds: 5
|
||||
resources:
|
||||
requests: {cpu: "1", memory: 512Mi}
|
||||
limits: {cpu: "4", memory: 8Gi}
|
||||
volumeMounts:
|
||||
- {name: workspace, mountPath: "__WORKSPACE_MOUNT_PATH__"}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: workspace
|
||||
persistentVolumeClaim: {claimName: __WORKSPACE_PVC_NAME__}
|
||||
- name: scratch
|
||||
emptyDir: {sizeLimit: 20Gi}
|
||||
@ -130,6 +130,7 @@ data:
|
||||
DROSYNC_COORDINATOR_PORT: "3003"
|
||||
DROBOTICFLOW_OBJECT_SYNC_ENABLED: "__DROBOTICFLOW_OBJECT_SYNC_ENABLED__"
|
||||
MONGO_DB: "__MONGO_DB__"
|
||||
ANNOTATION_PROCESSOR_MODE: "__ANNOTATION_PROCESSOR_MODE__"
|
||||
MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__"
|
||||
MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__"
|
||||
WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__"
|
||||
@ -288,6 +289,9 @@ spec:
|
||||
key: MONGO_URI
|
||||
- name: DIST_DIR
|
||||
value: /app/dist
|
||||
- name: ANNOTATION_PROCESSOR_MODE
|
||||
valueFrom:
|
||||
configMapKeyRef: {name: droboticflow-config, key: ANNOTATION_PROCESSOR_MODE}
|
||||
- name: K8S_NAMESPACE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
|
||||
@ -28,6 +28,11 @@ BOS_ENDPOINT="${BOS_ENDPOINT:-}"
|
||||
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
||||
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
||||
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
|
||||
ANNOTATION_PROCESSOR_ENABLED="${ANNOTATION_PROCESSOR_ENABLED:-false}"
|
||||
ANNOTATION_PROCESSOR_MODE=inprocess
|
||||
if [[ "$ANNOTATION_PROCESSOR_ENABLED" == "true" ]]; then ANNOTATION_PROCESSOR_MODE=external; fi
|
||||
ANNOTATION_WORKSPACE_CONNECTION_ID="${ANNOTATION_WORKSPACE_CONNECTION_ID:-}"
|
||||
ANNOTATION_WORKSPACE_BUCKET="${ANNOTATION_WORKSPACE_BUCKET:-}"
|
||||
API_NODE_PORT="${API_NODE_PORT:-30080}"
|
||||
API_SERVICE_TYPE="${API_SERVICE_TYPE:-NodePort}"
|
||||
API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
|
||||
@ -122,6 +127,10 @@ OPERATOR_COMMAND_ALLOWLIST_REGEX_BLOCK="$(render_block_scalar "$OPERATOR_COMMAND
|
||||
OPERATOR_COMMAND_DENYLIST_REGEX_BLOCK="$(render_block_scalar "$OPERATOR_COMMAND_DENYLIST_REGEX")"
|
||||
|
||||
export \
|
||||
ANNOTATION_PROCESSOR_ENABLED \
|
||||
ANNOTATION_PROCESSOR_MODE \
|
||||
ANNOTATION_WORKSPACE_CONNECTION_ID \
|
||||
ANNOTATION_WORKSPACE_BUCKET \
|
||||
TEMPLATE \
|
||||
K8S_NAMESPACE \
|
||||
MONGO_DB \
|
||||
@ -185,7 +194,12 @@ import re
|
||||
import sys
|
||||
|
||||
template = pathlib.Path(os.environ["TEMPLATE"]).read_text(encoding="utf-8")
|
||||
if os.environ["ANNOTATION_PROCESSOR_ENABLED"] == "true":
|
||||
template += pathlib.Path(os.environ["TEMPLATE"]).with_name("annotation-processor.template.yaml").read_text(encoding="utf-8")
|
||||
replacements = {
|
||||
"__ANNOTATION_PROCESSOR_MODE__": os.environ["ANNOTATION_PROCESSOR_MODE"],
|
||||
"__ANNOTATION_WORKSPACE_CONNECTION_ID__": os.environ["ANNOTATION_WORKSPACE_CONNECTION_ID"],
|
||||
"__ANNOTATION_WORKSPACE_BUCKET__": os.environ["ANNOTATION_WORKSPACE_BUCKET"],
|
||||
"__NAMESPACE__": os.environ["K8S_NAMESPACE"],
|
||||
"__MONGO_DB__": os.environ["MONGO_DB"],
|
||||
"__MONGO_CONNECT_RETRIES__": os.environ["MONGO_CONNECT_RETRIES"],
|
||||
|
||||
@ -662,7 +662,7 @@ export default function AnnotationOperations() {
|
||||
projectId ? annotationOperationsApi.claimCount(projectId).catch(() => null) : Promise.resolve(null),
|
||||
]);
|
||||
const owned = mine.filter((i) => i.legacyTaskId);
|
||||
const details = await Promise.all(owned.map((i) => annotationOperationsApi.getTask(i.legacyTaskId)));
|
||||
const details = await Promise.all(owned.map((i) => annotationOperationsApi.getTaskSummary(i.legacyTaskId)));
|
||||
if (requestID !== projectLoadRequestRef.current) return;
|
||||
setProjectMembers(members);
|
||||
setTemplates(nextTemplates);
|
||||
|
||||
@ -87,6 +87,8 @@ export const annotationOperationsApi = {
|
||||
) => (await apiRequest<{ items: AnnotationTaskRecord[] }>(`/api/plugins/annotation-qa/tasks${query(filters)}`)).items,
|
||||
getTask: (id: string, signal?: AbortSignal) =>
|
||||
apiRequest<AnnotationTaskDetail>(`/api/plugins/annotation-qa/tasks/${id}`, { signal }),
|
||||
getTaskSummary: (id: string) =>
|
||||
apiRequest<{ task: AnnotationTaskRecord }>(`/api/plugins/annotation-qa/tasks/${id}?view=summary`),
|
||||
getTaskStatus: (id: string, signal?: AbortSignal) =>
|
||||
apiRequest<{
|
||||
taskId: string;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user