package api import ( "bufio" "bytes" "compress/gzip" "context" "crypto/sha256" "droboticflow/internal/domain" "droboticflow/internal/objectsync" "droboticflow/internal/store" "encoding/hex" "encoding/json" "fmt" "io" "log" "math" "os" "os/exec" "path/filepath" "strconv" "strings" "time" ) const sourceIndexSchema = "droboticflow.source-slice-index/v1" const sourceCertificateSchema = "droboticflow.annotation-source-certificate/v1" const processingSourceSchema = "droboticflow.annotation-processing-index/v1" func processingSourceID(org, fp string) string { return "annotation-processing-v1:" + org + ":" + fp } type annotationPinnedSource struct { ArtifactID string `json:"artifactId"` Path string `json:"path"` Version string `json:"version"` SHA256 string `json:"sha256"` Bytes int64 `json:"bytes"` } type annotationSourceCertificate struct { Schema string `json:"schemaVersion"` Fingerprint string `json:"sourceFingerprint"` TaskID string `json:"taskId"` RunID string `json:"runId"` CollectionID string `json:"collectionId"` Sources map[string]annotationPinnedSource `json:"sources"` Timeline persistedAnnotationTimeline `json:"timeline"` LeftPTS []int64 `json:"leftPTS"` RightPTS []int64 `json:"rightPTS"` IMURowsBeforeBoundary []int64 `json:"imuRowsBeforeBoundary"` LeftDuration float64 `json:"leftDuration"` RightDuration float64 `json:"rightDuration"` } func sourceCertificateID(org, fingerprint string) string { return "annotation-source-v1:" + org + ":" + fingerprint } func annotationSourceInputs(task domain.AnnotationTaskRecord, run domain.RunRecord) (map[string]*domain.ExecutionArtifact, error) { for _, n := range run.NodeExecutions { if n.NodeID != task.NodeID { continue } left := artifactByID(artifactsForTarget(n, "fullLeftVideos"), task.ArtifactID) if left == nil { left = artifactByID(n.InputArtifacts, task.ArtifactID) } if left == nil { return nil, fmt.Errorf("left source unavailable") } collection := artifactCollectionID(*left) inputs := map[string]*domain.ExecutionArtifact{"leftVideo": left, "rightVideo": matchingArtifact(artifactsForTarget(n, "fullRightVideos"), collection, "right", ".mp4"), "leftPts": matchingArtifact(artifactsForTarget(n, "leftTimestamps"), collection, "left", "pts"), "rightPts": matchingArtifact(artifactsForTarget(n, "rightTimestamps"), collection, "right", "pts"), "imu": matchingArtifact(artifactsForTarget(n, "imu"), collection, "imu", ".csv")} for kind, a := range inputs { if a == nil { return nil, fmt.Errorf("%s source unavailable", kind) } } if inputs["rightVideo"].ID != task.RightArtifactID { return nil, fmt.Errorf("right source identity mismatch") } return inputs, nil } return nil, fmt.Errorf("source execution unavailable") } func decodeSourceCertificate(record *domain.AnnotationTimelineIndex, fingerprint string) (*annotationSourceCertificate, error) { return decodeAnnotationSourceIndex(record, fingerprint, sourceCertificateSchema) } func decodeAnnotationSourceIndex(record *domain.AnnotationTimelineIndex, fingerprint, schema string) (*annotationSourceCertificate, error) { if record == nil { return nil, nil } sum := sha256.Sum256(record.Body) if hex.EncodeToString(sum[:]) != record.SHA256 { return nil, fmt.Errorf("source certificate checksum mismatch") } z, e := gzip.NewReader(bytes.NewReader(record.Body)) if e != nil { return nil, e } defer z.Close() b, e := io.ReadAll(io.LimitReader(z, 8<<20+1)) if e != nil || len(b) > 8<<20 { return nil, fmt.Errorf("source certificate too large") } var c annotationSourceCertificate if e = json.Unmarshal(b, &c); e != nil { return nil, e } n := len(c.Timeline.Stamps) if c.Schema != schema || c.Fingerprint != fingerprint || c.Timeline.Fingerprint != fingerprint || n < 2 || len(c.Timeline.Seconds) != n || len(c.Timeline.RightSeconds) != n || len(c.Timeline.Left) != n || len(c.Timeline.Right) != n || len(c.IMURowsBeforeBoundary) != n || len(c.LeftPTS) != c.Timeline.LeftFrameCount || len(c.RightPTS) != c.Timeline.RightFrameCount || len(c.Sources) != 5 { return nil, fmt.Errorf("invalid source certificate") } for i := 0; i < n; i++ { if c.Timeline.Left[i] < 0 || c.Timeline.Left[i] > len(c.LeftPTS) || c.Timeline.Right[i] < 0 || c.Timeline.Right[i] > len(c.RightPTS) || c.IMURowsBeforeBoundary[i] < 0 || math.IsNaN(c.Timeline.Seconds[i]) || math.IsInf(c.Timeline.Seconds[i], 0) || math.IsNaN(c.Timeline.RightSeconds[i]) || math.IsInf(c.Timeline.RightSeconds[i], 0) { return nil, fmt.Errorf("invalid source certificate bounds") } if i > 0 && (c.Timeline.Stamps[i] <= c.Timeline.Stamps[i-1] || c.Timeline.Seconds[i] <= c.Timeline.Seconds[i-1] || c.Timeline.RightSeconds[i] <= c.Timeline.RightSeconds[i-1] || c.IMURowsBeforeBoundary[i] < c.IMURowsBeforeBoundary[i-1]) { return nil, fmt.Errorf("invalid source certificate ordering") } } for _, pts := range [][]int64{c.LeftPTS, c.RightPTS} { for i, value := range pts { if value <= 0 || i > 0 && value <= pts[i-1] { return nil, fmt.Errorf("invalid source PTS ordering") } } } return &c, nil } // A certificate is specific to authoritative object versions. Single-flight and // a separate two-slot semaphore prevent many light revision jobs from spawning // simultaneous full decodes. Only compact indexes, never IMU rows, are cached. func (s *server) annotationSourceCertificate(ctx context.Context, t domain.AnnotationTaskRecord, run domain.RunRecord, fingerprint string) (*annotationSourceCertificate, error) { return s.annotationSourceIndexForMode(ctx, t, run, fingerprint, store.AnnotationSourceAuditAdvisory()) } func (s *server) annotationSourceIndexForMode(ctx context.Context, t domain.AnnotationTaskRecord, run domain.RunRecord, fingerprint string, processing bool) (*annotationSourceCertificate, error) { cacheKey := t.OrganizationID + ":" + fingerprint if processing { cacheKey = "processing:" + cacheKey } v, e := s.annotationSourceCertificates.get(ctx, cacheKey, func(ctx context.Context) (map[string]any, error) { db, ok := s.store.(store.AnnotationProcessingStore) if !ok { return nil, fmt.Errorf("source certificate store unavailable") } record, e := db.GetAnnotationTimelineIndex(ctx, sourceCertificateID(t.OrganizationID, fingerprint)) if e != nil { return nil, e } validScope := func(r *domain.AnnotationTimelineIndex) bool { return r != nil && r.OrganizationID == t.OrganizationID && r.ProjectID == t.ProjectID && r.RunID == t.RunID } if cert, e := decodeSourceCertificate(record, fingerprint); e == nil && cert != nil && validScope(record) { return map[string]any{"certificate": cert}, nil } if processing { record, e = db.GetAnnotationTimelineIndex(ctx, processingSourceID(t.OrganizationID, fingerprint)) if e != nil { return nil, e } if cert, e := decodeAnnotationSourceIndex(record, fingerprint, processingSourceSchema); e == nil && cert != nil && validScope(record) { return map[string]any{"certificate": cert}, nil } } s.annotationValidationOnce.Do(func() { s.annotationValidationSlots = make(chan struct{}, 2) }) select { case s.annotationValidationSlots <- struct{}{}: case <-ctx.Done(): return nil, ctx.Err() } defer func() { <-s.annotationValidationSlots }() cert, e := s.buildAnnotationSourceIndex(ctx, t, run, fingerprint, !processing) if e != nil { return nil, e } b, e := json.Marshal(cert) if e != nil { return nil, e } if len(b) > 8<<20 { return nil, fmt.Errorf("source index exceeds size limit") } var compressed bytes.Buffer z := gzip.NewWriter(&compressed) if _, e = z.Write(b); e != nil { return nil, e } if e = z.Close(); e != nil { return nil, e } sum := sha256.Sum256(compressed.Bytes()) id := sourceCertificateID(t.OrganizationID, fingerprint) if processing { id = processingSourceID(t.OrganizationID, fingerprint) } e = db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: id, OrganizationID: t.OrganizationID, ProjectID: t.ProjectID, WorkflowID: t.WorkflowID, RunID: t.RunID, Body: compressed.Bytes(), SHA256: hex.EncodeToString(sum[:]), CreatedAt: nowISO()}) return map[string]any{"certificate": cert}, e }) if e != nil { return nil, e } cert := v["certificate"].(*annotationSourceCertificate) if cert.RunID != t.RunID { return nil, fmt.Errorf("source certificate run mismatch") } return cert, nil } // sourceClockTolerance accepts container rounding but nothing larger: a frame // time stored in a coarse time base (e.g. 1/1000 for 30fps) is up to half a // tick from the ideal fixed-rate time. Fine time bases keep the 0.1ms bound. func sourceClockTolerance(timeBase string) (float64, error) { tick, err := sourceClockRatio(timeBase) if err != nil { return 0, err } return math.Max(0.0001, tick/2+1e-9), nil } func sourceClockRatio(value string) (float64, error) { numerator, denominator, found := strings.Cut(strings.TrimSpace(value), "/") n, e1 := strconv.ParseFloat(numerator, 64) d, e2 := strconv.ParseFloat(denominator, 64) ratio := n / d if !found || e1 != nil || e2 != nil || n <= 0 || d <= 0 || math.IsInf(n, 0) || math.IsInf(d, 0) || math.IsNaN(ratio) || math.IsInf(ratio, 0) || ratio <= 0 { return 0, fmt.Errorf("invalid source video clock ratio %q", value) } return ratio, nil } type sourceVideoTiming struct { TimeBase string `json:"time_base"` FrameRate string `json:"r_frame_rate"` } func ffprobeSourceTiming(ctx context.Context, path string) (sourceVideoTiming, error) { output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=time_base,r_frame_rate", "-of", "json", path).CombinedOutput() if err != nil { return sourceVideoTiming{}, fmt.Errorf("ffprobe timing %s: %s", filepath.Base(path), strings.TrimSpace(string(output))) } var metadata struct { Streams []sourceVideoTiming `json:"streams"` } if err = json.Unmarshal(output, &metadata); err != nil || len(metadata.Streams) != 1 { return sourceVideoTiming{}, fmt.Errorf("invalid source video timing metadata: %s", filepath.Base(path)) } return metadata.Streams[0], nil } func validateSourceMediaClock(ctx context.Context, path string, count int, duration float64) error { if count < 2 || duration <= 0 || math.IsNaN(duration) || math.IsInf(duration, 0) { return fmt.Errorf("invalid source media duration/count") } timing, e := ffprobeSourceTiming(ctx, path) if e != nil { return e } tolerance, e := sourceClockTolerance(timing.TimeBase) if e != nil { return e } tick, _ := sourceClockRatio(timing.TimeBase) // validated by sourceClockTolerance rate, e := sourceClockRatio(timing.FrameRate) if e != nil { return e } period := 1 / rate // r_frame_rate is only a candidate cadence: verify every decoded frame // against it below, rather than trusting the metadata or deriving a cadence // from rounded total duration. MP4 edit lists additionally round the // presentation endpoint in the movie clock (often 1ms, even for a video // track at 1/15360). This never relaxes the decoded frame checks below. if math.IsInf(period, 0) || math.Abs(duration-float64(count)*period) > sourceClockEndTolerance(path, tick, period) { return fmt.Errorf("source video duration does not match its fixed frame cadence") } // Integer timestamps preserve half-tick boundaries; ffprobe's *_time // decimal output rounds to microseconds and can exceed the tolerance. // Bound decoding to two threads within the validator CPU budget. // Still decode every frame and validate its original presentation timestamp. cmd := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-threads", "2", "-select_streams", "v:0", "-show_frames", "-show_entries", "frame=best_effort_timestamp", "-of", "csv=p=0", path) stdout, e := cmd.StdoutPipe() if e != nil { return e } // Decoder diagnostics are not returned; do not accumulate unbounded stderr // from malformed media during a long-running validation. cmd.Stderr = io.Discard if e = cmd.Start(); e != nil { return e } scanner := bufio.NewScanner(stdout) scanner.Buffer(make([]byte, 4096), 1<<20) n := 0 previous := -1.0 var clockErr error for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" { continue } text := strings.SplitN(line, ",", 2)[0] ticks, err := strconv.ParseInt(text, 10, 64) stamp := float64(ticks) * tick if err != nil || math.IsNaN(stamp) || math.IsInf(stamp, 0) || stamp <= previous || math.Abs(stamp-float64(n)*period) > tolerance { clockErr = fmt.Errorf("源视频时间轴非连续固定帧序,不能使用源引用切片(帧 %d)", n) _ = cmd.Process.Kill() break } previous = stamp n++ } readErr := scanner.Err() if readErr != nil { _ = cmd.Process.Kill() } waitErr := cmd.Wait() if clockErr != nil { return clockErr } if readErr != nil { return readErr } if waitErr != nil { return fmt.Errorf("source decode failed: %w", waitErr) } if n != count { return fmt.Errorf("source decoded frame/PTS mismatch: %d/%d", n, count) } return nil } func (s *server) buildAnnotationSourceCertificate(ctx context.Context, t domain.AnnotationTaskRecord, run domain.RunRecord, fingerprint string) (*annotationSourceCertificate, error) { return s.buildAnnotationSourceIndex(ctx, t, run, fingerprint, true) } func (s *server) buildAnnotationSourceIndex(ctx context.Context, t domain.AnnotationTaskRecord, run domain.RunRecord, fingerprint string, fullAudit bool) (_ *annotationSourceCertificate, buildErr error) { started, phaseStarted := time.Now(), time.Now() phase := "fetch" durations := map[string]int64{} nextPhase := func(next string) { durations[phase] = time.Since(phaseStarted).Milliseconds() phase, phaseStarted = next, time.Now() } defer func() { durations[phase] = time.Since(phaseStarted).Milliseconds() log.Printf("[annotation-validator] source_timing task=%s total_ms=%d phase_ms=%v success=%t", t.ID, time.Since(started).Milliseconds(), durations, buildErr == nil) }() inputs, e := annotationSourceInputs(t, run) if e != nil { return nil, e } stage, e := os.MkdirTemp("", "annotation-source-") if e != nil { return nil, e } defer os.RemoveAll(stage) paths := map[string]string{} versions := map[string]string{} for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} { path, e := s.safeMaterializationSourcePath(inputs[kind]) if e != nil { return nil, e } versions[kind], e = s.annotationFileVersion(ctx, t.OrganizationID, path) if e != nil { return nil, e } ext := ".csv" if strings.HasSuffix(kind, "Video") { ext = ".mp4" } local, e := s.localAnnotationSource(ctx, t.OrganizationID, path, filepath.Join(stage, kind+ext)) if e != nil { return nil, e } paths[kind] = local } nextPhase("media_clock") left, e := readHardwarePTS(paths["leftPts"]) if e != nil { return nil, e } right, e := readHardwarePTS(paths["rightPts"]) if e != nil { return nil, e } ld, e := ffprobeDurationSeconds(ctx, paths["leftVideo"]) if e != nil { return nil, e } rd, e := ffprobeDurationSeconds(ctx, paths["rightVideo"]) if e != nil { return nil, e } validateMedia := validateSourceMediaAvailability schema := processingSourceSchema if fullAudit { validateMedia = validateSourceMediaClock schema = sourceCertificateSchema } if e = validateMedia(ctx, paths["leftVideo"], len(left), ld); e != nil { return nil, e } if e = validateMedia(ctx, paths["rightVideo"], len(right), rd); e != nil { return nil, e } nextPhase("sensor_index") // The existing builder validates ordered sensor pairs and the exact common // range. Its compact grid is the same seam used by editor validation. timeline, e := s.annotationFrameTimelineForTask(ctx, t, "left") if e != nil { return nil, e } b, _ := json.Marshal(timeline) var grid persistedAnnotationTimeline if e = json.Unmarshal(b, &grid); e != nil { return nil, e } if grid.Fingerprint != fingerprint { return nil, fmt.Errorf("source changed during validation") } c := &annotationSourceCertificate{Schema: schema, Fingerprint: fingerprint, TaskID: t.ID, RunID: t.RunID, CollectionID: artifactCollectionID(*inputs["leftVideo"]), Sources: map[string]annotationPinnedSource{}, Timeline: grid, LeftDuration: ld, RightDuration: rd, IMURowsBeforeBoundary: make([]int64, len(grid.Stamps))} for _, row := range left { c.LeftPTS = append(c.LeftPTS, row.TimestampUS) } for _, row := range right { c.RightPTS = append(c.RightPTS, row.TimestampUS) } position := 0 var count int64 _, e = scanHardwareIMU(ctx, paths["imu"], func(row hardwareIMURow) { for position < len(grid.Stamps) && grid.Stamps[position] <= row.TimestampUS { c.IMURowsBeforeBoundary[position] = count position++ } count++ }) if e != nil { return nil, e } for position < len(grid.Stamps) { c.IMURowsBeforeBoundary[position] = count position++ } nextPhase("pin") for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} { original, e := s.safeMaterializationSourcePath(inputs[kind]) if e != nil { return nil, e } current, e := s.annotationFileVersion(ctx, t.OrganizationID, original) if e != nil || current != versions[kind] { return nil, fmt.Errorf("源文件在校验期间发生变化") } digest, size, e := sha256File(paths[kind]) if e != nil { return nil, e } ext := ".csv" if strings.HasSuffix(kind, "Video") { ext = ".mp4" } target := filepath.Join(s.workspaceMountPath, "system", "runs", t.RunID, "annotation-sources", fingerprint, kind+ext) if e = s.pinAnnotationSource(ctx, t.OrganizationID, original, versions[kind], paths[kind], target, digest, size); e != nil { return nil, e } version, e := s.annotationFileVersion(ctx, t.OrganizationID, target) if e != nil { return nil, e } c.Sources[kind] = annotationPinnedSource{ArtifactID: inputs[kind].ID, Path: target, Version: version, SHA256: digest, Bytes: size} } after, e := s.annotationSourceFingerprint(ctx, t, run) if e != nil { return nil, e } if after != fingerprint { return nil, fmt.Errorf("源文件在固定版本期间发生变化") } return c, nil } // Each source version owns five immutable objects, shared by all its revisions. // BOS copies are conditional on source ETag and destination nonexistence. Local // publication uses an atomic hard link of a fully verified sibling temp file. func (s *server) pinAnnotationSource(ctx context.Context, org, original, expectedVersion, local, target, digest string, size int64) error { c, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, original, false) if e != nil { return e } if c != nil { to, tb, tk, e := s.annotationWorkspaceObject(ctx, org, target) if e != nil { return e } if to == nil || to.ID != c.ID || tb != bucket { return fmt.Errorf("cross-workspace source pin") } publisher := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint} old, e := publisher.Head(ctx, *c, bucket, tk) if e != nil { return e } if old.Exists { if old.SizeBytes == size && annotationObjectSHA(old) == digest { return nil } return fmt.Errorf("pinned source conflict") } meta, e := publisher.Head(ctx, *c, bucket, key) if e != nil { return e } version := fmt.Sprintf("bos:%s:%s:%s:%s:%d", c.ID, bucket, meta.ETag, meta.VersionID, meta.SizeBytes) if !meta.Exists || meta.SizeBytes != size || meta.ETag == "" || version != expectedVersion { return fmt.Errorf("source pin version unavailable") } _, e = publisher.PublishConditional(ctx, *c, bucket, key, tk, size, meta.ETag, "", "*", map[string]string{"sha256": digest}) if e != nil { old, he := publisher.Head(ctx, *c, bucket, tk) if he == nil && old.Exists && old.SizeBytes == size && annotationObjectSHA(old) == digest { return nil } } return e } if e = os.MkdirAll(filepath.Dir(target), 0755); e != nil { return e } if got, n, e := sha256File(target); e == nil { if got == digest && n == size { return nil } return fmt.Errorf("pinned source conflict") } temp, e := os.CreateTemp(filepath.Dir(target), ".pin-") if e != nil { return e } name := temp.Name() defer os.Remove(name) _ = temp.Close() if e = copyAnnotationFile(ctx, local, name, digest); e != nil { return e } if e = os.Link(name, target); e != nil { got, n, readErr := sha256File(target) if readErr == nil && got == digest && n == size { return nil } return e } return nil }