droboticflow/internal/api/annotation_source_certificate.go

522 lines
19 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package api
import (
"bufio"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"droboticflow/internal/store"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"log"
"math"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
)
const sourceIndexSchema = "droboticflow.source-slice-index/v1"
const sourceCertificateSchema = "droboticflow.annotation-source-certificate/v1"
type annotationPinnedSource struct {
ArtifactID string `json:"artifactId"`
Path string `json:"path"`
Version string `json:"version"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
}
type annotationSourceCertificate struct {
Schema string `json:"schemaVersion"`
Fingerprint string `json:"sourceFingerprint"`
TaskID string `json:"taskId"`
RunID string `json:"runId"`
CollectionID string `json:"collectionId"`
Sources map[string]annotationPinnedSource `json:"sources"`
Timeline persistedAnnotationTimeline `json:"timeline"`
LeftPTS []int64 `json:"leftPTS"`
RightPTS []int64 `json:"rightPTS"`
IMURowsBeforeBoundary []int64 `json:"imuRowsBeforeBoundary"`
LeftDuration float64 `json:"leftDuration"`
RightDuration float64 `json:"rightDuration"`
}
func sourceCertificateID(org, fingerprint string) string {
return "annotation-source-v1:" + org + ":" + fingerprint
}
func annotationSourceInputs(task domain.AnnotationTaskRecord, run domain.RunRecord) (map[string]*domain.ExecutionArtifact, error) {
for _, n := range run.NodeExecutions {
if n.NodeID != task.NodeID {
continue
}
left := artifactByID(artifactsForTarget(n, "fullLeftVideos"), task.ArtifactID)
if left == nil {
left = artifactByID(n.InputArtifacts, task.ArtifactID)
}
if left == nil {
return nil, fmt.Errorf("left source unavailable")
}
collection := artifactCollectionID(*left)
inputs := map[string]*domain.ExecutionArtifact{"leftVideo": left,
"rightVideo": matchingArtifact(artifactsForTarget(n, "fullRightVideos"), collection, "right", ".mp4"),
"leftPts": matchingArtifact(artifactsForTarget(n, "leftTimestamps"), collection, "left", "pts"),
"rightPts": matchingArtifact(artifactsForTarget(n, "rightTimestamps"), collection, "right", "pts"),
"imu": matchingArtifact(artifactsForTarget(n, "imu"), collection, "imu", ".csv")}
for kind, a := range inputs {
if a == nil {
return nil, fmt.Errorf("%s source unavailable", kind)
}
}
if inputs["rightVideo"].ID != task.RightArtifactID {
return nil, fmt.Errorf("right source identity mismatch")
}
return inputs, nil
}
return nil, fmt.Errorf("source execution unavailable")
}
func decodeSourceCertificate(record *domain.AnnotationTimelineIndex, fingerprint string) (*annotationSourceCertificate, error) {
if record == nil {
return nil, nil
}
sum := sha256.Sum256(record.Body)
if hex.EncodeToString(sum[:]) != record.SHA256 {
return nil, fmt.Errorf("source certificate checksum mismatch")
}
z, e := gzip.NewReader(bytes.NewReader(record.Body))
if e != nil {
return nil, e
}
defer z.Close()
b, e := io.ReadAll(io.LimitReader(z, 8<<20+1))
if e != nil || len(b) > 8<<20 {
return nil, fmt.Errorf("source certificate too large")
}
var c annotationSourceCertificate
if e = json.Unmarshal(b, &c); e != nil {
return nil, e
}
n := len(c.Timeline.Stamps)
if c.Schema != sourceCertificateSchema || c.Fingerprint != fingerprint || c.Timeline.Fingerprint != fingerprint || n < 2 || len(c.Timeline.Seconds) != n || len(c.Timeline.RightSeconds) != n || len(c.Timeline.Left) != n || len(c.Timeline.Right) != n || len(c.IMURowsBeforeBoundary) != n || len(c.LeftPTS) != c.Timeline.LeftFrameCount || len(c.RightPTS) != c.Timeline.RightFrameCount || len(c.Sources) != 5 {
return nil, fmt.Errorf("invalid source certificate")
}
for i := 0; i < n; i++ {
if c.Timeline.Left[i] < 0 || c.Timeline.Left[i] > len(c.LeftPTS) || c.Timeline.Right[i] < 0 || c.Timeline.Right[i] > len(c.RightPTS) || c.IMURowsBeforeBoundary[i] < 0 || math.IsNaN(c.Timeline.Seconds[i]) || math.IsInf(c.Timeline.Seconds[i], 0) || math.IsNaN(c.Timeline.RightSeconds[i]) || math.IsInf(c.Timeline.RightSeconds[i], 0) {
return nil, fmt.Errorf("invalid source certificate bounds")
}
if i > 0 && (c.Timeline.Stamps[i] <= c.Timeline.Stamps[i-1] || c.Timeline.Seconds[i] <= c.Timeline.Seconds[i-1] || c.Timeline.RightSeconds[i] <= c.Timeline.RightSeconds[i-1] || c.IMURowsBeforeBoundary[i] < c.IMURowsBeforeBoundary[i-1]) {
return nil, fmt.Errorf("invalid source certificate ordering")
}
}
for _, pts := range [][]int64{c.LeftPTS, c.RightPTS} {
for i, value := range pts {
if value <= 0 || i > 0 && value <= pts[i-1] {
return nil, fmt.Errorf("invalid source PTS ordering")
}
}
}
return &c, nil
}
// A certificate is specific to authoritative object versions. Single-flight and
// a separate two-slot semaphore prevent many light revision jobs from spawning
// simultaneous full decodes. Only compact indexes, never IMU rows, are cached.
func (s *server) annotationSourceCertificate(ctx context.Context, t domain.AnnotationTaskRecord, run domain.RunRecord, fingerprint string) (*annotationSourceCertificate, error) {
v, e := s.annotationSourceCertificates.get(ctx, fingerprint, func(ctx context.Context) (map[string]any, error) {
db, ok := s.store.(store.AnnotationProcessingStore)
if !ok {
return nil, fmt.Errorf("source certificate store unavailable")
}
record, e := db.GetAnnotationTimelineIndex(ctx, sourceCertificateID(t.OrganizationID, fingerprint))
if e != nil {
return nil, e
}
if cert, e := decodeSourceCertificate(record, fingerprint); e == nil && cert != nil {
return map[string]any{"certificate": cert}, nil
}
s.annotationValidationOnce.Do(func() { s.annotationValidationSlots = make(chan struct{}, 2) })
select {
case s.annotationValidationSlots <- struct{}{}:
case <-ctx.Done():
return nil, ctx.Err()
}
defer func() { <-s.annotationValidationSlots }()
cert, e := s.buildAnnotationSourceCertificate(ctx, t, run, fingerprint)
if e != nil {
return nil, e
}
b, e := json.Marshal(cert)
if e != nil {
return nil, e
}
if len(b) > 8<<20 {
return nil, fmt.Errorf("source index exceeds size limit")
}
var compressed bytes.Buffer
z := gzip.NewWriter(&compressed)
if _, e = z.Write(b); e != nil {
return nil, e
}
if e = z.Close(); e != nil {
return nil, e
}
sum := sha256.Sum256(compressed.Bytes())
e = db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: sourceCertificateID(t.OrganizationID, fingerprint), OrganizationID: t.OrganizationID, ProjectID: t.ProjectID, WorkflowID: t.WorkflowID, RunID: t.RunID, Body: compressed.Bytes(), SHA256: hex.EncodeToString(sum[:]), CreatedAt: nowISO()})
return map[string]any{"certificate": cert}, e
})
if e != nil {
return nil, e
}
cert := v["certificate"].(*annotationSourceCertificate)
if cert.RunID != t.RunID {
return nil, fmt.Errorf("source certificate run mismatch")
}
return cert, nil
}
// sourceClockTolerance accepts container rounding but nothing larger: a frame
// time stored in a coarse time base (e.g. 1/1000 for 30fps) is up to half a
// tick from the ideal fixed-rate time. Fine time bases keep the 0.1ms bound.
func sourceClockTolerance(timeBase string) (float64, error) {
tick, err := sourceClockRatio(timeBase)
if err != nil {
return 0, err
}
return math.Max(0.0001, tick/2+1e-9), nil
}
func sourceClockRatio(value string) (float64, error) {
numerator, denominator, found := strings.Cut(strings.TrimSpace(value), "/")
n, e1 := strconv.ParseFloat(numerator, 64)
d, e2 := strconv.ParseFloat(denominator, 64)
ratio := n / d
if !found || e1 != nil || e2 != nil || n <= 0 || d <= 0 || math.IsInf(n, 0) || math.IsInf(d, 0) || math.IsNaN(ratio) || math.IsInf(ratio, 0) || ratio <= 0 {
return 0, fmt.Errorf("invalid source video clock ratio %q", value)
}
return ratio, nil
}
type sourceVideoTiming struct {
TimeBase string `json:"time_base"`
FrameRate string `json:"r_frame_rate"`
}
func ffprobeSourceTiming(ctx context.Context, path string) (sourceVideoTiming, error) {
output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=time_base,r_frame_rate", "-of", "json", path).CombinedOutput()
if err != nil {
return sourceVideoTiming{}, fmt.Errorf("ffprobe timing %s: %s", filepath.Base(path), strings.TrimSpace(string(output)))
}
var metadata struct {
Streams []sourceVideoTiming `json:"streams"`
}
if err = json.Unmarshal(output, &metadata); err != nil || len(metadata.Streams) != 1 {
return sourceVideoTiming{}, fmt.Errorf("invalid source video timing metadata: %s", filepath.Base(path))
}
return metadata.Streams[0], nil
}
func validateSourceMediaClock(ctx context.Context, path string, count int, duration float64) error {
if count < 2 || duration <= 0 || math.IsNaN(duration) || math.IsInf(duration, 0) {
return fmt.Errorf("invalid source media duration/count")
}
timing, e := ffprobeSourceTiming(ctx, path)
if e != nil {
return e
}
tolerance, e := sourceClockTolerance(timing.TimeBase)
if e != nil {
return e
}
tick, _ := sourceClockRatio(timing.TimeBase) // validated by sourceClockTolerance
rate, e := sourceClockRatio(timing.FrameRate)
if e != nil {
return e
}
period := 1 / rate
// r_frame_rate is only a candidate cadence: verify every decoded frame
// against it below, rather than trusting the metadata or deriving a cadence
// from rounded total duration. MP4 edit lists additionally round the
// presentation endpoint in the movie clock (often 1ms, even for a video
// track at 1/15360). This never relaxes the decoded frame checks below.
if math.IsInf(period, 0) || math.Abs(duration-float64(count)*period) > sourceClockEndTolerance(path, tick, period) {
return fmt.Errorf("source video duration does not match its fixed frame cadence")
}
// Integer timestamps preserve half-tick boundaries; ffprobe's *_time
// decimal output rounds to microseconds and can exceed the tolerance.
// Bound decoding to two threads within the validator CPU budget.
// Still decode every frame and validate its original presentation timestamp.
cmd := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-threads", "2", "-select_streams", "v:0", "-show_frames", "-show_entries", "frame=best_effort_timestamp", "-of", "csv=p=0", path)
stdout, e := cmd.StdoutPipe()
if e != nil {
return e
}
// Decoder diagnostics are not returned; do not accumulate unbounded stderr
// from malformed media during a long-running validation.
cmd.Stderr = io.Discard
if e = cmd.Start(); e != nil {
return e
}
scanner := bufio.NewScanner(stdout)
scanner.Buffer(make([]byte, 4096), 1<<20)
n := 0
previous := -1.0
var clockErr error
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
text := strings.SplitN(line, ",", 2)[0]
ticks, err := strconv.ParseInt(text, 10, 64)
stamp := float64(ticks) * tick
if err != nil || math.IsNaN(stamp) || math.IsInf(stamp, 0) || stamp <= previous || math.Abs(stamp-float64(n)*period) > tolerance {
clockErr = fmt.Errorf("源视频时间轴非连续固定帧序,不能使用源引用切片(帧 %d)", n)
_ = cmd.Process.Kill()
break
}
previous = stamp
n++
}
readErr := scanner.Err()
if readErr != nil {
_ = cmd.Process.Kill()
}
waitErr := cmd.Wait()
if clockErr != nil {
return clockErr
}
if readErr != nil {
return readErr
}
if waitErr != nil {
return fmt.Errorf("source decode failed: %w", waitErr)
}
if n != count {
return fmt.Errorf("source decoded frame/PTS mismatch: %d/%d", n, count)
}
return nil
}
func (s *server) buildAnnotationSourceCertificate(ctx context.Context, t domain.AnnotationTaskRecord, run domain.RunRecord, fingerprint string) (_ *annotationSourceCertificate, buildErr error) {
started, phaseStarted := time.Now(), time.Now()
phase := "fetch"
durations := map[string]int64{}
nextPhase := func(next string) {
durations[phase] = time.Since(phaseStarted).Milliseconds()
phase, phaseStarted = next, time.Now()
}
defer func() {
durations[phase] = time.Since(phaseStarted).Milliseconds()
log.Printf("[annotation-validator] source_timing task=%s total_ms=%d phase_ms=%v success=%t", t.ID, time.Since(started).Milliseconds(), durations, buildErr == nil)
}()
inputs, e := annotationSourceInputs(t, run)
if e != nil {
return nil, e
}
stage, e := os.MkdirTemp("", "annotation-source-")
if e != nil {
return nil, e
}
defer os.RemoveAll(stage)
paths := map[string]string{}
versions := map[string]string{}
for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
path, e := s.safeMaterializationSourcePath(inputs[kind])
if e != nil {
return nil, e
}
versions[kind], e = s.annotationFileVersion(ctx, t.OrganizationID, path)
if e != nil {
return nil, e
}
ext := ".csv"
if strings.HasSuffix(kind, "Video") {
ext = ".mp4"
}
local, e := s.localAnnotationSource(ctx, t.OrganizationID, path, filepath.Join(stage, kind+ext))
if e != nil {
return nil, e
}
paths[kind] = local
}
nextPhase("media_clock")
left, e := readHardwarePTS(paths["leftPts"])
if e != nil {
return nil, e
}
right, e := readHardwarePTS(paths["rightPts"])
if e != nil {
return nil, e
}
ld, e := ffprobeDurationSeconds(ctx, paths["leftVideo"])
if e != nil {
return nil, e
}
rd, e := ffprobeDurationSeconds(ctx, paths["rightVideo"])
if e != nil {
return nil, e
}
if e = validateSourceMediaClock(ctx, paths["leftVideo"], len(left), ld); e != nil {
return nil, e
}
if e = validateSourceMediaClock(ctx, paths["rightVideo"], len(right), rd); e != nil {
return nil, e
}
nextPhase("sensor_index")
// The existing builder validates ordered sensor pairs and the exact common
// range. Its compact grid is the same seam used by editor validation.
timeline, e := s.annotationFrameTimelineForTask(ctx, t, "left")
if e != nil {
return nil, e
}
b, _ := json.Marshal(timeline)
var grid persistedAnnotationTimeline
if e = json.Unmarshal(b, &grid); e != nil {
return nil, e
}
if grid.Fingerprint != fingerprint {
return nil, fmt.Errorf("source changed during validation")
}
c := &annotationSourceCertificate{Schema: sourceCertificateSchema, Fingerprint: fingerprint, TaskID: t.ID, RunID: t.RunID, CollectionID: artifactCollectionID(*inputs["leftVideo"]), Sources: map[string]annotationPinnedSource{}, Timeline: grid, LeftDuration: ld, RightDuration: rd, IMURowsBeforeBoundary: make([]int64, len(grid.Stamps))}
for _, row := range left {
c.LeftPTS = append(c.LeftPTS, row.TimestampUS)
}
for _, row := range right {
c.RightPTS = append(c.RightPTS, row.TimestampUS)
}
position := 0
var count int64
_, e = scanHardwareIMU(ctx, paths["imu"], func(row hardwareIMURow) {
for position < len(grid.Stamps) && grid.Stamps[position] <= row.TimestampUS {
c.IMURowsBeforeBoundary[position] = count
position++
}
count++
})
if e != nil {
return nil, e
}
for position < len(grid.Stamps) {
c.IMURowsBeforeBoundary[position] = count
position++
}
nextPhase("pin")
for _, kind := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
original, e := s.safeMaterializationSourcePath(inputs[kind])
if e != nil {
return nil, e
}
current, e := s.annotationFileVersion(ctx, t.OrganizationID, original)
if e != nil || current != versions[kind] {
return nil, fmt.Errorf("源文件在校验期间发生变化")
}
digest, size, e := sha256File(paths[kind])
if e != nil {
return nil, e
}
ext := ".csv"
if strings.HasSuffix(kind, "Video") {
ext = ".mp4"
}
target := filepath.Join(s.workspaceMountPath, "system", "runs", t.RunID, "annotation-sources", fingerprint, kind+ext)
if e = s.pinAnnotationSource(ctx, t.OrganizationID, original, versions[kind], paths[kind], target, digest, size); e != nil {
return nil, e
}
version, e := s.annotationFileVersion(ctx, t.OrganizationID, target)
if e != nil {
return nil, e
}
c.Sources[kind] = annotationPinnedSource{ArtifactID: inputs[kind].ID, Path: target, Version: version, SHA256: digest, Bytes: size}
}
after, e := s.annotationSourceFingerprint(ctx, t, run)
if e != nil {
return nil, e
}
if after != fingerprint {
return nil, fmt.Errorf("源文件在固定版本期间发生变化")
}
return c, nil
}
// Each source version owns five immutable objects, shared by all its revisions.
// BOS copies are conditional on source ETag and destination nonexistence. Local
// publication uses an atomic hard link of a fully verified sibling temp file.
func (s *server) pinAnnotationSource(ctx context.Context, org, original, expectedVersion, local, target, digest string, size int64) error {
c, bucket, key, e := s.annotationWorkspaceObjectAccess(ctx, org, original, false)
if e != nil {
return e
}
if c != nil {
to, tb, tk, e := s.annotationWorkspaceObject(ctx, org, target)
if e != nil {
return e
}
if to == nil || to.ID != c.ID || tb != bucket {
return fmt.Errorf("cross-workspace source pin")
}
publisher := objectsync.BOSPublisher{GlobalEndpoint: s.globalBOSEndpoint}
old, e := publisher.Head(ctx, *c, bucket, tk)
if e != nil {
return e
}
if old.Exists {
if old.SizeBytes == size && annotationObjectSHA(old) == digest {
return nil
}
return fmt.Errorf("pinned source conflict")
}
meta, e := publisher.Head(ctx, *c, bucket, key)
if e != nil {
return e
}
version := fmt.Sprintf("bos:%s:%s:%s:%s:%d", c.ID, bucket, meta.ETag, meta.VersionID, meta.SizeBytes)
if !meta.Exists || meta.SizeBytes != size || meta.ETag == "" || version != expectedVersion {
return fmt.Errorf("source pin version unavailable")
}
_, e = publisher.PublishConditional(ctx, *c, bucket, key, tk, size, meta.ETag, "", "*", map[string]string{"sha256": digest})
if e != nil {
old, he := publisher.Head(ctx, *c, bucket, tk)
if he == nil && old.Exists && old.SizeBytes == size && annotationObjectSHA(old) == digest {
return nil
}
}
return e
}
if e = os.MkdirAll(filepath.Dir(target), 0755); e != nil {
return e
}
if got, n, e := sha256File(target); e == nil {
if got == digest && n == size {
return nil
}
return fmt.Errorf("pinned source conflict")
}
temp, e := os.CreateTemp(filepath.Dir(target), ".pin-")
if e != nil {
return e
}
name := temp.Name()
defer os.Remove(name)
_ = temp.Close()
if e = copyAnnotationFile(ctx, local, name, digest); e != nil {
return e
}
if e = os.Link(name, target); e != nil {
got, n, readErr := sha256File(target)
if readErr == nil && got == digest && n == size {
return nil
}
return e
}
return nil
}