package api import ( "context" "crypto/sha256" "encoding/csv" "encoding/hex" "encoding/json" "fmt" "io" "log" "math" "net/url" "os" "os/exec" "path/filepath" "sort" "strconv" "strings" "sync" "time" "droboticflow/internal/domain" "golang.org/x/sync/errgroup" ) type hardwarePTSRow struct { TimestampUS int64 } type hardwareIMURow struct { TimestampUS int64 X string Y string Z string Type string } type humanSliceMaterialization struct { Artifact domain.ExecutionArtifact Payload map[string]any Slices []domain.SliceSegment Segments []domain.AnnotationSegment CoverageStartSecond float64 CoverageEndSecond float64 } func decodeMaterializationValue[T any](value any) []T { encoded, err := json.Marshal(normalizeStructuredValue(value)) if err != nil { return nil } var items []T if err := json.Unmarshal(encoded, &items); err != nil { return nil } return items } func normalizedCSVHeader(value string) string { return strings.ToLower(strings.TrimSpace(strings.TrimPrefix(value, "\ufeff"))) } func csvRecords(path string) ([][]string, error) { file, err := os.Open(path) if err != nil { return nil, err } defer file.Close() prefix := make([]byte, 4096) count, _ := file.Read(prefix) if _, err := file.Seek(0, io.SeekStart); err != nil { return nil, err } reader := csv.NewReader(file) if strings.Count(string(prefix[:count]), "\t") > strings.Count(string(prefix[:count]), ",") { reader.Comma = '\t' } reader.FieldsPerRecord = -1 records, err := reader.ReadAll() if err != nil { return nil, err } if len(records) < 2 { return nil, fmt.Errorf("CSV %s has no data rows", filepath.Base(path)) } return records, nil } func headerIndex(headers []string, names ...string) int { wanted := map[string]struct{}{} for _, name := range names { wanted[normalizedCSVHeader(name)] = struct{}{} } for index, header := range headers { if _, ok := wanted[normalizedCSVHeader(header)]; ok { return index } } return -1 } func readHardwarePTS(path string) ([]hardwarePTSRow, error) { records, err := csvRecords(path) if err != nil { return nil, err } index := headerIndex(records[0], "timestamp_us", "timestampus", "pts_us") if index < 0 { return nil, fmt.Errorf("PTS CSV %s is missing timestamp_us", filepath.Base(path)) } rows := make([]hardwarePTSRow, 0, len(records)-1) var previous int64 for rowIndex, record := range records[1:] { if index >= len(record) { return nil, fmt.Errorf("PTS CSV %s row %d is incomplete", filepath.Base(path), rowIndex+2) } value, parseErr := strconv.ParseInt(strings.TrimSpace(record[index]), 10, 64) if parseErr != nil || value <= 0 || (len(rows) > 0 && value <= previous) { return nil, fmt.Errorf("PTS CSV %s is invalid or not strictly increasing at row %d", filepath.Base(path), rowIndex+2) } rows = append(rows, hardwarePTSRow{TimestampUS: value}) previous = value } return rows, nil } func readHardwareIMU(path string) ([]hardwareIMURow, error) { return readHardwareIMUContext(context.Background(), path) } func readHardwareIMUContext(ctx context.Context, path string) ([]hardwareIMURow, error) { rows := make([]hardwareIMURow, 0) _, err := scanHardwareIMU(ctx, path, func(row hardwareIMURow) { rows = append(rows, row) }) return rows, err } type hardwareIMUSummary struct { first, previous, last int64 count int } func (s hardwareIMUSummary) coverageEnd() (int64, error) { if s.count < 4 || s.previous <= 0 || s.last <= s.previous { return 0, fmt.Errorf("IMU requires at least two accel/gyro timestamp pairs") } return s.last + s.last - s.previous, nil } // Sorted timestamps allow pair validation in constant space. Timeline callers // consume only the summary, rather than allocating 1.7 million sample objects. func scanHardwareIMU(ctx context.Context, path string, consume func(hardwareIMURow)) (hardwareIMUSummary, error) { var summary hardwareIMUSummary file, err := os.Open(path) if err != nil { return summary, err } defer file.Close() prefix := make([]byte, 4096) n, _ := file.Read(prefix) if _, err = file.Seek(0, io.SeekStart); err != nil { return summary, err } reader := csv.NewReader(file) reader.FieldsPerRecord = -1 reader.ReuseRecord = true if strings.Count(string(prefix[:n]), "\t") > strings.Count(string(prefix[:n]), ",") { reader.Comma = '\t' } headers, err := reader.Read() if err != nil { return summary, err } timestampIndex := headerIndex(headers, "timestamp_us", "timestampus") xIndex, yIndex, zIndex := headerIndex(headers, "x"), headerIndex(headers, "y"), headerIndex(headers, "z") typeIndex := headerIndex(headers, "type") if timestampIndex < 0 || xIndex < 0 || yIndex < 0 || zIndex < 0 || typeIndex < 0 { return summary, fmt.Errorf("IMU CSV %s is missing timestamp_us/x/y/z/type", filepath.Base(path)) } maximum := max(timestampIndex, xIndex, yIndex, zIndex, typeIndex) accel, gyro := false, false for rowIndex := 2; ; rowIndex++ { if err := ctx.Err(); err != nil { return summary, err } record, err := reader.Read() if err == io.EOF { break } if err != nil { return summary, err } if maximum >= len(record) { return summary, fmt.Errorf("IMU CSV %s row %d is incomplete", filepath.Base(path), rowIndex) } timestamp, parseErr := strconv.ParseInt(strings.TrimSpace(record[timestampIndex]), 10, 64) kind := strings.ToLower(strings.TrimSpace(record[typeIndex])) if kind != "accel" && kind != "gyro" { return summary, fmt.Errorf("IMU CSV %s invalid type %q at row %d", filepath.Base(path), kind, rowIndex) } if parseErr != nil || timestamp <= 0 || (summary.count > 0 && timestamp < summary.last) { return summary, fmt.Errorf("IMU CSV %s has invalid timestamp ordering at row %d", filepath.Base(path), rowIndex) } for _, index := range []int{xIndex, yIndex, zIndex} { value, err := strconv.ParseFloat(strings.TrimSpace(record[index]), 64) if err != nil || math.IsNaN(value) || math.IsInf(value, 0) { return summary, fmt.Errorf("IMU CSV %s has non-finite or invalid numeric value at row %d", filepath.Base(path), rowIndex) } } if summary.count == 0 { summary.first = timestamp } if timestamp != summary.last { if summary.count > 0 && (!accel || !gyro) { return summary, fmt.Errorf("IMU timestamp %d is missing an accel/gyro pair", summary.last) } summary.previous = summary.last accel, gyro = false, false } if (kind == "accel" && accel) || (kind == "gyro" && gyro) { return summary, fmt.Errorf("IMU timestamp %d contains a duplicate %s sample", timestamp, kind) } accel = accel || kind == "accel" gyro = gyro || kind == "gyro" summary.last = timestamp summary.count++ if consume != nil { consume(hardwareIMURow{TimestampUS: timestamp, X: strings.TrimSpace(record[xIndex]), Y: strings.TrimSpace(record[yIndex]), Z: strings.TrimSpace(record[zIndex]), Type: kind}) } } if summary.count == 0 { return summary, fmt.Errorf("CSV %s has no data rows", filepath.Base(path)) } if !accel || !gyro { return summary, fmt.Errorf("IMU timestamp %d is missing an accel/gyro pair", summary.last) } return summary, nil } func artifactCollectionID(artifact domain.ExecutionArtifact) string { if artifact.Data == nil { return "" } return strings.TrimSpace(fmt.Sprint(artifact.Data["collectionId"])) } func artifactsForTarget(execution domain.NodeExecutionRecord, targetPortID string) []domain.ExecutionArtifact { for _, binding := range execution.InputBindings { if binding.TargetPortID == targetPortID { return append([]domain.ExecutionArtifact(nil), binding.Artifacts...) } } return nil } func artifactByID(items []domain.ExecutionArtifact, id string) *domain.ExecutionArtifact { for index := range items { if items[index].ID == id { copy := items[index] return © } } return nil } func matchingArtifact(items []domain.ExecutionArtifact, collectionID string, nameParts ...string) *domain.ExecutionArtifact { for index := range items { name := strings.ToLower(items[index].Name) if collectionID != "" && artifactCollectionID(items[index]) != "" && artifactCollectionID(items[index]) != collectionID { continue } matched := true for _, part := range nameParts { if !strings.Contains(name, strings.ToLower(part)) { matched = false } } if matched { copy := items[index] return © } } return nil } func (server *server) safeMaterializationSourcePath(artifact *domain.ExecutionArtifact) (string, error) { if artifact == nil { return "", fmt.Errorf("materialization input artifact is missing") } root := filepath.Clean(server.workspaceMountPath) if root == "." || root == "" { return "", fmt.Errorf("materialization workspace is unavailable") } pathValue := strings.TrimSpace(artifact.Path) candidates := make([]string, 0, 3) if pathValue != "" && !strings.Contains(pathValue, "://") { candidates = append(candidates, filepath.Clean(pathValue)) } if strings.HasPrefix(strings.ToLower(pathValue), "bos://") { if parsed, err := url.Parse(pathValue); err == nil { key := strings.TrimPrefix(strings.TrimSpace(parsed.Path), "/") if key != "" { candidates = append(candidates, filepath.Join(root, filepath.FromSlash(key))) } } } if artifact.Payload != nil && artifact.Payload.Ref != nil { key := strings.TrimPrefix(strings.TrimSpace(artifact.Payload.Ref.Key), "/") if key == "" { key = strings.TrimPrefix(strings.TrimSpace(artifact.Payload.Ref.Prefix), "/") } if key != "" { candidates = append(candidates, filepath.Join(root, filepath.FromSlash(key))) } } for _, candidate := range candidates { candidate = filepath.Clean(candidate) if candidate != root && !strings.HasPrefix(candidate, root+string(os.PathSeparator)) { continue } info, err := os.Stat(candidate) if err == nil && info.Mode().IsRegular() && info.Size() > 0 { return candidate, nil } } return "", fmt.Errorf("materialization input %s is unavailable on the authorized workspace mount", artifact.Name) } func ffprobeFrameCount(ctx context.Context, path string) (int, error) { output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-threads", "2", "-select_streams", "v:0", "-count_frames", "-show_entries", "stream=nb_read_frames", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput() if err != nil { return 0, fmt.Errorf("ffprobe %s: %s", filepath.Base(path), strings.TrimSpace(string(output))) } count, err := strconv.Atoi(strings.TrimSpace(string(output))) if err != nil || count <= 0 { return 0, fmt.Errorf("ffprobe returned invalid frame count for %s", filepath.Base(path)) } return count, nil } // Timeline loading reads MP4 metadata, not every decoded frame of a long Capture. // Materialization still performs the full frame-count integrity check above. func ffprobeDeclaredFrameCount(ctx context.Context, path string) (int, error) { output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=nb_frames", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput() if err != nil { return 0, fmt.Errorf("读取视频帧元数据失败:%s", filepath.Base(path)) } count, err := strconv.Atoi(strings.TrimSpace(string(output))) if err != nil || count <= 0 { return 0, fmt.Errorf("视频缺少有效帧数元数据:%s", filepath.Base(path)) } return count, nil } func ffprobeDurationSeconds(ctx context.Context, path string) (float64, error) { output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=duration", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput() if err != nil { return 0, fmt.Errorf("ffprobe duration %s: %s", filepath.Base(path), strings.TrimSpace(string(output))) } duration, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64) if err != nil || math.IsNaN(duration) || math.IsInf(duration, 0) || duration <= 0 { return 0, fmt.Errorf("ffprobe returned invalid duration for %s", filepath.Base(path)) } return duration, nil } func writeFrameAccurateSlice(ctx context.Context, source string, destination string, startFrame int, endFrame int) error { if startFrame < 0 || endFrame <= startFrame { return fmt.Errorf("frame slice is empty") } return writeFrameAccurateSlices(ctx, source, []annotationFrameSlice{{Start: startFrame, End: endFrame, Output: destination}}) } func sha256File(path string) (string, int64, error) { file, err := os.Open(path) if err != nil { return "", 0, err } defer file.Close() hash := sha256.New() size, err := io.Copy(hash, file) if err != nil { return "", 0, err } return hex.EncodeToString(hash.Sum(nil)), size, nil } func writePTSSlice(path string, rows []hardwarePTSRow, start int, end int, windowStartUS int64) (map[string]any, error) { if start < 0 || end > len(rows) || end <= start { return nil, fmt.Errorf("PTS slice is empty") } file, err := os.Create(path) if err != nil { return nil, err } writer := csv.NewWriter(file) _ = writer.Write([]string{"frame_index", "timestamp_ms", "clip_time_ms"}) for index, row := range rows[start:end] { _ = writer.Write([]string{strconv.Itoa(index), fmt.Sprintf("%.3f", float64(row.TimestampUS)/1000.0), fmt.Sprintf("%.3f", float64(row.TimestampUS-windowStartUS)/1000.0)}) } writer.Flush() closeErr := file.Close() if writer.Error() != nil { return nil, writer.Error() } if closeErr != nil { return nil, closeErr } sha, size, err := sha256File(path) return map[string]any{"name": filepath.Base(path), "sha256": sha, "bytes": size, "rowCount": end - start, "firstTimestampUs": rows[start].TimestampUS, "lastTimestampUs": rows[end-1].TimestampUS}, err } func writeIMUSlice(path string, rows []hardwareIMURow, startUS int64, endUS int64) (map[string]any, error) { start := sort.Search(len(rows), func(i int) bool { return rows[i].TimestampUS >= startUS }) end := sort.Search(len(rows), func(i int) bool { return rows[i].TimestampUS >= endUS }) selected := rows[start:end] if len(selected) == 0 { return nil, fmt.Errorf("IMU slice is empty") } file, err := os.Create(path) if err != nil { return nil, err } writer := csv.NewWriter(file) _ = writer.Write([]string{"timestamp_ms", "clip_time_ms", "x", "y", "z", "type"}) for _, row := range selected { _ = writer.Write([]string{fmt.Sprintf("%.3f", float64(row.TimestampUS)/1000.0), fmt.Sprintf("%.3f", float64(row.TimestampUS-startUS)/1000.0), row.X, row.Y, row.Z, row.Type}) } writer.Flush() closeErr := file.Close() if writer.Error() != nil { return nil, writer.Error() } if closeErr != nil { return nil, closeErr } sha, size, err := sha256File(path) return map[string]any{"name": filepath.Base(path), "sha256": sha, "bytes": size, "sampleCount": len(selected), "firstTimestampUs": selected[0].TimestampUS, "lastTimestampUs": selected[len(selected)-1].TimestampUS}, err } func writeJSONFile(path string, payload any) (string, int64, error) { encoded, err := json.MarshalIndent(payload, "", " ") if err != nil { return "", 0, err } encoded = append(encoded, '\n') if err := os.WriteFile(path, encoded, 0o644); err != nil { return "", 0, err } digest := sha256.Sum256(encoded) return hex.EncodeToString(digest[:]), int64(len(encoded)), nil } func artifactForMaterializationPath(path string, id string, artifactType domain.PortType) map[string]any { sha, size, _ := sha256File(path) return map[string]any{"id": id, "name": filepath.Base(path), "sha256": sha, "bytes": size, "type": string(artifactType)} } func ptsCoverageEnd(rows []hardwarePTSRow) (int64, error) { if len(rows) < 2 { return 0, fmt.Errorf("PTS requires at least two rows to resolve the half-open coverage end") } interval := rows[len(rows)-1].TimestampUS - rows[len(rows)-2].TimestampUS if interval <= 0 { return 0, fmt.Errorf("PTS terminal interval is invalid") } return rows[len(rows)-1].TimestampUS + interval, nil } func imuCoverageEnd(rows []hardwareIMURow) (int64, error) { if len(rows) < 4 { return 0, fmt.Errorf("IMU requires at least two accel/gyro timestamp pairs") } last := rows[len(rows)-1].TimestampUS previous := int64(0) for index := len(rows) - 2; index >= 0; index-- { if rows[index].TimestampUS < last { previous = rows[index].TimestampUS break } } if previous <= 0 || last <= previous { return 0, fmt.Errorf("IMU terminal interval is invalid") } return last + (last - previous), nil } func maximumNearestAlignmentUS(left []hardwarePTSRow, leftStart int, leftEnd int, right []hardwarePTSRow, rightStart int, rightEnd int) int64 { maximum := int64(0) for _, row := range left[leftStart:leftEnd] { index := sort.Search(rightEnd-rightStart, func(index int) bool { return right[rightStart+index].TimestampUS >= row.TimestampUS }) nearest := int64(math.MaxInt64) if index < rightEnd-rightStart { nearest = absInt64(right[rightStart+index].TimestampUS - row.TimestampUS) } if index > 0 { nearest = min(nearest, absInt64(right[rightStart+index-1].TimestampUS-row.TimestampUS)) } maximum = max(maximum, nearest) } return maximum } func absInt64(value int64) int64 { if value < 0 { return -value } return value } func (server *server) materializeCombinedSliceSubmission(ctx context.Context, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, payload combinedAnnotationSubmissionInput, revision int) (*humanSliceMaterialization, error) { if err := server.assertAnnotationLease(ctx); err != nil { return nil, err } continuous := !payload.sliceOnly && server.continuousActionsForItem(ctx, &item) if continuous { payload.Segments = fillPendingActionGaps(payload.SliceSegments, payload.Segments) if issues := validateContinuousActions(payload.SliceSegments, payload.Segments, true); len(issues) > 0 { return nil, fmt.Errorf("%s", issues[0].Message) } } started := time.Now() ctx, cancel := context.WithTimeout(ctx, 24*time.Hour) defer cancel() progressKey := task.OrganizationID + ":" + task.ID progress := func(stage string, completed int) { server.materializationProgress.Store(progressKey, map[string]any{"stage": stage, "completedSlices": completed, "totalSlices": len(payload.SliceSegments), "startedAt": started.UTC().Format(time.RFC3339Nano)}) } progress("queued", 0) defer server.materializationProgress.Delete(progressKey) server.materializationSlotsOnce.Do(func() { server.materializationSlots = make(chan struct{}, 1) }) select { case server.materializationSlots <- struct{}{}: case <-ctx.Done(): return nil, ctx.Err() } defer func() { <-server.materializationSlots }() queueMS := time.Since(started).Milliseconds() progress("validating", 0) defer func() { log.Printf("[api] materialization_timing task=%s revision=%d queue_ms=%d total_ms=%d", task.ID, revision, queueMS, time.Since(started).Milliseconds()) }() run, err := server.store.GetRun(ctx, task.RunID) if err != nil || run == nil { return nil, fmt.Errorf("load materialization run: %w", err) } // Multiple Captures may share a legacy Run and node. Revision numbers are // task-local, so both cache identity and storage must include the task ID. artifactID := fmt.Sprintf("%s-%s-%s-materialized-revision-%06d", task.RunID, task.NodeID, task.ID, revision) for _, existing := range run.Artifacts { if existing.ID == artifactID { if isSourceIndexArtifact(&existing) { document := server.sourceIndexManifestDocument(ctx, run, &existing) if document.Message != "" || len(document.Members) == 0 { return nil, fmt.Errorf("源引用修订不可复用:%s", document.Message) } } if stringValue(existing.Data["boundaryContract"]) != "hardware-frame-v1" { return nil, fmt.Errorf("历史切片不使用统一边界,请重新确认切片;不会自动移动已有标注") } manifestSHA := "" if existing.Metadata != nil { manifestSHA = existing.Metadata.SHA256 } coverage := mapValue(existing.Data["coverage"]) slices := decodeMaterializationValue[domain.SliceSegment](existing.Data["normalizedSlices"]) segments := decodeMaterializationValue[domain.AnnotationSegment](existing.Data["normalizedSegments"]) sliceCount := intValue(existing.Data["sliceCount"]) if sliceCount == 0 { sliceCount = intValue(coverage["sliceCount"]) } if slices == nil { slices = append([]domain.SliceSegment(nil), payload.SliceSegments...) } if segments == nil { segments = append([]domain.AnnotationSegment(nil), payload.Segments...) } if continuous { if issues := validateContinuousActions(slices, segments, true); len(issues) > 0 { return nil, fmt.Errorf("%s", issues[0].Message) } } return &humanSliceMaterialization{Artifact: existing, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": sliceCount, "coverage": coverage, "deliverySelection": existing.Data["deliverySelection"], "runId": task.RunID, "artifactId": existing.ID, "manifestSha256": manifestSHA}, Slices: slices, Segments: segments, CoverageStartSecond: floatValue(coverage["startSecond"]), CoverageEndSecond: floatValue(coverage["endSecond"])}, nil } } sourceFingerprint, err := server.annotationSourceFingerprint(ctx, task, *run) if err != nil { return nil, err } if server.annotationSourceIndex { return server.materializeSourceIndexRevision(ctx, task, item, run, payload, revision, sourceFingerprint) } reusable := reusableAnnotationMedia(*run, task, item, server.workspaceMountPath, sourceFingerprint, revision) if metadata, e := server.tryAnnotationMetadataRevision(ctx, task, item, run, payload, revision, sourceFingerprint); e != nil { return nil, e } else if metadata != nil { return metadata, nil } stage, err := os.MkdirTemp("", "annotation-materialize-") if err != nil { return nil, err } defer os.RemoveAll(stage) var execution *domain.NodeExecutionRecord for index := range run.NodeExecutions { if run.NodeExecutions[index].NodeID == task.NodeID { copy := run.NodeExecutions[index] execution = © break } } if execution == nil { return nil, fmt.Errorf("human gate execution is unavailable") } leftArtifact := artifactByID(artifactsForTarget(*execution, "fullLeftVideos"), task.ArtifactID) if leftArtifact == nil { leftArtifact = artifactByID(execution.InputArtifacts, task.ArtifactID) } if leftArtifact == nil { return nil, fmt.Errorf("left source video is unavailable") } collectionID := artifactCollectionID(*leftArtifact) rightArtifact := matchingArtifact(artifactsForTarget(*execution, "fullRightVideos"), collectionID, "right", ".mp4") leftPTSArtifact := matchingArtifact(artifactsForTarget(*execution, "leftTimestamps"), collectionID, "left", "pts") rightPTSArtifact := matchingArtifact(artifactsForTarget(*execution, "rightTimestamps"), collectionID, "right", "pts") imuArtifact := matchingArtifact(artifactsForTarget(*execution, "imu"), collectionID, "imu", ".csv") leftPath, err := server.safeMaterializationSourcePath(leftArtifact) if err != nil { return nil, err } rightPath, err := server.safeMaterializationSourcePath(rightArtifact) if err != nil { return nil, err } leftPTSPath, err := server.safeMaterializationSourcePath(leftPTSArtifact) if err != nil { return nil, err } rightPTSPath, err := server.safeMaterializationSourcePath(rightPTSArtifact) if err != nil { return nil, err } imuPath, err := server.safeMaterializationSourcePath(imuArtifact) if err != nil { return nil, err } // Decode and hash on local disk, not repeatedly through the BOS filesystem. // Source videos are only copied if an unchanged slice cannot be reused. stagedVideos := false stageVideos := func() error { if stagedVideos { return nil } leftLocal, rightLocal := filepath.Join(stage, "source-left.mp4"), filepath.Join(stage, "source-right.mp4") group, gctx := errgroup.WithContext(ctx) group.Go(func() error { p, e := server.localAnnotationSource(gctx, task.OrganizationID, leftPath, leftLocal) if e == nil && p != leftLocal { e = copyAnnotationFile(gctx, p, leftLocal, "") } return e }) group.Go(func() error { p, e := server.localAnnotationSource(gctx, task.OrganizationID, rightPath, rightLocal) if e == nil && p != rightLocal { e = copyAnnotationFile(gctx, p, rightLocal, "") } return e }) if e := group.Wait(); e != nil { return e } leftPath, rightPath, stagedVideos = leftLocal, rightLocal, true return nil } inputGroup, inputCtx := errgroup.WithContext(ctx) inputGroup.Go(func() error { p, e := server.localAnnotationSource(inputCtx, task.OrganizationID, leftPTSPath, filepath.Join(stage, "source-left.csv")) if e == nil { leftPTSPath = p } return e }) inputGroup.Go(func() error { p, e := server.localAnnotationSource(inputCtx, task.OrganizationID, rightPTSPath, filepath.Join(stage, "source-right.csv")) if e == nil { rightPTSPath = p } return e }) inputGroup.Go(func() error { p, e := server.localAnnotationSource(inputCtx, task.OrganizationID, imuPath, filepath.Join(stage, "source-imu.csv")) if e == nil { imuPath = p } return e }) if e := inputGroup.Wait(); e != nil { return nil, e } leftPTS, err := readHardwarePTS(leftPTSPath) if err != nil { return nil, err } rightPTS, err := readHardwarePTS(rightPTSPath) if err != nil { return nil, err } imuRows, err := readHardwareIMUContext(ctx, imuPath) if err != nil { return nil, err } if len(reusable) == 0 { if err := stageVideos(); err != nil { return nil, err } leftFrames, err := ffprobeFrameCount(ctx, leftPath) if err != nil || leftFrames != len(leftPTS) { return nil, fmt.Errorf("left video/PTS frame mismatch: video=%d pts=%d err=%v", leftFrames, len(leftPTS), err) } rightFrames, err := ffprobeFrameCount(ctx, rightPath) if err != nil || rightFrames != len(rightPTS) { return nil, fmt.Errorf("right video/PTS frame mismatch: video=%d pts=%d err=%v", rightFrames, len(rightPTS), err) } } leftMediaDuration, err := ffprobeDurationSeconds(ctx, leftPath) if err != nil { return nil, err } rightMediaDuration, err := ffprobeDurationSeconds(ctx, rightPath) if err != nil { return nil, err } leftCoverageEnd, err := ptsCoverageEnd(leftPTS) if err != nil { return nil, err } rightCoverageEnd, err := ptsCoverageEnd(rightPTS) if err != nil { return nil, err } imuEnd, err := imuCoverageEnd(imuRows) if err != nil { return nil, err } commonStart := max(leftPTS[0].TimestampUS, rightPTS[0].TimestampUS, imuRows[0].TimestampUS) commonEnd := min(leftCoverageEnd, rightCoverageEnd, imuEnd) if commonEnd <= commonStart { return nil, fmt.Errorf("required streams have no common hardware-time range") } grid, err := newAnnotationBoundaryGrid(leftPTS, rightPTS, commonStart, commonEnd, leftMediaDuration) if err != nil { return nil, err } ptsToMediaScale := grid.Scale timestampToMediaSecond := func(timestampUS int64) float64 { i := sort.Search(len(grid.Timestamps), func(i int) bool { return grid.Timestamps[i] >= timestampUS }) if i >= len(grid.Seconds) { i = len(grid.Seconds) - 1 } return grid.Seconds[i] } mediaSecondToTimestamp := func(second float64) int64 { i := sort.SearchFloat64s(grid.Seconds, second) if i >= len(grid.Seconds) { i-- } if i > 0 && math.Abs(grid.Seconds[i-1]-second) < math.Abs(grid.Seconds[i]-second) { i-- } return grid.Timestamps[i] } commonMediaStart := timestampToMediaSecond(commonStart) commonMediaEnd := timestampToMediaSecond(commonEnd) if commonMediaEnd <= commonMediaStart { return nil, fmt.Errorf("common hardware-time range cannot be mapped to the media timeline") } publishedRoot := filepath.Join(filepath.Clean(server.workspaceMountPath), "system", "runs", task.RunID, "human-materializations", item.ID, task.ID, fmt.Sprintf("revision-%06d", revision)) root := filepath.Join(stage, "output") if err := os.MkdirAll(filepath.Join(root, "slices"), 0o755); err != nil { return nil, err } slices := append([]domain.SliceSegment(nil), payload.SliceSegments...) sort.Slice(slices, func(left, right int) bool { return slices[left].Start < slices[right].Start }) if len(slices) == 0 { return nil, fmt.Errorf("full synchronized coverage requires at least one slice") } normalizedSlices := make([]domain.SliceSegment, 0, len(slices)) previousEndUS := commonStart for _, slice := range slices { startIndex, err := grid.resolve(slice.Start) if err != nil { return nil, err } endIndex, err := grid.resolve(slice.End) if err != nil { return nil, err } startUS, endUS := grid.Timestamps[startIndex], grid.Timestamps[endIndex] if startUS != previousEndUS || endUS <= startUS || endUS > commonEnd { return nil, fmt.Errorf("slice %s leaves a gap or overlap in the common hardware-time range", slice.ID) } normalized := slice normalized.Start = timestampToMediaSecond(startUS) normalized.End = timestampToMediaSecond(endUS) normalized.StartTimestampUS, normalized.EndTimestampUS = startUS, endUS normalizedSlices = append(normalizedSlices, normalized) previousEndUS = endUS } if previousEndUS != commonEnd { return nil, fmt.Errorf("submitted slices do not reach the common hardware-time end") } normalizedSegments := make([]domain.AnnotationSegment, 0, len(payload.Segments)) for _, segment := range payload.Segments { si, e := grid.resolve(segment.Start) if e != nil { return nil, e } ei, e := grid.resolve(segment.End) if e != nil { return nil, e } if ei <= si { return nil, fmt.Errorf("动作区间必须包含有效帧,不能在同一边界上使用不同时间") } segment.Start, segment.End = grid.Seconds[si], grid.Seconds[ei] segment.StartTimestampUS, segment.EndTimestampUS = grid.Timestamps[si], grid.Timestamps[ei] normalizedSegments = append(normalizedSegments, segment) } if len(normalizedSegments) > 0 || continuous { if issues := validateDescriptionIntervals(normalizedSlices, normalizedSegments, continuous || task.TemplateVersion < 6); len(issues) > 0 { return nil, fmt.Errorf("%s", issues[0].Message) } } sort.Slice(normalizedSegments, func(i, j int) bool { return normalizedSegments[i].Start < normalizedSegments[j].Start }) leftCommonStart := sort.Search(len(leftPTS), func(index int) bool { return leftPTS[index].TimestampUS >= commonStart }) leftCommonEnd := sort.Search(len(leftPTS), func(index int) bool { return leftPTS[index].TimestampUS >= commonEnd }) rightCommonStart := sort.Search(len(rightPTS), func(index int) bool { return rightPTS[index].TimestampUS >= commonStart }) rightCommonEnd := sort.Search(len(rightPTS), func(index int) bool { return rightPTS[index].TimestampUS >= commonEnd }) imuCommonCount := 0 for _, row := range imuRows { if row.TimestampUS >= commonStart && row.TimestampUS < commonEnd { imuCommonCount++ } } deliveredLeftFrames, deliveredRightFrames, deliveredIMURows := 0, 0, 0 manifestItems := make([]map[string]any, 0, len(slices)) keptSliceCount, discardedSliceCount := 0, 0 reusedSlices := 0 reusedByIndex := map[int]bool{} leftCuts, rightCuts := []annotationFrameSlice{}, []annotationFrameSlice{} for i, slice := range normalizedSlices { a, b := mediaSecondToTimestamp(slice.Start), mediaSecondToTimestamp(slice.End) ls := sort.Search(len(leftPTS), func(j int) bool { return leftPTS[j].TimestampUS >= a }) le := sort.Search(len(leftPTS), func(j int) bool { return leftPTS[j].TimestampUS >= b }) rs := sort.Search(len(rightPTS), func(j int) bool { return rightPTS[j].TimestampUS >= a }) re := sort.Search(len(rightPTS), func(j int) bool { return rightPTS[j].TimestampUS >= b }) dir := filepath.Join(root, "slices", fmt.Sprintf("slice-%06d", i+1)) if e := os.MkdirAll(dir, 0755); e != nil { return nil, e } leftOut, rightOut := filepath.Join(dir, "left.mp4"), filepath.Join(dir, "right.mp4") key := fmt.Sprintf("%s:h264-ultrafast-crf23-frame-order-v2:%d:%d:%d:%d", sourceFingerprint, ls, le, rs, re) cached, ok := reusable[key] reusedByIndex[i] = ok && copyReusableVideo(ctx, cached, "left", leftOut) && copyReusableVideo(ctx, cached, "right", rightOut) if !reusedByIndex[i] { leftCuts = append(leftCuts, annotationFrameSlice{ls, le, leftOut}) rightCuts = append(rightCuts, annotationFrameSlice{rs, re, rightOut}) } } if len(leftCuts) > 0 { if err := stageVideos(); err != nil { return nil, err } progress("slicing", 0) var leftErr, rightErr error var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done(); leftErr = writeFrameAccurateSlices(ctx, leftPath, leftCuts) }() go func() { defer wg.Done(); rightErr = writeFrameAccurateSlices(ctx, rightPath, rightCuts) }() wg.Wait() if leftErr != nil { return nil, leftErr } if rightErr != nil { return nil, rightErr } } keptDuration, discardedDuration := 0.0, 0.0 for index, slice := range normalizedSlices { progress("slicing", index) if slice.IsDiscarded() { discardedSliceCount++ discardedDuration += slice.End - slice.Start } else { keptSliceCount++ keptDuration += slice.End - slice.Start } startUS := mediaSecondToTimestamp(slice.Start) endUS := mediaSecondToTimestamp(slice.End) if index == 0 { startUS = commonStart } if index == len(normalizedSlices)-1 { endUS = commonEnd } leftStart := sort.Search(len(leftPTS), func(i int) bool { return leftPTS[i].TimestampUS >= startUS }) leftEnd := sort.Search(len(leftPTS), func(i int) bool { return leftPTS[i].TimestampUS >= endUS }) rightStart := sort.Search(len(rightPTS), func(i int) bool { return rightPTS[i].TimestampUS >= startUS }) rightEnd := sort.Search(len(rightPTS), func(i int) bool { return rightPTS[i].TimestampUS >= endUS }) sliceID := fmt.Sprintf("slice-%06d", index+1) directory := filepath.Join(root, "slices", sliceID) if err := os.MkdirAll(directory, 0o755); err != nil { return nil, err } leftOutput, rightOutput := filepath.Join(directory, "left.mp4"), filepath.Join(directory, "right.mp4") mediaKey := fmt.Sprintf("%s:h264-ultrafast-crf23-frame-order-v2:%d:%d:%d:%d", sourceFingerprint, leftStart, leftEnd, rightStart, rightEnd) reused := reusedByIndex[index] if reused { reusedSlices++ } leftCount, rightCount := leftEnd-leftStart, rightEnd-rightStart if !reused { var leftErr, rightErr error leftCount, leftErr = ffprobeFrameCount(ctx, leftOutput) rightCount, rightErr = ffprobeFrameCount(ctx, rightOutput) if leftErr != nil || rightErr != nil || leftCount != leftEnd-leftStart || rightCount != rightEnd-rightStart { return nil, fmt.Errorf("materialized video frame count does not match sliced PTS rows: slice=%s left=%d expected=%d right=%d expected=%d leftProbe=%v rightProbe=%v context=%v", sliceID, leftCount, leftEnd-leftStart, rightCount, rightEnd-rightStart, leftErr, rightErr, ctx.Err()) } } deliveredLeftFrames += leftCount deliveredRightFrames += rightCount leftManifest, err := writePTSSlice(filepath.Join(directory, "left_pts.csv"), leftPTS, leftStart, leftEnd, startUS) if err != nil { return nil, err } rightManifest, err := writePTSSlice(filepath.Join(directory, "right_pts.csv"), rightPTS, rightStart, rightEnd, startUS) if err != nil { return nil, err } imuManifest, err := writeIMUSlice(filepath.Join(directory, "imu.csv"), imuRows, startUS, endUS) if err != nil { return nil, err } deliveredIMURows += int(imuManifest["sampleCount"].(int)) actions := make([]map[string]any, 0) for _, segment := range normalizedSegments { if slice.IsDiscarded() { continue } segmentStart := math.Max(segment.Start, slice.Start) segmentEnd := math.Min(segment.End, slice.End) if segmentEnd <= segmentStart { continue } relativeStart := segmentStart - slice.Start relativeEnd := segmentEnd - slice.Start actions = append(actions, map[string]any{"id": segment.ID, "status": effectiveActionStatus(segment), "isInvalid": segment.IsInvalid, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "dictionary": segment.Dictionary, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": relativeStart * 1000, "clipEndMs": relativeEnd * 1000, "startTimestampUs": mediaSecondToTimestamp(segmentStart), "endTimestampUs": mediaSecondToTimestamp(segmentEnd)}) } annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": sliceID, "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions} annotationSHA, annotationBytes, err := writeJSONFile(filepath.Join(directory, "annotation.json"), annotation) if err != nil { return nil, err } leftVideo := artifactForMaterializationPath(leftOutput, "leftVideo", domain.PortTypeVideo) rightVideo := artifactForMaterializationPath(rightOutput, "rightVideo", domain.PortTypeVideo) maximumAlignmentUS := maximumNearestAlignmentUS(leftPTS, leftStart, leftEnd, rightPTS, rightStart, rightEnd) boundaryTimes, rightBoundaryTimes := []float64{}, []float64{} boundaryIDs := []int64{} for k, stamp := range grid.Timestamps { if stamp < startUS || stamp > endUS { continue } boundaryIDs = append(boundaryIDs, stamp) boundaryTimes = append(boundaryTimes, grid.Seconds[k]-slice.Start) rightBoundaryTimes = append(rightBoundaryTimes, float64(grid.RightFrames[k]-rightStart)*rightMediaDuration/float64(len(rightPTS))) } synchronization := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "collectionId": collectionID, "sliceIndex": index + 1, "sliceCount": len(normalizedSlices), "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "commonRange": map[string]any{"startTimestampUs": commonStart, "endTimestampUs": commonEnd, "startSecond": commonMediaStart, "endSecond": commonMediaEnd}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true, "previousEndTimestampUs": func() any { if index == 0 { return nil } return manifestItems[index-1]["endTimestampUs"] }(), "nextStartTimestampUs": func() any { if index == len(normalizedSlices)-1 { return nil } return endUS }()}, "maxStereoAlignmentErrorUs": maximumAlignmentUS, "left": map[string]any{"video": leftVideo, "pts": leftManifest}, "right": map[string]any{"video": rightVideo, "pts": rightManifest}, "imu": imuManifest, "lineage": map[string]any{"leftVideoArtifactId": leftArtifact.ID, "rightVideoArtifactId": rightArtifact.ID, "leftPtsArtifactId": leftPTSArtifact.ID, "rightPtsArtifactId": rightPTSArtifact.ID, "imuArtifactId": imuArtifact.ID}} synchronization["boundaryContract"] = "hardware-frame-v1" synchronization["deliveryDisposition"] = sliceDisposition(slice) if slice.IsDiscarded() { synchronization["discardEvidence"] = discardReasonEvidence(slice) } synchronization["boundaryTimestampUs"] = boundaryIDs synchronization["boundarySeconds"] = boundaryTimes synchronization["rightBoundarySeconds"] = rightBoundaryTimes syncSHA, syncBytes, err := writeJSONFile(filepath.Join(directory, "synchronized-slice.json"), synchronization) if err != nil { return nil, err } manifestItems = append(manifestItems, map[string]any{"sliceId": sliceID, "sourceSliceId": slice.ID, "sliceName": slice.Name, "dictionary": slice.Dictionary, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": startUS, "endTimestampUs": endUS, "directory": filepath.ToSlash(filepath.Join("slices", sliceID)), "files": map[string]any{"leftVideo": leftVideo, "rightVideo": rightVideo, "leftPts": leftManifest, "rightPts": rightManifest, "imu": imuManifest, "annotation": map[string]any{"name": "annotation.json", "sha256": annotationSHA, "bytes": annotationBytes}, "synchronization": map[string]any{"name": "synchronized-slice.json", "sha256": syncSHA, "bytes": syncBytes}}}) manifestItems[len(manifestItems)-1]["deliveryDisposition"] = sliceDisposition(slice) manifestItems[len(manifestItems)-1]["mediaKey"] = mediaKey if slice.IsDiscarded() { manifestItems[len(manifestItems)-1]["discardEvidence"] = synchronization["discardEvidence"] } } expectedLeftFrames := leftCommonEnd - leftCommonStart expectedRightFrames := rightCommonEnd - rightCommonStart if deliveredLeftFrames != expectedLeftFrames || deliveredRightFrames != expectedRightFrames || deliveredIMURows != imuCommonCount { return nil, fmt.Errorf("slice reconstruction coverage mismatch: left=%d/%d right=%d/%d imu=%d/%d", deliveredLeftFrames, expectedLeftFrames, deliveredRightFrames, expectedRightFrames, deliveredIMURows, imuCommonCount) } coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": commonMediaStart, "endSecond": commonMediaEnd, "durationSeconds": commonMediaEnd - commonMediaStart, "startTimestampUs": commonStart, "endTimestampUs": commonEnd, "ptsToMediaScale": ptsToMediaScale, "sliceCount": len(manifestItems), "reconstruction": map[string]any{"leftFrameCount": deliveredLeftFrames, "rightFrameCount": deliveredRightFrames, "imuRowCount": deliveredIMURows}} captureEvidence := map[string]any{} var originalSource map[string]any if capture, captureErr := server.store.GetCaptureRecordByRunID(ctx, task.RunID); captureErr != nil { return nil, captureErr } else if capture != nil { originalSource, err = annotationRevisionOriginalSource(run, capture) if err != nil { return nil, err } captureEvidence = map[string]any{ "captureRecordId": capture.ID, "captureId": capture.CaptureID, "prefix": capture.Prefix, "fingerprint": capture.Fingerprint, "contractRevision": capture.ContractRevision, } } revisionManifest := map[string]any{"schemaVersion": "droboticflow.slice-materialization/v2", "revision": revision, "humanItemId": item.ID, "taskId": task.ID, "runId": task.RunID, "collectionId": collectionID, "capture": captureEvidence, "status": "materialized", "sliceCount": len(manifestItems), "coverage": coverage, "items": manifestItems} if originalSource != nil { revisionManifest["originalSource"] = originalSource } deliverySelection := map[string]any{"keptSliceCount": keptSliceCount, "discardedSliceCount": discardedSliceCount, "keptDurationSeconds": keptDuration, "discardedDurationSeconds": discardedDuration} if discardedSliceCount > 0 { revisionManifest["schemaVersion"] = "droboticflow.slice-materialization/v3" revisionManifest["deliverySelection"] = deliverySelection } manifestPath := filepath.Join(root, "revision-manifest.json") manifestSHA, manifestBytes, err := writeJSONFile(manifestPath, revisionManifest) if err != nil { return nil, err } afterFingerprint, err := server.annotationSourceFingerprint(ctx, task, *run) if err != nil { return nil, err } if afterFingerprint != sourceFingerprint { return nil, fmt.Errorf("源文件在切片期间发生变化,请重新确认切片") } progress("publishing", len(normalizedSlices)) if err := server.assertAnnotationLease(ctx); err != nil { return nil, err } manifestSHA, manifestBytes, err = server.publishAnnotationDirectory(ctx, task.OrganizationID, root, publishedRoot, revisionManifest) if err != nil { return nil, err } root, manifestPath = publishedRoot, filepath.Join(publishedRoot, "revision-manifest.json") log.Printf("[api] materialization_media task=%s revision=%d reused_slices=%d slices=%d", task.ID, revision, reusedSlices, len(normalizedSlices)) artifact := domain.ExecutionArtifact{ID: artifactID, RunID: task.RunID, NodeID: task.NodeID, PortID: "materializedSliceDirectory", PortName: "materializedSliceDirectory", Name: fmt.Sprintf("revision-%06d.directory.index", revision), Type: domain.PortTypeFile, Size: fmt.Sprintf("%d files", len(manifestItems)*7+1), SizeBytes: manifestBytes, Path: root, Metadata: &domain.ArtifactMetadata{SHA256: manifestSHA, MimeType: "application/json"}, Data: map[string]any{"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": root, "manifestPath": manifestPath, "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "normalizedSlices": normalizedSlices, "normalizedSegments": normalizedSegments}} artifact.Data["boundaryContract"] = "hardware-frame-v1" artifact.Data["sourceFingerprint"] = sourceFingerprint artifact.Data["reusedSliceCount"] = reusedSlices artifact.Data["deliverySelection"] = deliverySelection if err := server.storeAnnotationMaterializationArtifact(ctx, task, run, artifact); err != nil { return nil, err } return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "deliverySelection": deliverySelection, "runId": task.RunID, "artifactId": artifact.ID, "manifestSha256": manifestSHA}, Slices: normalizedSlices, Segments: normalizedSegments, CoverageStartSecond: commonMediaStart, CoverageEndSecond: commonMediaEnd}, nil }