droboticflow/internal/api/human_slice_materialization.go

1041 lines
43 KiB
Go

package api
import (
"context"
"crypto/sha256"
"encoding/csv"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"log"
"math"
"net/url"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
"golang.org/x/sync/errgroup"
)
type hardwarePTSRow struct {
TimestampUS int64
}
type hardwareIMURow struct {
TimestampUS int64
X string
Y string
Z string
Type string
}
type humanSliceMaterialization struct {
Artifact domain.ExecutionArtifact
Payload map[string]any
Slices []domain.SliceSegment
Segments []domain.AnnotationSegment
CoverageStartSecond float64
CoverageEndSecond float64
}
func decodeMaterializationValue[T any](value any) []T {
encoded, err := json.Marshal(normalizeStructuredValue(value))
if err != nil {
return nil
}
var items []T
if err := json.Unmarshal(encoded, &items); err != nil {
return nil
}
return items
}
func normalizedCSVHeader(value string) string {
return strings.ToLower(strings.TrimSpace(strings.TrimPrefix(value, "\ufeff")))
}
func csvRecords(path string) ([][]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
prefix := make([]byte, 4096)
count, _ := file.Read(prefix)
if _, err := file.Seek(0, io.SeekStart); err != nil {
return nil, err
}
reader := csv.NewReader(file)
if strings.Count(string(prefix[:count]), "\t") > strings.Count(string(prefix[:count]), ",") {
reader.Comma = '\t'
}
reader.FieldsPerRecord = -1
records, err := reader.ReadAll()
if err != nil {
return nil, err
}
if len(records) < 2 {
return nil, fmt.Errorf("CSV %s has no data rows", filepath.Base(path))
}
return records, nil
}
func headerIndex(headers []string, names ...string) int {
wanted := map[string]struct{}{}
for _, name := range names {
wanted[normalizedCSVHeader(name)] = struct{}{}
}
for index, header := range headers {
if _, ok := wanted[normalizedCSVHeader(header)]; ok {
return index
}
}
return -1
}
func readHardwarePTS(path string) ([]hardwarePTSRow, error) {
records, err := csvRecords(path)
if err != nil {
return nil, err
}
index := headerIndex(records[0], "timestamp_us", "timestampus", "pts_us")
if index < 0 {
return nil, fmt.Errorf("PTS CSV %s is missing timestamp_us", filepath.Base(path))
}
rows := make([]hardwarePTSRow, 0, len(records)-1)
var previous int64
for rowIndex, record := range records[1:] {
if index >= len(record) {
return nil, fmt.Errorf("PTS CSV %s row %d is incomplete", filepath.Base(path), rowIndex+2)
}
value, parseErr := strconv.ParseInt(strings.TrimSpace(record[index]), 10, 64)
if parseErr != nil || value <= 0 || (len(rows) > 0 && value <= previous) {
return nil, fmt.Errorf("PTS CSV %s is invalid or not strictly increasing at row %d", filepath.Base(path), rowIndex+2)
}
rows = append(rows, hardwarePTSRow{TimestampUS: value})
previous = value
}
return rows, nil
}
func readHardwareIMU(path string) ([]hardwareIMURow, error) {
return readHardwareIMUContext(context.Background(), path)
}
func readHardwareIMUContext(ctx context.Context, path string) ([]hardwareIMURow, error) {
rows := make([]hardwareIMURow, 0)
_, err := scanHardwareIMU(ctx, path, func(row hardwareIMURow) { rows = append(rows, row) })
return rows, err
}
type hardwareIMUSummary struct {
first, previous, last int64
count int
}
func (s hardwareIMUSummary) coverageEnd() (int64, error) {
if s.count < 4 || s.previous <= 0 || s.last <= s.previous {
return 0, fmt.Errorf("IMU requires at least two accel/gyro timestamp pairs")
}
return s.last + s.last - s.previous, nil
}
// Sorted timestamps allow pair validation in constant space. Timeline callers
// consume only the summary, rather than allocating 1.7 million sample objects.
func scanHardwareIMU(ctx context.Context, path string, consume func(hardwareIMURow)) (hardwareIMUSummary, error) {
var summary hardwareIMUSummary
file, err := os.Open(path)
if err != nil {
return summary, err
}
defer file.Close()
prefix := make([]byte, 4096)
n, _ := file.Read(prefix)
if _, err = file.Seek(0, io.SeekStart); err != nil {
return summary, err
}
reader := csv.NewReader(file)
reader.FieldsPerRecord = -1
reader.ReuseRecord = true
if strings.Count(string(prefix[:n]), "\t") > strings.Count(string(prefix[:n]), ",") {
reader.Comma = '\t'
}
headers, err := reader.Read()
if err != nil {
return summary, err
}
timestampIndex := headerIndex(headers, "timestamp_us", "timestampus")
xIndex, yIndex, zIndex := headerIndex(headers, "x"), headerIndex(headers, "y"), headerIndex(headers, "z")
typeIndex := headerIndex(headers, "type")
if timestampIndex < 0 || xIndex < 0 || yIndex < 0 || zIndex < 0 || typeIndex < 0 {
return summary, fmt.Errorf("IMU CSV %s is missing timestamp_us/x/y/z/type", filepath.Base(path))
}
maximum := max(timestampIndex, xIndex, yIndex, zIndex, typeIndex)
accel, gyro := false, false
for rowIndex := 2; ; rowIndex++ {
if err := ctx.Err(); err != nil {
return summary, err
}
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return summary, err
}
if maximum >= len(record) {
return summary, fmt.Errorf("IMU CSV %s row %d is incomplete", filepath.Base(path), rowIndex)
}
timestamp, parseErr := strconv.ParseInt(strings.TrimSpace(record[timestampIndex]), 10, 64)
kind := strings.ToLower(strings.TrimSpace(record[typeIndex]))
if parseErr != nil || timestamp <= 0 || (summary.count > 0 && timestamp < summary.last) || (kind != "accel" && kind != "gyro") {
return summary, fmt.Errorf("IMU CSV %s has invalid ordering or type at row %d", filepath.Base(path), rowIndex)
}
if summary.count == 0 {
summary.first = timestamp
}
if timestamp != summary.last {
if summary.count > 0 && (!accel || !gyro) {
return summary, fmt.Errorf("IMU timestamp %d is missing an accel/gyro pair", summary.last)
}
summary.previous = summary.last
accel, gyro = false, false
}
accel = accel || kind == "accel"
gyro = gyro || kind == "gyro"
summary.last = timestamp
summary.count++
if consume != nil {
consume(hardwareIMURow{TimestampUS: timestamp, X: strings.TrimSpace(record[xIndex]), Y: strings.TrimSpace(record[yIndex]), Z: strings.TrimSpace(record[zIndex]), Type: kind})
}
}
if summary.count == 0 {
return summary, fmt.Errorf("CSV %s has no data rows", filepath.Base(path))
}
if !accel || !gyro {
return summary, fmt.Errorf("IMU timestamp %d is missing an accel/gyro pair", summary.last)
}
return summary, nil
}
func artifactCollectionID(artifact domain.ExecutionArtifact) string {
if artifact.Data == nil {
return ""
}
return strings.TrimSpace(fmt.Sprint(artifact.Data["collectionId"]))
}
func artifactsForTarget(execution domain.NodeExecutionRecord, targetPortID string) []domain.ExecutionArtifact {
for _, binding := range execution.InputBindings {
if binding.TargetPortID == targetPortID {
return append([]domain.ExecutionArtifact(nil), binding.Artifacts...)
}
}
return nil
}
func artifactByID(items []domain.ExecutionArtifact, id string) *domain.ExecutionArtifact {
for index := range items {
if items[index].ID == id {
copy := items[index]
return &copy
}
}
return nil
}
func matchingArtifact(items []domain.ExecutionArtifact, collectionID string, nameParts ...string) *domain.ExecutionArtifact {
for index := range items {
name := strings.ToLower(items[index].Name)
if collectionID != "" && artifactCollectionID(items[index]) != "" && artifactCollectionID(items[index]) != collectionID {
continue
}
matched := true
for _, part := range nameParts {
if !strings.Contains(name, strings.ToLower(part)) {
matched = false
}
}
if matched {
copy := items[index]
return &copy
}
}
return nil
}
func (server *server) safeMaterializationSourcePath(artifact *domain.ExecutionArtifact) (string, error) {
if artifact == nil {
return "", fmt.Errorf("materialization input artifact is missing")
}
root := filepath.Clean(server.workspaceMountPath)
if root == "." || root == "" {
return "", fmt.Errorf("materialization workspace is unavailable")
}
pathValue := strings.TrimSpace(artifact.Path)
candidates := make([]string, 0, 3)
if pathValue != "" && !strings.Contains(pathValue, "://") {
candidates = append(candidates, filepath.Clean(pathValue))
}
if strings.HasPrefix(strings.ToLower(pathValue), "bos://") {
if parsed, err := url.Parse(pathValue); err == nil {
key := strings.TrimPrefix(strings.TrimSpace(parsed.Path), "/")
if key != "" {
candidates = append(candidates, filepath.Join(root, filepath.FromSlash(key)))
}
}
}
if artifact.Payload != nil && artifact.Payload.Ref != nil {
key := strings.TrimPrefix(strings.TrimSpace(artifact.Payload.Ref.Key), "/")
if key == "" {
key = strings.TrimPrefix(strings.TrimSpace(artifact.Payload.Ref.Prefix), "/")
}
if key != "" {
candidates = append(candidates, filepath.Join(root, filepath.FromSlash(key)))
}
}
for _, candidate := range candidates {
candidate = filepath.Clean(candidate)
if candidate != root && !strings.HasPrefix(candidate, root+string(os.PathSeparator)) {
continue
}
info, err := os.Stat(candidate)
if err == nil && info.Mode().IsRegular() && info.Size() > 0 {
return candidate, nil
}
}
return "", fmt.Errorf("materialization input %s is unavailable on the authorized workspace mount", artifact.Name)
}
func ffprobeFrameCount(ctx context.Context, path string) (int, error) {
output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-threads", "2", "-select_streams", "v:0", "-count_frames", "-show_entries", "stream=nb_read_frames", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput()
if err != nil {
return 0, fmt.Errorf("ffprobe %s: %s", filepath.Base(path), strings.TrimSpace(string(output)))
}
count, err := strconv.Atoi(strings.TrimSpace(string(output)))
if err != nil || count <= 0 {
return 0, fmt.Errorf("ffprobe returned invalid frame count for %s", filepath.Base(path))
}
return count, nil
}
// Timeline loading reads MP4 metadata, not every decoded frame of a long Capture.
// Materialization still performs the full frame-count integrity check above.
func ffprobeDeclaredFrameCount(ctx context.Context, path string) (int, error) {
output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=nb_frames", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput()
if err != nil {
return 0, fmt.Errorf("读取视频帧元数据失败:%s", filepath.Base(path))
}
count, err := strconv.Atoi(strings.TrimSpace(string(output)))
if err != nil || count <= 0 {
return 0, fmt.Errorf("视频缺少有效帧数元数据:%s", filepath.Base(path))
}
return count, nil
}
func ffprobeDurationSeconds(ctx context.Context, path string) (float64, error) {
output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=duration", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput()
if err != nil {
return 0, fmt.Errorf("ffprobe duration %s: %s", filepath.Base(path), strings.TrimSpace(string(output)))
}
duration, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64)
if err != nil || math.IsNaN(duration) || math.IsInf(duration, 0) || duration <= 0 {
return 0, fmt.Errorf("ffprobe returned invalid duration for %s", filepath.Base(path))
}
return duration, nil
}
func writeFrameAccurateSlice(ctx context.Context, source string, destination string, startFrame int, endFrame int) error {
if startFrame < 0 || endFrame <= startFrame {
return fmt.Errorf("frame slice is empty")
}
return writeFrameAccurateSlices(ctx, source, []annotationFrameSlice{{Start: startFrame, End: endFrame, Output: destination}})
}
func sha256File(path string) (string, int64, error) {
file, err := os.Open(path)
if err != nil {
return "", 0, err
}
defer file.Close()
hash := sha256.New()
size, err := io.Copy(hash, file)
if err != nil {
return "", 0, err
}
return hex.EncodeToString(hash.Sum(nil)), size, nil
}
func writePTSSlice(path string, rows []hardwarePTSRow, start int, end int, windowStartUS int64) (map[string]any, error) {
if start < 0 || end > len(rows) || end <= start {
return nil, fmt.Errorf("PTS slice is empty")
}
file, err := os.Create(path)
if err != nil {
return nil, err
}
writer := csv.NewWriter(file)
_ = writer.Write([]string{"frame_index", "timestamp_ms", "clip_time_ms"})
for index, row := range rows[start:end] {
_ = writer.Write([]string{strconv.Itoa(index), fmt.Sprintf("%.3f", float64(row.TimestampUS)/1000.0), fmt.Sprintf("%.3f", float64(row.TimestampUS-windowStartUS)/1000.0)})
}
writer.Flush()
closeErr := file.Close()
if writer.Error() != nil {
return nil, writer.Error()
}
if closeErr != nil {
return nil, closeErr
}
sha, size, err := sha256File(path)
return map[string]any{"name": filepath.Base(path), "sha256": sha, "bytes": size, "rowCount": end - start, "firstTimestampUs": rows[start].TimestampUS, "lastTimestampUs": rows[end-1].TimestampUS}, err
}
func writeIMUSlice(path string, rows []hardwareIMURow, startUS int64, endUS int64) (map[string]any, error) {
start := sort.Search(len(rows), func(i int) bool { return rows[i].TimestampUS >= startUS })
end := sort.Search(len(rows), func(i int) bool { return rows[i].TimestampUS >= endUS })
selected := rows[start:end]
if len(selected) == 0 {
return nil, fmt.Errorf("IMU slice is empty")
}
file, err := os.Create(path)
if err != nil {
return nil, err
}
writer := csv.NewWriter(file)
_ = writer.Write([]string{"timestamp_ms", "clip_time_ms", "x", "y", "z", "type"})
for _, row := range selected {
_ = writer.Write([]string{fmt.Sprintf("%.3f", float64(row.TimestampUS)/1000.0), fmt.Sprintf("%.3f", float64(row.TimestampUS-startUS)/1000.0), row.X, row.Y, row.Z, row.Type})
}
writer.Flush()
closeErr := file.Close()
if writer.Error() != nil {
return nil, writer.Error()
}
if closeErr != nil {
return nil, closeErr
}
sha, size, err := sha256File(path)
return map[string]any{"name": filepath.Base(path), "sha256": sha, "bytes": size, "sampleCount": len(selected), "firstTimestampUs": selected[0].TimestampUS, "lastTimestampUs": selected[len(selected)-1].TimestampUS}, err
}
func writeJSONFile(path string, payload any) (string, int64, error) {
encoded, err := json.MarshalIndent(payload, "", " ")
if err != nil {
return "", 0, err
}
encoded = append(encoded, '\n')
if err := os.WriteFile(path, encoded, 0o644); err != nil {
return "", 0, err
}
digest := sha256.Sum256(encoded)
return hex.EncodeToString(digest[:]), int64(len(encoded)), nil
}
func artifactForMaterializationPath(path string, id string, artifactType domain.PortType) map[string]any {
sha, size, _ := sha256File(path)
return map[string]any{"id": id, "name": filepath.Base(path), "sha256": sha, "bytes": size, "type": string(artifactType)}
}
func ptsCoverageEnd(rows []hardwarePTSRow) (int64, error) {
if len(rows) < 2 {
return 0, fmt.Errorf("PTS requires at least two rows to resolve the half-open coverage end")
}
interval := rows[len(rows)-1].TimestampUS - rows[len(rows)-2].TimestampUS
if interval <= 0 {
return 0, fmt.Errorf("PTS terminal interval is invalid")
}
return rows[len(rows)-1].TimestampUS + interval, nil
}
func imuCoverageEnd(rows []hardwareIMURow) (int64, error) {
if len(rows) < 4 {
return 0, fmt.Errorf("IMU requires at least two accel/gyro timestamp pairs")
}
last := rows[len(rows)-1].TimestampUS
previous := int64(0)
for index := len(rows) - 2; index >= 0; index-- {
if rows[index].TimestampUS < last {
previous = rows[index].TimestampUS
break
}
}
if previous <= 0 || last <= previous {
return 0, fmt.Errorf("IMU terminal interval is invalid")
}
return last + (last - previous), nil
}
func maximumNearestAlignmentUS(left []hardwarePTSRow, leftStart int, leftEnd int, right []hardwarePTSRow, rightStart int, rightEnd int) int64 {
maximum := int64(0)
for _, row := range left[leftStart:leftEnd] {
index := sort.Search(rightEnd-rightStart, func(index int) bool {
return right[rightStart+index].TimestampUS >= row.TimestampUS
})
nearest := int64(math.MaxInt64)
if index < rightEnd-rightStart {
nearest = absInt64(right[rightStart+index].TimestampUS - row.TimestampUS)
}
if index > 0 {
nearest = min(nearest, absInt64(right[rightStart+index-1].TimestampUS-row.TimestampUS))
}
maximum = max(maximum, nearest)
}
return maximum
}
func absInt64(value int64) int64 {
if value < 0 {
return -value
}
return value
}
func (server *server) materializeCombinedSliceSubmission(ctx context.Context, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, payload combinedAnnotationSubmissionInput, revision int) (*humanSliceMaterialization, error) {
if err := server.assertAnnotationLease(ctx); err != nil {
return nil, err
}
continuous := !payload.sliceOnly && server.continuousActionsForItem(ctx, &item)
if continuous {
if issues := validateContinuousActions(payload.SliceSegments, payload.Segments, true); len(issues) > 0 {
return nil, fmt.Errorf("%s", issues[0].Message)
}
}
started := time.Now()
ctx, cancel := context.WithTimeout(ctx, 24*time.Hour)
defer cancel()
progressKey := task.OrganizationID + ":" + task.ID
progress := func(stage string, completed int) {
server.materializationProgress.Store(progressKey, map[string]any{"stage": stage, "completedSlices": completed, "totalSlices": len(payload.SliceSegments), "startedAt": started.UTC().Format(time.RFC3339Nano)})
}
progress("queued", 0)
defer server.materializationProgress.Delete(progressKey)
server.materializationSlotsOnce.Do(func() { server.materializationSlots = make(chan struct{}, 1) })
select {
case server.materializationSlots <- struct{}{}:
case <-ctx.Done():
return nil, ctx.Err()
}
defer func() { <-server.materializationSlots }()
queueMS := time.Since(started).Milliseconds()
progress("validating", 0)
defer func() {
log.Printf("[api] materialization_timing task=%s revision=%d queue_ms=%d total_ms=%d", task.ID, revision, queueMS, time.Since(started).Milliseconds())
}()
run, err := server.store.GetRun(ctx, task.RunID)
if err != nil || run == nil {
return nil, fmt.Errorf("load materialization run: %w", err)
}
// Multiple Captures may share a legacy Run and node. Revision numbers are
// task-local, so both cache identity and storage must include the task ID.
artifactID := fmt.Sprintf("%s-%s-%s-materialized-revision-%06d", task.RunID, task.NodeID, task.ID, revision)
for _, existing := range run.Artifacts {
if existing.ID == artifactID {
if stringValue(existing.Data["boundaryContract"]) != "hardware-frame-v1" {
return nil, fmt.Errorf("历史切片不使用统一边界,请重新确认切片;不会自动移动已有标注")
}
manifestSHA := ""
if existing.Metadata != nil {
manifestSHA = existing.Metadata.SHA256
}
coverage := mapValue(existing.Data["coverage"])
slices := decodeMaterializationValue[domain.SliceSegment](existing.Data["normalizedSlices"])
segments := decodeMaterializationValue[domain.AnnotationSegment](existing.Data["normalizedSegments"])
sliceCount := intValue(existing.Data["sliceCount"])
if sliceCount == 0 {
sliceCount = intValue(coverage["sliceCount"])
}
if slices == nil {
slices = append([]domain.SliceSegment(nil), payload.SliceSegments...)
}
if segments == nil {
segments = append([]domain.AnnotationSegment(nil), payload.Segments...)
}
if continuous {
if issues := validateContinuousActions(slices, segments, true); len(issues) > 0 {
return nil, fmt.Errorf("%s", issues[0].Message)
}
}
return &humanSliceMaterialization{Artifact: existing, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": sliceCount, "coverage": coverage, "deliverySelection": existing.Data["deliverySelection"], "runId": task.RunID, "artifactId": existing.ID, "manifestSha256": manifestSHA}, Slices: slices, Segments: segments, CoverageStartSecond: floatValue(coverage["startSecond"]), CoverageEndSecond: floatValue(coverage["endSecond"])}, nil
}
}
sourceFingerprint, err := server.annotationSourceFingerprint(ctx, task, *run)
if err != nil {
return nil, err
}
reusable := reusableAnnotationMedia(*run, task, item, server.workspaceMountPath, sourceFingerprint, revision)
if metadata, e := server.tryAnnotationMetadataRevision(ctx, task, item, run, payload, revision, sourceFingerprint); e != nil {
return nil, e
} else if metadata != nil {
return metadata, nil
}
stage, err := os.MkdirTemp("", "annotation-materialize-")
if err != nil {
return nil, err
}
defer os.RemoveAll(stage)
var execution *domain.NodeExecutionRecord
for index := range run.NodeExecutions {
if run.NodeExecutions[index].NodeID == task.NodeID {
copy := run.NodeExecutions[index]
execution = &copy
break
}
}
if execution == nil {
return nil, fmt.Errorf("human gate execution is unavailable")
}
leftArtifact := artifactByID(artifactsForTarget(*execution, "fullLeftVideos"), task.ArtifactID)
if leftArtifact == nil {
leftArtifact = artifactByID(execution.InputArtifacts, task.ArtifactID)
}
if leftArtifact == nil {
return nil, fmt.Errorf("left source video is unavailable")
}
collectionID := artifactCollectionID(*leftArtifact)
rightArtifact := matchingArtifact(artifactsForTarget(*execution, "fullRightVideos"), collectionID, "right", ".mp4")
leftPTSArtifact := matchingArtifact(artifactsForTarget(*execution, "leftTimestamps"), collectionID, "left", "pts")
rightPTSArtifact := matchingArtifact(artifactsForTarget(*execution, "rightTimestamps"), collectionID, "right", "pts")
imuArtifact := matchingArtifact(artifactsForTarget(*execution, "imu"), collectionID, "imu", ".csv")
leftPath, err := server.safeMaterializationSourcePath(leftArtifact)
if err != nil {
return nil, err
}
rightPath, err := server.safeMaterializationSourcePath(rightArtifact)
if err != nil {
return nil, err
}
leftPTSPath, err := server.safeMaterializationSourcePath(leftPTSArtifact)
if err != nil {
return nil, err
}
rightPTSPath, err := server.safeMaterializationSourcePath(rightPTSArtifact)
if err != nil {
return nil, err
}
imuPath, err := server.safeMaterializationSourcePath(imuArtifact)
if err != nil {
return nil, err
}
// Decode and hash on local disk, not repeatedly through the BOS filesystem.
// Source videos are only copied if an unchanged slice cannot be reused.
stagedVideos := false
stageVideos := func() error {
if stagedVideos {
return nil
}
leftLocal, rightLocal := filepath.Join(stage, "source-left.mp4"), filepath.Join(stage, "source-right.mp4")
group, gctx := errgroup.WithContext(ctx)
group.Go(func() error {
p, e := server.localAnnotationSource(gctx, task.OrganizationID, leftPath, leftLocal)
if e == nil && p != leftLocal {
e = copyAnnotationFile(gctx, p, leftLocal, "")
}
return e
})
group.Go(func() error {
p, e := server.localAnnotationSource(gctx, task.OrganizationID, rightPath, rightLocal)
if e == nil && p != rightLocal {
e = copyAnnotationFile(gctx, p, rightLocal, "")
}
return e
})
if e := group.Wait(); e != nil {
return e
}
leftPath, rightPath, stagedVideos = leftLocal, rightLocal, true
return nil
}
inputGroup, inputCtx := errgroup.WithContext(ctx)
inputGroup.Go(func() error {
p, e := server.localAnnotationSource(inputCtx, task.OrganizationID, leftPTSPath, filepath.Join(stage, "source-left.csv"))
if e == nil {
leftPTSPath = p
}
return e
})
inputGroup.Go(func() error {
p, e := server.localAnnotationSource(inputCtx, task.OrganizationID, rightPTSPath, filepath.Join(stage, "source-right.csv"))
if e == nil {
rightPTSPath = p
}
return e
})
inputGroup.Go(func() error {
p, e := server.localAnnotationSource(inputCtx, task.OrganizationID, imuPath, filepath.Join(stage, "source-imu.csv"))
if e == nil {
imuPath = p
}
return e
})
if e := inputGroup.Wait(); e != nil {
return nil, e
}
leftPTS, err := readHardwarePTS(leftPTSPath)
if err != nil {
return nil, err
}
rightPTS, err := readHardwarePTS(rightPTSPath)
if err != nil {
return nil, err
}
imuRows, err := readHardwareIMUContext(ctx, imuPath)
if err != nil {
return nil, err
}
if len(reusable) == 0 {
if err := stageVideos(); err != nil {
return nil, err
}
leftFrames, err := ffprobeFrameCount(ctx, leftPath)
if err != nil || leftFrames != len(leftPTS) {
return nil, fmt.Errorf("left video/PTS frame mismatch: video=%d pts=%d err=%v", leftFrames, len(leftPTS), err)
}
rightFrames, err := ffprobeFrameCount(ctx, rightPath)
if err != nil || rightFrames != len(rightPTS) {
return nil, fmt.Errorf("right video/PTS frame mismatch: video=%d pts=%d err=%v", rightFrames, len(rightPTS), err)
}
}
leftMediaDuration, err := ffprobeDurationSeconds(ctx, leftPath)
if err != nil {
return nil, err
}
rightMediaDuration, err := ffprobeDurationSeconds(ctx, rightPath)
if err != nil {
return nil, err
}
leftCoverageEnd, err := ptsCoverageEnd(leftPTS)
if err != nil {
return nil, err
}
rightCoverageEnd, err := ptsCoverageEnd(rightPTS)
if err != nil {
return nil, err
}
imuEnd, err := imuCoverageEnd(imuRows)
if err != nil {
return nil, err
}
commonStart := max(leftPTS[0].TimestampUS, rightPTS[0].TimestampUS, imuRows[0].TimestampUS)
commonEnd := min(leftCoverageEnd, rightCoverageEnd, imuEnd)
if commonEnd <= commonStart {
return nil, fmt.Errorf("required streams have no common hardware-time range")
}
grid, err := newAnnotationBoundaryGrid(leftPTS, rightPTS, commonStart, commonEnd, leftMediaDuration)
if err != nil {
return nil, err
}
ptsToMediaScale := grid.Scale
timestampToMediaSecond := func(timestampUS int64) float64 {
i := sort.Search(len(grid.Timestamps), func(i int) bool { return grid.Timestamps[i] >= timestampUS })
if i >= len(grid.Seconds) {
i = len(grid.Seconds) - 1
}
return grid.Seconds[i]
}
mediaSecondToTimestamp := func(second float64) int64 {
i := sort.SearchFloat64s(grid.Seconds, second)
if i >= len(grid.Seconds) {
i--
}
if i > 0 && math.Abs(grid.Seconds[i-1]-second) < math.Abs(grid.Seconds[i]-second) {
i--
}
return grid.Timestamps[i]
}
commonMediaStart := timestampToMediaSecond(commonStart)
commonMediaEnd := timestampToMediaSecond(commonEnd)
if commonMediaEnd <= commonMediaStart {
return nil, fmt.Errorf("common hardware-time range cannot be mapped to the media timeline")
}
publishedRoot := filepath.Join(filepath.Clean(server.workspaceMountPath), "system", "runs", task.RunID, "human-materializations", item.ID, task.ID, fmt.Sprintf("revision-%06d", revision))
root := filepath.Join(stage, "output")
if err := os.MkdirAll(filepath.Join(root, "slices"), 0o755); err != nil {
return nil, err
}
slices := append([]domain.SliceSegment(nil), payload.SliceSegments...)
sort.Slice(slices, func(left, right int) bool { return slices[left].Start < slices[right].Start })
if len(slices) == 0 {
return nil, fmt.Errorf("full synchronized coverage requires at least one slice")
}
normalizedSlices := make([]domain.SliceSegment, 0, len(slices))
previousEndUS := commonStart
for _, slice := range slices {
startIndex, err := grid.resolve(slice.Start)
if err != nil {
return nil, err
}
endIndex, err := grid.resolve(slice.End)
if err != nil {
return nil, err
}
startUS, endUS := grid.Timestamps[startIndex], grid.Timestamps[endIndex]
if startUS != previousEndUS || endUS <= startUS || endUS > commonEnd {
return nil, fmt.Errorf("slice %s leaves a gap or overlap in the common hardware-time range", slice.ID)
}
normalized := slice
normalized.Start = timestampToMediaSecond(startUS)
normalized.End = timestampToMediaSecond(endUS)
normalized.StartTimestampUS, normalized.EndTimestampUS = startUS, endUS
normalizedSlices = append(normalizedSlices, normalized)
previousEndUS = endUS
}
if previousEndUS != commonEnd {
return nil, fmt.Errorf("submitted slices do not reach the common hardware-time end")
}
normalizedSegments := make([]domain.AnnotationSegment, 0, len(payload.Segments))
for _, segment := range payload.Segments {
si, e := grid.resolve(segment.Start)
if e != nil {
return nil, e
}
ei, e := grid.resolve(segment.End)
if e != nil {
return nil, e
}
if ei <= si {
return nil, fmt.Errorf("动作区间必须包含有效帧,不能在同一边界上使用不同时间")
}
segment.Start, segment.End = grid.Seconds[si], grid.Seconds[ei]
segment.StartTimestampUS, segment.EndTimestampUS = grid.Timestamps[si], grid.Timestamps[ei]
normalizedSegments = append(normalizedSegments, segment)
}
if len(normalizedSegments) > 0 || continuous {
if issues := validateDescriptionIntervals(normalizedSlices, normalizedSegments, continuous || task.TemplateVersion < 6); len(issues) > 0 {
return nil, fmt.Errorf("%s", issues[0].Message)
}
}
sort.Slice(normalizedSegments, func(i, j int) bool { return normalizedSegments[i].Start < normalizedSegments[j].Start })
leftCommonStart := sort.Search(len(leftPTS), func(index int) bool { return leftPTS[index].TimestampUS >= commonStart })
leftCommonEnd := sort.Search(len(leftPTS), func(index int) bool { return leftPTS[index].TimestampUS >= commonEnd })
rightCommonStart := sort.Search(len(rightPTS), func(index int) bool { return rightPTS[index].TimestampUS >= commonStart })
rightCommonEnd := sort.Search(len(rightPTS), func(index int) bool { return rightPTS[index].TimestampUS >= commonEnd })
imuCommonCount := 0
for _, row := range imuRows {
if row.TimestampUS >= commonStart && row.TimestampUS < commonEnd {
imuCommonCount++
}
}
deliveredLeftFrames, deliveredRightFrames, deliveredIMURows := 0, 0, 0
manifestItems := make([]map[string]any, 0, len(slices))
keptSliceCount, discardedSliceCount := 0, 0
reusedSlices := 0
reusedByIndex := map[int]bool{}
leftCuts, rightCuts := []annotationFrameSlice{}, []annotationFrameSlice{}
for i, slice := range normalizedSlices {
a, b := mediaSecondToTimestamp(slice.Start), mediaSecondToTimestamp(slice.End)
ls := sort.Search(len(leftPTS), func(j int) bool { return leftPTS[j].TimestampUS >= a })
le := sort.Search(len(leftPTS), func(j int) bool { return leftPTS[j].TimestampUS >= b })
rs := sort.Search(len(rightPTS), func(j int) bool { return rightPTS[j].TimestampUS >= a })
re := sort.Search(len(rightPTS), func(j int) bool { return rightPTS[j].TimestampUS >= b })
dir := filepath.Join(root, "slices", fmt.Sprintf("slice-%06d", i+1))
if e := os.MkdirAll(dir, 0755); e != nil {
return nil, e
}
leftOut, rightOut := filepath.Join(dir, "left.mp4"), filepath.Join(dir, "right.mp4")
key := fmt.Sprintf("%s:h264-ultrafast-crf23-frame-order-v2:%d:%d:%d:%d", sourceFingerprint, ls, le, rs, re)
cached, ok := reusable[key]
reusedByIndex[i] = ok && copyReusableVideo(ctx, cached, "left", leftOut) && copyReusableVideo(ctx, cached, "right", rightOut)
if !reusedByIndex[i] {
leftCuts = append(leftCuts, annotationFrameSlice{ls, le, leftOut})
rightCuts = append(rightCuts, annotationFrameSlice{rs, re, rightOut})
}
}
if len(leftCuts) > 0 {
if err := stageVideos(); err != nil {
return nil, err
}
progress("slicing", 0)
var leftErr, rightErr error
var wg sync.WaitGroup
wg.Add(2)
go func() { defer wg.Done(); leftErr = writeFrameAccurateSlices(ctx, leftPath, leftCuts) }()
go func() { defer wg.Done(); rightErr = writeFrameAccurateSlices(ctx, rightPath, rightCuts) }()
wg.Wait()
if leftErr != nil {
return nil, leftErr
}
if rightErr != nil {
return nil, rightErr
}
}
keptDuration, discardedDuration := 0.0, 0.0
for index, slice := range normalizedSlices {
progress("slicing", index)
if slice.IsDiscarded() {
discardedSliceCount++
discardedDuration += slice.End - slice.Start
} else {
keptSliceCount++
keptDuration += slice.End - slice.Start
}
startUS := mediaSecondToTimestamp(slice.Start)
endUS := mediaSecondToTimestamp(slice.End)
if index == 0 {
startUS = commonStart
}
if index == len(normalizedSlices)-1 {
endUS = commonEnd
}
leftStart := sort.Search(len(leftPTS), func(i int) bool { return leftPTS[i].TimestampUS >= startUS })
leftEnd := sort.Search(len(leftPTS), func(i int) bool { return leftPTS[i].TimestampUS >= endUS })
rightStart := sort.Search(len(rightPTS), func(i int) bool { return rightPTS[i].TimestampUS >= startUS })
rightEnd := sort.Search(len(rightPTS), func(i int) bool { return rightPTS[i].TimestampUS >= endUS })
sliceID := fmt.Sprintf("slice-%06d", index+1)
directory := filepath.Join(root, "slices", sliceID)
if err := os.MkdirAll(directory, 0o755); err != nil {
return nil, err
}
leftOutput, rightOutput := filepath.Join(directory, "left.mp4"), filepath.Join(directory, "right.mp4")
mediaKey := fmt.Sprintf("%s:h264-ultrafast-crf23-frame-order-v2:%d:%d:%d:%d", sourceFingerprint, leftStart, leftEnd, rightStart, rightEnd)
reused := reusedByIndex[index]
if reused {
reusedSlices++
}
leftCount, rightCount := leftEnd-leftStart, rightEnd-rightStart
if !reused {
var leftErr, rightErr error
leftCount, leftErr = ffprobeFrameCount(ctx, leftOutput)
rightCount, rightErr = ffprobeFrameCount(ctx, rightOutput)
if leftErr != nil || rightErr != nil || leftCount != leftEnd-leftStart || rightCount != rightEnd-rightStart {
return nil, fmt.Errorf("materialized video frame count does not match sliced PTS rows: slice=%s left=%d expected=%d right=%d expected=%d leftProbe=%v rightProbe=%v context=%v", sliceID, leftCount, leftEnd-leftStart, rightCount, rightEnd-rightStart, leftErr, rightErr, ctx.Err())
}
}
deliveredLeftFrames += leftCount
deliveredRightFrames += rightCount
leftManifest, err := writePTSSlice(filepath.Join(directory, "left_pts.csv"), leftPTS, leftStart, leftEnd, startUS)
if err != nil {
return nil, err
}
rightManifest, err := writePTSSlice(filepath.Join(directory, "right_pts.csv"), rightPTS, rightStart, rightEnd, startUS)
if err != nil {
return nil, err
}
imuManifest, err := writeIMUSlice(filepath.Join(directory, "imu.csv"), imuRows, startUS, endUS)
if err != nil {
return nil, err
}
deliveredIMURows += int(imuManifest["sampleCount"].(int))
actions := make([]map[string]any, 0)
for _, segment := range normalizedSegments {
if slice.IsDiscarded() {
continue
}
segmentStart := math.Max(segment.Start, slice.Start)
segmentEnd := math.Min(segment.End, slice.End)
if segmentEnd <= segmentStart {
continue
}
relativeStart := segmentStart - slice.Start
relativeEnd := segmentEnd - slice.Start
actions = append(actions, map[string]any{"id": segment.ID, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": relativeStart * 1000, "clipEndMs": relativeEnd * 1000, "startTimestampUs": mediaSecondToTimestamp(segmentStart), "endTimestampUs": mediaSecondToTimestamp(segmentEnd)})
}
annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": sliceID, "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions}
annotationSHA, annotationBytes, err := writeJSONFile(filepath.Join(directory, "annotation.json"), annotation)
if err != nil {
return nil, err
}
leftVideo := artifactForMaterializationPath(leftOutput, "leftVideo", domain.PortTypeVideo)
rightVideo := artifactForMaterializationPath(rightOutput, "rightVideo", domain.PortTypeVideo)
maximumAlignmentUS := maximumNearestAlignmentUS(leftPTS, leftStart, leftEnd, rightPTS, rightStart, rightEnd)
boundaryTimes, rightBoundaryTimes := []float64{}, []float64{}
boundaryIDs := []int64{}
for k, stamp := range grid.Timestamps {
if stamp < startUS || stamp > endUS {
continue
}
boundaryIDs = append(boundaryIDs, stamp)
boundaryTimes = append(boundaryTimes, grid.Seconds[k]-slice.Start)
rightBoundaryTimes = append(rightBoundaryTimes, float64(grid.RightFrames[k]-rightStart)*rightMediaDuration/float64(len(rightPTS)))
}
synchronization := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "collectionId": collectionID, "sliceIndex": index + 1, "sliceCount": len(normalizedSlices), "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "commonRange": map[string]any{"startTimestampUs": commonStart, "endTimestampUs": commonEnd, "startSecond": commonMediaStart, "endSecond": commonMediaEnd}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true, "previousEndTimestampUs": func() any {
if index == 0 {
return nil
}
return manifestItems[index-1]["endTimestampUs"]
}(), "nextStartTimestampUs": func() any {
if index == len(normalizedSlices)-1 {
return nil
}
return endUS
}()}, "maxStereoAlignmentErrorUs": maximumAlignmentUS, "left": map[string]any{"video": leftVideo, "pts": leftManifest}, "right": map[string]any{"video": rightVideo, "pts": rightManifest}, "imu": imuManifest, "lineage": map[string]any{"leftVideoArtifactId": leftArtifact.ID, "rightVideoArtifactId": rightArtifact.ID, "leftPtsArtifactId": leftPTSArtifact.ID, "rightPtsArtifactId": rightPTSArtifact.ID, "imuArtifactId": imuArtifact.ID}}
synchronization["boundaryContract"] = "hardware-frame-v1"
synchronization["deliveryDisposition"] = sliceDisposition(slice)
if slice.IsDiscarded() {
synchronization["discardEvidence"] = discardReasonEvidence(slice)
}
synchronization["boundaryTimestampUs"] = boundaryIDs
synchronization["boundarySeconds"] = boundaryTimes
synchronization["rightBoundarySeconds"] = rightBoundaryTimes
syncSHA, syncBytes, err := writeJSONFile(filepath.Join(directory, "synchronized-slice.json"), synchronization)
if err != nil {
return nil, err
}
manifestItems = append(manifestItems, map[string]any{"sliceId": sliceID, "sourceSliceId": slice.ID, "sliceName": slice.Name, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": startUS, "endTimestampUs": endUS, "directory": filepath.ToSlash(filepath.Join("slices", sliceID)), "files": map[string]any{"leftVideo": leftVideo, "rightVideo": rightVideo, "leftPts": leftManifest, "rightPts": rightManifest, "imu": imuManifest, "annotation": map[string]any{"name": "annotation.json", "sha256": annotationSHA, "bytes": annotationBytes}, "synchronization": map[string]any{"name": "synchronized-slice.json", "sha256": syncSHA, "bytes": syncBytes}}})
manifestItems[len(manifestItems)-1]["deliveryDisposition"] = sliceDisposition(slice)
manifestItems[len(manifestItems)-1]["mediaKey"] = mediaKey
if slice.IsDiscarded() {
manifestItems[len(manifestItems)-1]["discardEvidence"] = synchronization["discardEvidence"]
}
}
expectedLeftFrames := leftCommonEnd - leftCommonStart
expectedRightFrames := rightCommonEnd - rightCommonStart
if deliveredLeftFrames != expectedLeftFrames || deliveredRightFrames != expectedRightFrames || deliveredIMURows != imuCommonCount {
return nil, fmt.Errorf("slice reconstruction coverage mismatch: left=%d/%d right=%d/%d imu=%d/%d", deliveredLeftFrames, expectedLeftFrames, deliveredRightFrames, expectedRightFrames, deliveredIMURows, imuCommonCount)
}
coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": commonMediaStart, "endSecond": commonMediaEnd, "durationSeconds": commonMediaEnd - commonMediaStart, "startTimestampUs": commonStart, "endTimestampUs": commonEnd, "ptsToMediaScale": ptsToMediaScale, "sliceCount": len(manifestItems), "reconstruction": map[string]any{"leftFrameCount": deliveredLeftFrames, "rightFrameCount": deliveredRightFrames, "imuRowCount": deliveredIMURows}}
captureEvidence := map[string]any{}
if capture, captureErr := server.store.GetCaptureRecordByRunID(ctx, task.RunID); captureErr != nil {
return nil, captureErr
} else if capture != nil {
captureEvidence = map[string]any{
"captureRecordId": capture.ID, "captureId": capture.CaptureID, "prefix": capture.Prefix,
"fingerprint": capture.Fingerprint, "contractRevision": capture.ContractRevision,
}
}
revisionManifest := map[string]any{"schemaVersion": "droboticflow.slice-materialization/v2", "revision": revision, "humanItemId": item.ID, "taskId": task.ID, "runId": task.RunID, "collectionId": collectionID, "capture": captureEvidence, "status": "materialized", "sliceCount": len(manifestItems), "coverage": coverage, "items": manifestItems}
deliverySelection := map[string]any{"keptSliceCount": keptSliceCount, "discardedSliceCount": discardedSliceCount, "keptDurationSeconds": keptDuration, "discardedDurationSeconds": discardedDuration}
if discardedSliceCount > 0 {
revisionManifest["schemaVersion"] = "droboticflow.slice-materialization/v3"
revisionManifest["deliverySelection"] = deliverySelection
}
manifestPath := filepath.Join(root, "revision-manifest.json")
manifestSHA, manifestBytes, err := writeJSONFile(manifestPath, revisionManifest)
if err != nil {
return nil, err
}
afterFingerprint, err := server.annotationSourceFingerprint(ctx, task, *run)
if err != nil {
return nil, err
}
if afterFingerprint != sourceFingerprint {
return nil, fmt.Errorf("源文件在切片期间发生变化,请重新确认切片")
}
progress("publishing", len(normalizedSlices))
if err := server.assertAnnotationLease(ctx); err != nil {
return nil, err
}
manifestSHA, manifestBytes, err = server.publishAnnotationDirectory(ctx, task.OrganizationID, root, publishedRoot, revisionManifest)
if err != nil {
return nil, err
}
root, manifestPath = publishedRoot, filepath.Join(publishedRoot, "revision-manifest.json")
log.Printf("[api] materialization_media task=%s revision=%d reused_slices=%d slices=%d", task.ID, revision, reusedSlices, len(normalizedSlices))
artifact := domain.ExecutionArtifact{ID: artifactID, RunID: task.RunID, NodeID: task.NodeID, PortID: "materializedSliceDirectory", PortName: "materializedSliceDirectory", Name: fmt.Sprintf("revision-%06d.directory.index", revision), Type: domain.PortTypeFile, Size: fmt.Sprintf("%d files", len(manifestItems)*7+1), SizeBytes: manifestBytes, Path: root, Metadata: &domain.ArtifactMetadata{SHA256: manifestSHA, MimeType: "application/json"}, Data: map[string]any{"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": root, "manifestPath": manifestPath, "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "normalizedSlices": normalizedSlices, "normalizedSegments": normalizedSegments}}
artifact.Data["boundaryContract"] = "hardware-frame-v1"
artifact.Data["sourceFingerprint"] = sourceFingerprint
artifact.Data["reusedSliceCount"] = reusedSlices
artifact.Data["deliverySelection"] = deliverySelection
if err := server.storeAnnotationMaterializationArtifact(ctx, task, run, artifact); err != nil {
return nil, err
}
return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "deliverySelection": deliverySelection, "runId": task.RunID, "artifactId": artifact.ID, "manifestSha256": manifestSHA}, Slices: normalizedSlices, Segments: normalizedSegments, CoverageStartSecond: commonMediaStart, CoverageEndSecond: commonMediaEnd}, nil
}