droboticflow/internal/api/human_slice_materialization.go
2026-09-01 02:40:27 +08:00

745 lines
32 KiB
Go

package api
import (
"context"
"crypto/sha256"
"encoding/csv"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"math"
"net/url"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"droboticflow/internal/domain"
)
type hardwarePTSRow struct {
TimestampUS int64
}
type hardwareIMURow struct {
TimestampUS int64
X string
Y string
Z string
Type string
}
type humanSliceMaterialization struct {
Artifact domain.ExecutionArtifact
Payload map[string]any
Slices []domain.SliceSegment
Segments []domain.AnnotationSegment
CoverageStartSecond float64
CoverageEndSecond float64
}
func decodeMaterializationValue[T any](value any) []T {
encoded, err := json.Marshal(normalizeStructuredValue(value))
if err != nil {
return nil
}
var items []T
if err := json.Unmarshal(encoded, &items); err != nil {
return nil
}
return items
}
func normalizedCSVHeader(value string) string {
return strings.ToLower(strings.TrimSpace(strings.TrimPrefix(value, "\ufeff")))
}
func csvRecords(path string) ([][]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
prefix := make([]byte, 4096)
count, _ := file.Read(prefix)
if _, err := file.Seek(0, io.SeekStart); err != nil {
return nil, err
}
reader := csv.NewReader(file)
if strings.Count(string(prefix[:count]), "\t") > strings.Count(string(prefix[:count]), ",") {
reader.Comma = '\t'
}
reader.FieldsPerRecord = -1
records, err := reader.ReadAll()
if err != nil {
return nil, err
}
if len(records) < 2 {
return nil, fmt.Errorf("CSV %s has no data rows", filepath.Base(path))
}
return records, nil
}
func headerIndex(headers []string, names ...string) int {
wanted := map[string]struct{}{}
for _, name := range names {
wanted[normalizedCSVHeader(name)] = struct{}{}
}
for index, header := range headers {
if _, ok := wanted[normalizedCSVHeader(header)]; ok {
return index
}
}
return -1
}
func readHardwarePTS(path string) ([]hardwarePTSRow, error) {
records, err := csvRecords(path)
if err != nil {
return nil, err
}
index := headerIndex(records[0], "timestamp_us", "timestampus", "pts_us")
if index < 0 {
return nil, fmt.Errorf("PTS CSV %s is missing timestamp_us", filepath.Base(path))
}
rows := make([]hardwarePTSRow, 0, len(records)-1)
var previous int64
for rowIndex, record := range records[1:] {
if index >= len(record) {
return nil, fmt.Errorf("PTS CSV %s row %d is incomplete", filepath.Base(path), rowIndex+2)
}
value, parseErr := strconv.ParseInt(strings.TrimSpace(record[index]), 10, 64)
if parseErr != nil || value <= 0 || (len(rows) > 0 && value <= previous) {
return nil, fmt.Errorf("PTS CSV %s is invalid or not strictly increasing at row %d", filepath.Base(path), rowIndex+2)
}
rows = append(rows, hardwarePTSRow{TimestampUS: value})
previous = value
}
return rows, nil
}
func readHardwareIMU(path string) ([]hardwareIMURow, error) {
records, err := csvRecords(path)
if err != nil {
return nil, err
}
timestampIndex := headerIndex(records[0], "timestamp_us", "timestampus")
xIndex, yIndex, zIndex := headerIndex(records[0], "x"), headerIndex(records[0], "y"), headerIndex(records[0], "z")
typeIndex := headerIndex(records[0], "type")
if timestampIndex < 0 || xIndex < 0 || yIndex < 0 || zIndex < 0 || typeIndex < 0 {
return nil, fmt.Errorf("IMU CSV %s is missing timestamp_us/x/y/z/type", filepath.Base(path))
}
rows := make([]hardwareIMURow, 0, len(records)-1)
var previous int64
pairs := map[int64]map[string]bool{}
for rowIndex, record := range records[1:] {
maximum := max(timestampIndex, xIndex, yIndex, zIndex, typeIndex)
if maximum >= len(record) {
return nil, fmt.Errorf("IMU CSV %s row %d is incomplete", filepath.Base(path), rowIndex+2)
}
timestamp, parseErr := strconv.ParseInt(strings.TrimSpace(record[timestampIndex]), 10, 64)
kind := strings.ToLower(strings.TrimSpace(record[typeIndex]))
if parseErr != nil || timestamp <= 0 || (len(rows) > 0 && timestamp < previous) || (kind != "accel" && kind != "gyro") {
return nil, fmt.Errorf("IMU CSV %s has invalid ordering or type at row %d", filepath.Base(path), rowIndex+2)
}
rows = append(rows, hardwareIMURow{TimestampUS: timestamp, X: strings.TrimSpace(record[xIndex]), Y: strings.TrimSpace(record[yIndex]), Z: strings.TrimSpace(record[zIndex]), Type: kind})
if pairs[timestamp] == nil {
pairs[timestamp] = map[string]bool{}
}
pairs[timestamp][kind] = true
previous = timestamp
}
for timestamp, kinds := range pairs {
if !kinds["accel"] || !kinds["gyro"] {
return nil, fmt.Errorf("IMU timestamp %d is missing an accel/gyro pair", timestamp)
}
}
return rows, nil
}
func artifactCollectionID(artifact domain.ExecutionArtifact) string {
if artifact.Data == nil {
return ""
}
return strings.TrimSpace(fmt.Sprint(artifact.Data["collectionId"]))
}
func artifactsForTarget(execution domain.NodeExecutionRecord, targetPortID string) []domain.ExecutionArtifact {
for _, binding := range execution.InputBindings {
if binding.TargetPortID == targetPortID {
return append([]domain.ExecutionArtifact(nil), binding.Artifacts...)
}
}
return nil
}
func artifactByID(items []domain.ExecutionArtifact, id string) *domain.ExecutionArtifact {
for index := range items {
if items[index].ID == id {
copy := items[index]
return &copy
}
}
return nil
}
func matchingArtifact(items []domain.ExecutionArtifact, collectionID string, nameParts ...string) *domain.ExecutionArtifact {
for index := range items {
name := strings.ToLower(items[index].Name)
if collectionID != "" && artifactCollectionID(items[index]) != "" && artifactCollectionID(items[index]) != collectionID {
continue
}
matched := true
for _, part := range nameParts {
if !strings.Contains(name, strings.ToLower(part)) {
matched = false
}
}
if matched {
copy := items[index]
return &copy
}
}
return nil
}
func (server *server) safeMaterializationSourcePath(artifact *domain.ExecutionArtifact) (string, error) {
if artifact == nil {
return "", fmt.Errorf("materialization input artifact is missing")
}
root := filepath.Clean(server.workspaceMountPath)
if root == "." || root == "" {
return "", fmt.Errorf("materialization workspace is unavailable")
}
pathValue := strings.TrimSpace(artifact.Path)
candidates := make([]string, 0, 3)
if pathValue != "" && !strings.Contains(pathValue, "://") {
candidates = append(candidates, filepath.Clean(pathValue))
}
if strings.HasPrefix(strings.ToLower(pathValue), "bos://") {
if parsed, err := url.Parse(pathValue); err == nil {
key := strings.TrimPrefix(strings.TrimSpace(parsed.Path), "/")
if key != "" {
candidates = append(candidates, filepath.Join(root, filepath.FromSlash(key)))
}
}
}
if artifact.Payload != nil && artifact.Payload.Ref != nil {
key := strings.TrimPrefix(strings.TrimSpace(artifact.Payload.Ref.Key), "/")
if key == "" {
key = strings.TrimPrefix(strings.TrimSpace(artifact.Payload.Ref.Prefix), "/")
}
if key != "" {
candidates = append(candidates, filepath.Join(root, filepath.FromSlash(key)))
}
}
for _, candidate := range candidates {
candidate = filepath.Clean(candidate)
if candidate != root && !strings.HasPrefix(candidate, root+string(os.PathSeparator)) {
continue
}
info, err := os.Stat(candidate)
if err == nil && info.Mode().IsRegular() && info.Size() > 0 {
return candidate, nil
}
}
return "", fmt.Errorf("materialization input %s is unavailable on the authorized workspace mount", artifact.Name)
}
func ffprobeFrameCount(ctx context.Context, path string) (int, error) {
output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-select_streams", "v:0", "-count_frames", "-show_entries", "stream=nb_read_frames", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput()
if err != nil {
return 0, fmt.Errorf("ffprobe %s: %s", filepath.Base(path), strings.TrimSpace(string(output)))
}
count, err := strconv.Atoi(strings.TrimSpace(string(output)))
if err != nil || count <= 0 {
return 0, fmt.Errorf("ffprobe returned invalid frame count for %s", filepath.Base(path))
}
return count, nil
}
func ffprobeDurationSeconds(ctx context.Context, path string) (float64, error) {
output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput()
if err != nil {
return 0, fmt.Errorf("ffprobe duration %s: %s", filepath.Base(path), strings.TrimSpace(string(output)))
}
duration, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64)
if err != nil || math.IsNaN(duration) || math.IsInf(duration, 0) || duration <= 0 {
return 0, fmt.Errorf("ffprobe returned invalid duration for %s", filepath.Base(path))
}
return duration, nil
}
func writeFrameAccurateSlice(ctx context.Context, source string, destination string, startFrame int, endFrame int) error {
if startFrame < 0 || endFrame <= startFrame {
return fmt.Errorf("frame slice is empty")
}
filter := fmt.Sprintf("trim=start_frame=%d:end_frame=%d,setpts=PTS-STARTPTS", startFrame, endFrame)
output, err := exec.CommandContext(ctx, "ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-i", source, "-an", "-vf", filter, "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-pix_fmt", "yuv420p", destination).CombinedOutput()
if err != nil {
return fmt.Errorf("ffmpeg slice %s: %s", filepath.Base(source), strings.TrimSpace(string(output)))
}
return nil
}
func sha256File(path string) (string, int64, error) {
file, err := os.Open(path)
if err != nil {
return "", 0, err
}
defer file.Close()
hash := sha256.New()
size, err := io.Copy(hash, file)
if err != nil {
return "", 0, err
}
return hex.EncodeToString(hash.Sum(nil)), size, nil
}
func writePTSSlice(path string, rows []hardwarePTSRow, start int, end int, windowStartUS int64) (map[string]any, error) {
if start < 0 || end > len(rows) || end <= start {
return nil, fmt.Errorf("PTS slice is empty")
}
file, err := os.Create(path)
if err != nil {
return nil, err
}
writer := csv.NewWriter(file)
_ = writer.Write([]string{"frame_index", "timestamp_ms", "clip_time_ms"})
for index, row := range rows[start:end] {
_ = writer.Write([]string{strconv.Itoa(index), fmt.Sprintf("%.3f", float64(row.TimestampUS)/1000.0), fmt.Sprintf("%.3f", float64(row.TimestampUS-windowStartUS)/1000.0)})
}
writer.Flush()
closeErr := file.Close()
if writer.Error() != nil {
return nil, writer.Error()
}
if closeErr != nil {
return nil, closeErr
}
sha, size, err := sha256File(path)
return map[string]any{"name": filepath.Base(path), "sha256": sha, "bytes": size, "rowCount": end - start, "firstTimestampUs": rows[start].TimestampUS, "lastTimestampUs": rows[end-1].TimestampUS}, err
}
func writeIMUSlice(path string, rows []hardwareIMURow, startUS int64, endUS int64) (map[string]any, error) {
selected := make([]hardwareIMURow, 0)
for _, row := range rows {
if row.TimestampUS >= startUS && row.TimestampUS < endUS {
selected = append(selected, row)
}
}
if len(selected) == 0 {
return nil, fmt.Errorf("IMU slice is empty")
}
file, err := os.Create(path)
if err != nil {
return nil, err
}
writer := csv.NewWriter(file)
_ = writer.Write([]string{"timestamp_ms", "clip_time_ms", "x", "y", "z", "type"})
for _, row := range selected {
_ = writer.Write([]string{fmt.Sprintf("%.3f", float64(row.TimestampUS)/1000.0), fmt.Sprintf("%.3f", float64(row.TimestampUS-startUS)/1000.0), row.X, row.Y, row.Z, row.Type})
}
writer.Flush()
closeErr := file.Close()
if writer.Error() != nil {
return nil, writer.Error()
}
if closeErr != nil {
return nil, closeErr
}
sha, size, err := sha256File(path)
return map[string]any{"name": filepath.Base(path), "sha256": sha, "bytes": size, "sampleCount": len(selected), "firstTimestampUs": selected[0].TimestampUS, "lastTimestampUs": selected[len(selected)-1].TimestampUS}, err
}
func writeJSONFile(path string, payload any) (string, int64, error) {
encoded, err := json.MarshalIndent(payload, "", " ")
if err != nil {
return "", 0, err
}
encoded = append(encoded, '\n')
if err := os.WriteFile(path, encoded, 0o644); err != nil {
return "", 0, err
}
digest := sha256.Sum256(encoded)
return hex.EncodeToString(digest[:]), int64(len(encoded)), nil
}
func artifactForMaterializationPath(path string, id string, artifactType domain.PortType) map[string]any {
sha, size, _ := sha256File(path)
return map[string]any{"id": id, "name": filepath.Base(path), "sha256": sha, "bytes": size, "type": string(artifactType)}
}
func ptsCoverageEnd(rows []hardwarePTSRow) (int64, error) {
if len(rows) < 2 {
return 0, fmt.Errorf("PTS requires at least two rows to resolve the half-open coverage end")
}
interval := rows[len(rows)-1].TimestampUS - rows[len(rows)-2].TimestampUS
if interval <= 0 {
return 0, fmt.Errorf("PTS terminal interval is invalid")
}
return rows[len(rows)-1].TimestampUS + interval, nil
}
func imuCoverageEnd(rows []hardwareIMURow) (int64, error) {
if len(rows) < 4 {
return 0, fmt.Errorf("IMU requires at least two accel/gyro timestamp pairs")
}
last := rows[len(rows)-1].TimestampUS
previous := int64(0)
for index := len(rows) - 2; index >= 0; index-- {
if rows[index].TimestampUS < last {
previous = rows[index].TimestampUS
break
}
}
if previous <= 0 || last <= previous {
return 0, fmt.Errorf("IMU terminal interval is invalid")
}
return last + (last - previous), nil
}
func maximumNearestAlignmentUS(left []hardwarePTSRow, leftStart int, leftEnd int, right []hardwarePTSRow, rightStart int, rightEnd int) int64 {
maximum := int64(0)
for _, row := range left[leftStart:leftEnd] {
index := sort.Search(rightEnd-rightStart, func(index int) bool {
return right[rightStart+index].TimestampUS >= row.TimestampUS
})
nearest := int64(math.MaxInt64)
if index < rightEnd-rightStart {
nearest = absInt64(right[rightStart+index].TimestampUS - row.TimestampUS)
}
if index > 0 {
nearest = min(nearest, absInt64(right[rightStart+index-1].TimestampUS-row.TimestampUS))
}
maximum = max(maximum, nearest)
}
return maximum
}
func absInt64(value int64) int64 {
if value < 0 {
return -value
}
return value
}
func (server *server) materializeCombinedSliceSubmission(ctx context.Context, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, payload combinedAnnotationSubmissionInput, revision int) (*humanSliceMaterialization, error) {
run, err := server.store.GetRun(ctx, task.RunID)
if err != nil || run == nil {
return nil, fmt.Errorf("load materialization run: %w", err)
}
artifactID := fmt.Sprintf("%s-%s-materialized-revision-%06d", task.RunID, task.NodeID, revision)
for _, existing := range run.Artifacts {
if existing.ID == artifactID {
manifestSHA := ""
if existing.Metadata != nil {
manifestSHA = existing.Metadata.SHA256
}
coverage := mapValue(existing.Data["coverage"])
slices := decodeMaterializationValue[domain.SliceSegment](existing.Data["normalizedSlices"])
segments := decodeMaterializationValue[domain.AnnotationSegment](existing.Data["normalizedSegments"])
sliceCount := intValue(existing.Data["sliceCount"])
if sliceCount == 0 {
sliceCount = intValue(coverage["sliceCount"])
}
if slices == nil {
slices = append([]domain.SliceSegment(nil), payload.SliceSegments...)
}
if segments == nil {
segments = append([]domain.AnnotationSegment(nil), payload.Segments...)
}
return &humanSliceMaterialization{Artifact: existing, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": sliceCount, "coverage": coverage, "runId": task.RunID, "artifactId": existing.ID, "manifestSha256": manifestSHA}, Slices: slices, Segments: segments, CoverageStartSecond: floatValue(coverage["startSecond"]), CoverageEndSecond: floatValue(coverage["endSecond"])}, nil
}
}
var execution *domain.NodeExecutionRecord
for index := range run.NodeExecutions {
if run.NodeExecutions[index].NodeID == task.NodeID {
copy := run.NodeExecutions[index]
execution = &copy
break
}
}
if execution == nil {
return nil, fmt.Errorf("human gate execution is unavailable")
}
leftArtifact := artifactByID(artifactsForTarget(*execution, "fullLeftVideos"), task.ArtifactID)
if leftArtifact == nil {
leftArtifact = artifactByID(execution.InputArtifacts, task.ArtifactID)
}
if leftArtifact == nil {
return nil, fmt.Errorf("left source video is unavailable")
}
collectionID := artifactCollectionID(*leftArtifact)
rightArtifact := matchingArtifact(artifactsForTarget(*execution, "fullRightVideos"), collectionID, "right", ".mp4")
leftPTSArtifact := matchingArtifact(artifactsForTarget(*execution, "leftTimestamps"), collectionID, "left", "pts")
rightPTSArtifact := matchingArtifact(artifactsForTarget(*execution, "rightTimestamps"), collectionID, "right", "pts")
imuArtifact := matchingArtifact(artifactsForTarget(*execution, "imu"), collectionID, "imu", ".csv")
leftPath, err := server.safeMaterializationSourcePath(leftArtifact)
if err != nil {
return nil, err
}
rightPath, err := server.safeMaterializationSourcePath(rightArtifact)
if err != nil {
return nil, err
}
leftPTSPath, err := server.safeMaterializationSourcePath(leftPTSArtifact)
if err != nil {
return nil, err
}
rightPTSPath, err := server.safeMaterializationSourcePath(rightPTSArtifact)
if err != nil {
return nil, err
}
imuPath, err := server.safeMaterializationSourcePath(imuArtifact)
if err != nil {
return nil, err
}
leftPTS, err := readHardwarePTS(leftPTSPath)
if err != nil {
return nil, err
}
rightPTS, err := readHardwarePTS(rightPTSPath)
if err != nil {
return nil, err
}
imuRows, err := readHardwareIMU(imuPath)
if err != nil {
return nil, err
}
leftFrames, err := ffprobeFrameCount(ctx, leftPath)
if err != nil || leftFrames != len(leftPTS) {
return nil, fmt.Errorf("left video/PTS frame mismatch: video=%d pts=%d err=%v", leftFrames, len(leftPTS), err)
}
rightFrames, err := ffprobeFrameCount(ctx, rightPath)
if err != nil || rightFrames != len(rightPTS) {
return nil, fmt.Errorf("right video/PTS frame mismatch: video=%d pts=%d err=%v", rightFrames, len(rightPTS), err)
}
leftMediaDuration, err := ffprobeDurationSeconds(ctx, leftPath)
if err != nil {
return nil, err
}
leftCoverageEnd, err := ptsCoverageEnd(leftPTS)
if err != nil {
return nil, err
}
rightCoverageEnd, err := ptsCoverageEnd(rightPTS)
if err != nil {
return nil, err
}
imuEnd, err := imuCoverageEnd(imuRows)
if err != nil {
return nil, err
}
commonStart := max(leftPTS[0].TimestampUS, rightPTS[0].TimestampUS, imuRows[0].TimestampUS)
commonEnd := min(leftCoverageEnd, rightCoverageEnd, imuEnd)
if commonEnd <= commonStart {
return nil, fmt.Errorf("required streams have no common hardware-time range")
}
leftTimelineOrigin := leftPTS[0].TimestampUS
leftPTSDurationSeconds := float64(leftCoverageEnd-leftTimelineOrigin) / 1_000_000
ptsToMediaScale := leftMediaDuration / leftPTSDurationSeconds
if math.IsNaN(ptsToMediaScale) || math.IsInf(ptsToMediaScale, 0) || ptsToMediaScale < 0.5 || ptsToMediaScale > 2 {
return nil, fmt.Errorf("left video and PTS timelines have an implausible scale mismatch")
}
timestampToMediaSecond := func(timestampUS int64) float64 {
return float64(timestampUS-leftTimelineOrigin) / 1_000_000 * ptsToMediaScale
}
mediaSecondToTimestamp := func(second float64) int64 {
return leftTimelineOrigin + int64(math.Round(second/ptsToMediaScale*1_000_000))
}
commonMediaStart := timestampToMediaSecond(commonStart)
commonMediaEnd := timestampToMediaSecond(commonEnd)
if commonMediaEnd <= commonMediaStart {
return nil, fmt.Errorf("common hardware-time range cannot be mapped to the media timeline")
}
root := filepath.Join(filepath.Clean(server.workspaceMountPath), "system", "runs", task.RunID, "human-materializations", item.ID, fmt.Sprintf("revision-%06d", revision))
if err := os.MkdirAll(filepath.Join(root, "slices"), 0o755); err != nil {
return nil, err
}
slices := append([]domain.SliceSegment(nil), payload.SliceSegments...)
sort.Slice(slices, func(left, right int) bool { return slices[left].Start < slices[right].Start })
if len(slices) == 0 {
return nil, fmt.Errorf("full synchronized coverage requires at least one slice")
}
requestedStartUS := mediaSecondToTimestamp(slices[0].Start)
requestedEndUS := mediaSecondToTimestamp(slices[len(slices)-1].End)
boundaryToleranceUS := int64(math.Ceil(sliceCoverageToleranceSeconds / ptsToMediaScale * 1_000_000))
if requestedStartUS > commonStart+boundaryToleranceUS || requestedEndUS < commonEnd-boundaryToleranceUS {
return nil, fmt.Errorf("submitted slices do not cover the full common hardware-time range")
}
clippedSlices := make([]domain.SliceSegment, 0, len(slices))
for _, slice := range slices {
slice.Start = max(slice.Start, commonMediaStart)
slice.End = min(slice.End, commonMediaEnd)
if slice.End > slice.Start {
clippedSlices = append(clippedSlices, slice)
}
}
if len(clippedSlices) == 0 {
return nil, fmt.Errorf("submitted slices do not intersect the common hardware-time range")
}
normalizedSlices := make([]domain.SliceSegment, 0, len(slices))
previousEndUS := commonStart
for index, slice := range clippedSlices {
startUS := mediaSecondToTimestamp(slice.Start)
endUS := mediaSecondToTimestamp(slice.End)
if index == 0 {
startUS = commonStart
} else if absInt64(startUS-previousEndUS) <= boundaryToleranceUS {
startUS = previousEndUS
}
if index == len(clippedSlices)-1 {
endUS = commonEnd
}
if startUS != previousEndUS || endUS <= startUS || endUS > commonEnd {
return nil, fmt.Errorf("slice %s leaves a gap or overlap in the common hardware-time range", slice.ID)
}
normalized := slice
normalized.Start = timestampToMediaSecond(startUS)
normalized.End = timestampToMediaSecond(endUS)
normalizedSlices = append(normalizedSlices, normalized)
previousEndUS = endUS
}
if previousEndUS != commonEnd {
return nil, fmt.Errorf("submitted slices do not reach the common hardware-time end")
}
normalizedSegments := make([]domain.AnnotationSegment, 0, len(payload.Segments))
for _, segment := range payload.Segments {
segment.Start = max(segment.Start, commonMediaStart)
segment.End = min(segment.End, commonMediaEnd)
if segment.End > segment.Start {
normalizedSegments = append(normalizedSegments, segment)
}
}
leftCommonStart := sort.Search(len(leftPTS), func(index int) bool { return leftPTS[index].TimestampUS >= commonStart })
leftCommonEnd := sort.Search(len(leftPTS), func(index int) bool { return leftPTS[index].TimestampUS >= commonEnd })
rightCommonStart := sort.Search(len(rightPTS), func(index int) bool { return rightPTS[index].TimestampUS >= commonStart })
rightCommonEnd := sort.Search(len(rightPTS), func(index int) bool { return rightPTS[index].TimestampUS >= commonEnd })
imuCommonCount := 0
for _, row := range imuRows {
if row.TimestampUS >= commonStart && row.TimestampUS < commonEnd {
imuCommonCount++
}
}
deliveredLeftFrames, deliveredRightFrames, deliveredIMURows := 0, 0, 0
manifestItems := make([]map[string]any, 0, len(slices))
for index, slice := range normalizedSlices {
startUS := mediaSecondToTimestamp(slice.Start)
endUS := mediaSecondToTimestamp(slice.End)
if index == 0 {
startUS = commonStart
}
if index == len(normalizedSlices)-1 {
endUS = commonEnd
}
leftStart := sort.Search(len(leftPTS), func(i int) bool { return leftPTS[i].TimestampUS >= startUS })
leftEnd := sort.Search(len(leftPTS), func(i int) bool { return leftPTS[i].TimestampUS >= endUS })
rightStart := sort.Search(len(rightPTS), func(i int) bool { return rightPTS[i].TimestampUS >= startUS })
rightEnd := sort.Search(len(rightPTS), func(i int) bool { return rightPTS[i].TimestampUS >= endUS })
sliceID := fmt.Sprintf("slice-%06d", index+1)
directory := filepath.Join(root, "slices", sliceID)
if err := os.MkdirAll(directory, 0o755); err != nil {
return nil, err
}
leftOutput, rightOutput := filepath.Join(directory, "left.mp4"), filepath.Join(directory, "right.mp4")
if err := writeFrameAccurateSlice(ctx, leftPath, leftOutput, leftStart, leftEnd); err != nil {
return nil, err
}
if err := writeFrameAccurateSlice(ctx, rightPath, rightOutput, rightStart, rightEnd); err != nil {
return nil, err
}
leftCount, leftErr := ffprobeFrameCount(ctx, leftOutput)
rightCount, rightErr := ffprobeFrameCount(ctx, rightOutput)
if leftErr != nil || rightErr != nil || leftCount != leftEnd-leftStart || rightCount != rightEnd-rightStart {
return nil, fmt.Errorf("materialized video frame count does not match sliced PTS rows")
}
deliveredLeftFrames += leftCount
deliveredRightFrames += rightCount
leftManifest, err := writePTSSlice(filepath.Join(directory, "left_pts.csv"), leftPTS, leftStart, leftEnd, startUS)
if err != nil {
return nil, err
}
rightManifest, err := writePTSSlice(filepath.Join(directory, "right_pts.csv"), rightPTS, rightStart, rightEnd, startUS)
if err != nil {
return nil, err
}
imuManifest, err := writeIMUSlice(filepath.Join(directory, "imu.csv"), imuRows, startUS, endUS)
if err != nil {
return nil, err
}
deliveredIMURows += int(imuManifest["sampleCount"].(int))
actions := make([]map[string]any, 0)
for _, segment := range normalizedSegments {
segmentStart := math.Max(segment.Start, slice.Start)
segmentEnd := math.Min(segment.End, slice.End)
if segmentEnd <= segmentStart {
continue
}
relativeStart := segmentStart - slice.Start
relativeEnd := segmentEnd - slice.Start
actions = append(actions, map[string]any{"id": segment.ID, "labelId": segment.LabelID, "label": segment.Label, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": relativeStart * 1000, "clipEndMs": relativeEnd * 1000, "startTimestampUs": mediaSecondToTimestamp(segmentStart), "endTimestampUs": mediaSecondToTimestamp(segmentEnd)})
}
annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-annotation/v1", "sliceId": sliceID, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "actions": actions}
annotationSHA, annotationBytes, err := writeJSONFile(filepath.Join(directory, "annotation.json"), annotation)
if err != nil {
return nil, err
}
leftVideo := artifactForMaterializationPath(leftOutput, "leftVideo", domain.PortTypeVideo)
rightVideo := artifactForMaterializationPath(rightOutput, "rightVideo", domain.PortTypeVideo)
maximumAlignmentUS := maximumNearestAlignmentUS(leftPTS, leftStart, leftEnd, rightPTS, rightStart, rightEnd)
synchronization := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "collectionId": collectionID, "sliceIndex": index + 1, "sliceCount": len(normalizedSlices), "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "commonRange": map[string]any{"startTimestampUs": commonStart, "endTimestampUs": commonEnd, "startSecond": commonMediaStart, "endSecond": commonMediaEnd}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true, "previousEndTimestampUs": func() any {
if index == 0 {
return nil
}
return manifestItems[index-1]["endTimestampUs"]
}(), "nextStartTimestampUs": func() any {
if index == len(normalizedSlices)-1 {
return nil
}
return endUS
}()}, "maxStereoAlignmentErrorUs": maximumAlignmentUS, "left": map[string]any{"video": leftVideo, "pts": leftManifest}, "right": map[string]any{"video": rightVideo, "pts": rightManifest}, "imu": imuManifest, "lineage": map[string]any{"leftVideoArtifactId": leftArtifact.ID, "rightVideoArtifactId": rightArtifact.ID, "leftPtsArtifactId": leftPTSArtifact.ID, "rightPtsArtifactId": rightPTSArtifact.ID, "imuArtifactId": imuArtifact.ID}}
syncSHA, syncBytes, err := writeJSONFile(filepath.Join(directory, "synchronized-slice.json"), synchronization)
if err != nil {
return nil, err
}
manifestItems = append(manifestItems, map[string]any{"sliceId": sliceID, "sourceSliceId": slice.ID, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": startUS, "endTimestampUs": endUS, "directory": filepath.ToSlash(filepath.Join("slices", sliceID)), "files": map[string]any{"leftVideo": leftVideo, "rightVideo": rightVideo, "leftPts": leftManifest, "rightPts": rightManifest, "imu": imuManifest, "annotation": map[string]any{"name": "annotation.json", "sha256": annotationSHA, "bytes": annotationBytes}, "synchronization": map[string]any{"name": "synchronized-slice.json", "sha256": syncSHA, "bytes": syncBytes}}})
}
expectedLeftFrames := leftCommonEnd - leftCommonStart
expectedRightFrames := rightCommonEnd - rightCommonStart
if deliveredLeftFrames != expectedLeftFrames || deliveredRightFrames != expectedRightFrames || deliveredIMURows != imuCommonCount {
return nil, fmt.Errorf("slice reconstruction coverage mismatch: left=%d/%d right=%d/%d imu=%d/%d", deliveredLeftFrames, expectedLeftFrames, deliveredRightFrames, expectedRightFrames, deliveredIMURows, imuCommonCount)
}
coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": commonMediaStart, "endSecond": commonMediaEnd, "durationSeconds": commonMediaEnd - commonMediaStart, "startTimestampUs": commonStart, "endTimestampUs": commonEnd, "ptsToMediaScale": ptsToMediaScale, "sliceCount": len(manifestItems), "reconstruction": map[string]any{"leftFrameCount": deliveredLeftFrames, "rightFrameCount": deliveredRightFrames, "imuRowCount": deliveredIMURows}}
captureEvidence := map[string]any{}
if capture, captureErr := server.store.GetCaptureRecordByRunID(ctx, task.RunID); captureErr != nil {
return nil, captureErr
} else if capture != nil {
captureEvidence = map[string]any{
"captureRecordId": capture.ID, "captureId": capture.CaptureID, "prefix": capture.Prefix,
"fingerprint": capture.Fingerprint, "contractRevision": capture.ContractRevision,
}
}
revisionManifest := map[string]any{"schemaVersion": "droboticflow.slice-materialization/v2", "revision": revision, "humanItemId": item.ID, "taskId": task.ID, "runId": task.RunID, "collectionId": collectionID, "capture": captureEvidence, "status": "materialized", "sliceCount": len(manifestItems), "coverage": coverage, "items": manifestItems}
manifestPath := filepath.Join(root, "revision-manifest.json")
manifestSHA, manifestBytes, err := writeJSONFile(manifestPath, revisionManifest)
if err != nil {
return nil, err
}
artifact := domain.ExecutionArtifact{ID: artifactID, RunID: task.RunID, NodeID: task.NodeID, PortID: "materializedSliceDirectory", PortName: "materializedSliceDirectory", Name: fmt.Sprintf("revision-%06d.directory.index", revision), Type: domain.PortTypeFile, Size: fmt.Sprintf("%d files", len(manifestItems)*7+1), SizeBytes: manifestBytes, Path: root, Metadata: &domain.ArtifactMetadata{SHA256: manifestSHA, MimeType: "application/json"}, Data: map[string]any{"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": root, "manifestPath": manifestPath, "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "normalizedSlices": normalizedSlices, "normalizedSegments": normalizedSegments}}
run.Artifacts = append(run.Artifacts, artifact)
for index := range run.NodeExecutions {
if run.NodeExecutions[index].NodeID == task.NodeID {
run.NodeExecutions[index].Artifacts = append(run.NodeExecutions[index].Artifacts, artifact)
}
}
run.UpdatedAt = nowISO()
if err := server.store.UpsertRun(ctx, *run); err != nil {
return nil, err
}
return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "runId": task.RunID, "artifactId": artifact.ID, "manifestSha256": manifestSHA}, Slices: normalizedSlices, Segments: normalizedSegments, CoverageStartSecond: commonMediaStart, CoverageEndSecond: commonMediaEnd}, nil
}