forked from longtao.wu/droboticflow
758 lines
32 KiB
Go
758 lines
32 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/csv"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
type hardwarePTSRow struct {
|
|
TimestampUS int64
|
|
}
|
|
|
|
type hardwareIMURow struct {
|
|
TimestampUS int64
|
|
X string
|
|
Y string
|
|
Z string
|
|
Type string
|
|
}
|
|
|
|
type humanSliceMaterialization struct {
|
|
Artifact domain.ExecutionArtifact
|
|
Payload map[string]any
|
|
Slices []domain.SliceSegment
|
|
Segments []domain.AnnotationSegment
|
|
CoverageStartSecond float64
|
|
CoverageEndSecond float64
|
|
}
|
|
|
|
func decodeMaterializationValue[T any](value any) []T {
|
|
encoded, err := json.Marshal(normalizeStructuredValue(value))
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var items []T
|
|
if err := json.Unmarshal(encoded, &items); err != nil {
|
|
return nil
|
|
}
|
|
return items
|
|
}
|
|
|
|
func normalizedCSVHeader(value string) string {
|
|
return strings.ToLower(strings.TrimSpace(strings.TrimPrefix(value, "\ufeff")))
|
|
}
|
|
|
|
func csvRecords(path string) ([][]string, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer file.Close()
|
|
prefix := make([]byte, 4096)
|
|
count, _ := file.Read(prefix)
|
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
return nil, err
|
|
}
|
|
reader := csv.NewReader(file)
|
|
if strings.Count(string(prefix[:count]), "\t") > strings.Count(string(prefix[:count]), ",") {
|
|
reader.Comma = '\t'
|
|
}
|
|
reader.FieldsPerRecord = -1
|
|
records, err := reader.ReadAll()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(records) < 2 {
|
|
return nil, fmt.Errorf("CSV %s has no data rows", filepath.Base(path))
|
|
}
|
|
return records, nil
|
|
}
|
|
|
|
func headerIndex(headers []string, names ...string) int {
|
|
wanted := map[string]struct{}{}
|
|
for _, name := range names {
|
|
wanted[normalizedCSVHeader(name)] = struct{}{}
|
|
}
|
|
for index, header := range headers {
|
|
if _, ok := wanted[normalizedCSVHeader(header)]; ok {
|
|
return index
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func readHardwarePTS(path string) ([]hardwarePTSRow, error) {
|
|
records, err := csvRecords(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
index := headerIndex(records[0], "timestamp_us", "timestampus", "pts_us")
|
|
if index < 0 {
|
|
return nil, fmt.Errorf("PTS CSV %s is missing timestamp_us", filepath.Base(path))
|
|
}
|
|
rows := make([]hardwarePTSRow, 0, len(records)-1)
|
|
var previous int64
|
|
for rowIndex, record := range records[1:] {
|
|
if index >= len(record) {
|
|
return nil, fmt.Errorf("PTS CSV %s row %d is incomplete", filepath.Base(path), rowIndex+2)
|
|
}
|
|
value, parseErr := strconv.ParseInt(strings.TrimSpace(record[index]), 10, 64)
|
|
if parseErr != nil || value <= 0 || (len(rows) > 0 && value <= previous) {
|
|
return nil, fmt.Errorf("PTS CSV %s is invalid or not strictly increasing at row %d", filepath.Base(path), rowIndex+2)
|
|
}
|
|
rows = append(rows, hardwarePTSRow{TimestampUS: value})
|
|
previous = value
|
|
}
|
|
return rows, nil
|
|
}
|
|
|
|
func readHardwareIMU(path string) ([]hardwareIMURow, error) {
|
|
records, err := csvRecords(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
timestampIndex := headerIndex(records[0], "timestamp_us", "timestampus")
|
|
xIndex, yIndex, zIndex := headerIndex(records[0], "x"), headerIndex(records[0], "y"), headerIndex(records[0], "z")
|
|
typeIndex := headerIndex(records[0], "type")
|
|
if timestampIndex < 0 || xIndex < 0 || yIndex < 0 || zIndex < 0 || typeIndex < 0 {
|
|
return nil, fmt.Errorf("IMU CSV %s is missing timestamp_us/x/y/z/type", filepath.Base(path))
|
|
}
|
|
rows := make([]hardwareIMURow, 0, len(records)-1)
|
|
var previous int64
|
|
pairs := map[int64]map[string]bool{}
|
|
for rowIndex, record := range records[1:] {
|
|
maximum := max(timestampIndex, xIndex, yIndex, zIndex, typeIndex)
|
|
if maximum >= len(record) {
|
|
return nil, fmt.Errorf("IMU CSV %s row %d is incomplete", filepath.Base(path), rowIndex+2)
|
|
}
|
|
timestamp, parseErr := strconv.ParseInt(strings.TrimSpace(record[timestampIndex]), 10, 64)
|
|
kind := strings.ToLower(strings.TrimSpace(record[typeIndex]))
|
|
if parseErr != nil || timestamp <= 0 || (len(rows) > 0 && timestamp < previous) || (kind != "accel" && kind != "gyro") {
|
|
return nil, fmt.Errorf("IMU CSV %s has invalid ordering or type at row %d", filepath.Base(path), rowIndex+2)
|
|
}
|
|
rows = append(rows, hardwareIMURow{TimestampUS: timestamp, X: strings.TrimSpace(record[xIndex]), Y: strings.TrimSpace(record[yIndex]), Z: strings.TrimSpace(record[zIndex]), Type: kind})
|
|
if pairs[timestamp] == nil {
|
|
pairs[timestamp] = map[string]bool{}
|
|
}
|
|
pairs[timestamp][kind] = true
|
|
previous = timestamp
|
|
}
|
|
for timestamp, kinds := range pairs {
|
|
if !kinds["accel"] || !kinds["gyro"] {
|
|
return nil, fmt.Errorf("IMU timestamp %d is missing an accel/gyro pair", timestamp)
|
|
}
|
|
}
|
|
return rows, nil
|
|
}
|
|
|
|
func artifactCollectionID(artifact domain.ExecutionArtifact) string {
|
|
if artifact.Data == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(fmt.Sprint(artifact.Data["collectionId"]))
|
|
}
|
|
|
|
func artifactsForTarget(execution domain.NodeExecutionRecord, targetPortID string) []domain.ExecutionArtifact {
|
|
for _, binding := range execution.InputBindings {
|
|
if binding.TargetPortID == targetPortID {
|
|
return append([]domain.ExecutionArtifact(nil), binding.Artifacts...)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func artifactByID(items []domain.ExecutionArtifact, id string) *domain.ExecutionArtifact {
|
|
for index := range items {
|
|
if items[index].ID == id {
|
|
copy := items[index]
|
|
return ©
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func matchingArtifact(items []domain.ExecutionArtifact, collectionID string, nameParts ...string) *domain.ExecutionArtifact {
|
|
for index := range items {
|
|
name := strings.ToLower(items[index].Name)
|
|
if collectionID != "" && artifactCollectionID(items[index]) != "" && artifactCollectionID(items[index]) != collectionID {
|
|
continue
|
|
}
|
|
matched := true
|
|
for _, part := range nameParts {
|
|
if !strings.Contains(name, strings.ToLower(part)) {
|
|
matched = false
|
|
}
|
|
}
|
|
if matched {
|
|
copy := items[index]
|
|
return ©
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (server *server) safeMaterializationSourcePath(artifact *domain.ExecutionArtifact) (string, error) {
|
|
if artifact == nil {
|
|
return "", fmt.Errorf("materialization input artifact is missing")
|
|
}
|
|
root := filepath.Clean(server.workspaceMountPath)
|
|
if root == "." || root == "" {
|
|
return "", fmt.Errorf("materialization workspace is unavailable")
|
|
}
|
|
pathValue := strings.TrimSpace(artifact.Path)
|
|
candidates := make([]string, 0, 3)
|
|
if pathValue != "" && !strings.Contains(pathValue, "://") {
|
|
candidates = append(candidates, filepath.Clean(pathValue))
|
|
}
|
|
if strings.HasPrefix(strings.ToLower(pathValue), "bos://") {
|
|
if parsed, err := url.Parse(pathValue); err == nil {
|
|
key := strings.TrimPrefix(strings.TrimSpace(parsed.Path), "/")
|
|
if key != "" {
|
|
candidates = append(candidates, filepath.Join(root, filepath.FromSlash(key)))
|
|
}
|
|
}
|
|
}
|
|
if artifact.Payload != nil && artifact.Payload.Ref != nil {
|
|
key := strings.TrimPrefix(strings.TrimSpace(artifact.Payload.Ref.Key), "/")
|
|
if key == "" {
|
|
key = strings.TrimPrefix(strings.TrimSpace(artifact.Payload.Ref.Prefix), "/")
|
|
}
|
|
if key != "" {
|
|
candidates = append(candidates, filepath.Join(root, filepath.FromSlash(key)))
|
|
}
|
|
}
|
|
for _, candidate := range candidates {
|
|
candidate = filepath.Clean(candidate)
|
|
if candidate != root && !strings.HasPrefix(candidate, root+string(os.PathSeparator)) {
|
|
continue
|
|
}
|
|
info, err := os.Stat(candidate)
|
|
if err == nil && info.Mode().IsRegular() && info.Size() > 0 {
|
|
return candidate, nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("materialization input %s is unavailable on the authorized workspace mount", artifact.Name)
|
|
}
|
|
|
|
func ffprobeFrameCount(ctx context.Context, path string) (int, error) {
|
|
output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-select_streams", "v:0", "-count_frames", "-show_entries", "stream=nb_read_frames", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput()
|
|
if err != nil {
|
|
return 0, fmt.Errorf("ffprobe %s: %s", filepath.Base(path), strings.TrimSpace(string(output)))
|
|
}
|
|
count, err := strconv.Atoi(strings.TrimSpace(string(output)))
|
|
if err != nil || count <= 0 {
|
|
return 0, fmt.Errorf("ffprobe returned invalid frame count for %s", filepath.Base(path))
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
func ffprobeDurationSeconds(ctx context.Context, path string) (float64, error) {
|
|
output, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=nokey=1:noprint_wrappers=1", path).CombinedOutput()
|
|
if err != nil {
|
|
return 0, fmt.Errorf("ffprobe duration %s: %s", filepath.Base(path), strings.TrimSpace(string(output)))
|
|
}
|
|
duration, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64)
|
|
if err != nil || math.IsNaN(duration) || math.IsInf(duration, 0) || duration <= 0 {
|
|
return 0, fmt.Errorf("ffprobe returned invalid duration for %s", filepath.Base(path))
|
|
}
|
|
return duration, nil
|
|
}
|
|
|
|
func writeFrameAccurateSlice(ctx context.Context, source string, destination string, startFrame int, endFrame int) error {
|
|
if startFrame < 0 || endFrame <= startFrame {
|
|
return fmt.Errorf("frame slice is empty")
|
|
}
|
|
filter := fmt.Sprintf("trim=start_frame=%d:end_frame=%d,setpts=PTS-STARTPTS", startFrame, endFrame)
|
|
output, err := exec.CommandContext(ctx, "ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-i", source, "-an", "-vf", filter, "-c:v", "libx264", "-preset", "ultrafast", "-crf", "23", "-pix_fmt", "yuv420p", destination).CombinedOutput()
|
|
if err != nil {
|
|
return fmt.Errorf("ffmpeg slice %s: %s", filepath.Base(source), strings.TrimSpace(string(output)))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sha256File(path string) (string, int64, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
defer file.Close()
|
|
hash := sha256.New()
|
|
size, err := io.Copy(hash, file)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
return hex.EncodeToString(hash.Sum(nil)), size, nil
|
|
}
|
|
|
|
func writePTSSlice(path string, rows []hardwarePTSRow, start int, end int, windowStartUS int64) (map[string]any, error) {
|
|
if start < 0 || end > len(rows) || end <= start {
|
|
return nil, fmt.Errorf("PTS slice is empty")
|
|
}
|
|
file, err := os.Create(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
writer := csv.NewWriter(file)
|
|
_ = writer.Write([]string{"frame_index", "timestamp_ms", "clip_time_ms"})
|
|
for index, row := range rows[start:end] {
|
|
_ = writer.Write([]string{strconv.Itoa(index), fmt.Sprintf("%.3f", float64(row.TimestampUS)/1000.0), fmt.Sprintf("%.3f", float64(row.TimestampUS-windowStartUS)/1000.0)})
|
|
}
|
|
writer.Flush()
|
|
closeErr := file.Close()
|
|
if writer.Error() != nil {
|
|
return nil, writer.Error()
|
|
}
|
|
if closeErr != nil {
|
|
return nil, closeErr
|
|
}
|
|
sha, size, err := sha256File(path)
|
|
return map[string]any{"name": filepath.Base(path), "sha256": sha, "bytes": size, "rowCount": end - start, "firstTimestampUs": rows[start].TimestampUS, "lastTimestampUs": rows[end-1].TimestampUS}, err
|
|
}
|
|
|
|
func writeIMUSlice(path string, rows []hardwareIMURow, startUS int64, endUS int64) (map[string]any, error) {
|
|
selected := make([]hardwareIMURow, 0)
|
|
for _, row := range rows {
|
|
if row.TimestampUS >= startUS && row.TimestampUS < endUS {
|
|
selected = append(selected, row)
|
|
}
|
|
}
|
|
if len(selected) == 0 {
|
|
return nil, fmt.Errorf("IMU slice is empty")
|
|
}
|
|
file, err := os.Create(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
writer := csv.NewWriter(file)
|
|
_ = writer.Write([]string{"timestamp_ms", "clip_time_ms", "x", "y", "z", "type"})
|
|
for _, row := range selected {
|
|
_ = writer.Write([]string{fmt.Sprintf("%.3f", float64(row.TimestampUS)/1000.0), fmt.Sprintf("%.3f", float64(row.TimestampUS-startUS)/1000.0), row.X, row.Y, row.Z, row.Type})
|
|
}
|
|
writer.Flush()
|
|
closeErr := file.Close()
|
|
if writer.Error() != nil {
|
|
return nil, writer.Error()
|
|
}
|
|
if closeErr != nil {
|
|
return nil, closeErr
|
|
}
|
|
sha, size, err := sha256File(path)
|
|
return map[string]any{"name": filepath.Base(path), "sha256": sha, "bytes": size, "sampleCount": len(selected), "firstTimestampUs": selected[0].TimestampUS, "lastTimestampUs": selected[len(selected)-1].TimestampUS}, err
|
|
}
|
|
|
|
func writeJSONFile(path string, payload any) (string, int64, error) {
|
|
encoded, err := json.MarshalIndent(payload, "", " ")
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
encoded = append(encoded, '\n')
|
|
if err := os.WriteFile(path, encoded, 0o644); err != nil {
|
|
return "", 0, err
|
|
}
|
|
digest := sha256.Sum256(encoded)
|
|
return hex.EncodeToString(digest[:]), int64(len(encoded)), nil
|
|
}
|
|
|
|
func artifactForMaterializationPath(path string, id string, artifactType domain.PortType) map[string]any {
|
|
sha, size, _ := sha256File(path)
|
|
return map[string]any{"id": id, "name": filepath.Base(path), "sha256": sha, "bytes": size, "type": string(artifactType)}
|
|
}
|
|
|
|
func ptsCoverageEnd(rows []hardwarePTSRow) (int64, error) {
|
|
if len(rows) < 2 {
|
|
return 0, fmt.Errorf("PTS requires at least two rows to resolve the half-open coverage end")
|
|
}
|
|
interval := rows[len(rows)-1].TimestampUS - rows[len(rows)-2].TimestampUS
|
|
if interval <= 0 {
|
|
return 0, fmt.Errorf("PTS terminal interval is invalid")
|
|
}
|
|
return rows[len(rows)-1].TimestampUS + interval, nil
|
|
}
|
|
|
|
func imuCoverageEnd(rows []hardwareIMURow) (int64, error) {
|
|
if len(rows) < 4 {
|
|
return 0, fmt.Errorf("IMU requires at least two accel/gyro timestamp pairs")
|
|
}
|
|
last := rows[len(rows)-1].TimestampUS
|
|
previous := int64(0)
|
|
for index := len(rows) - 2; index >= 0; index-- {
|
|
if rows[index].TimestampUS < last {
|
|
previous = rows[index].TimestampUS
|
|
break
|
|
}
|
|
}
|
|
if previous <= 0 || last <= previous {
|
|
return 0, fmt.Errorf("IMU terminal interval is invalid")
|
|
}
|
|
return last + (last - previous), nil
|
|
}
|
|
|
|
func maximumNearestAlignmentUS(left []hardwarePTSRow, leftStart int, leftEnd int, right []hardwarePTSRow, rightStart int, rightEnd int) int64 {
|
|
maximum := int64(0)
|
|
for _, row := range left[leftStart:leftEnd] {
|
|
index := sort.Search(rightEnd-rightStart, func(index int) bool {
|
|
return right[rightStart+index].TimestampUS >= row.TimestampUS
|
|
})
|
|
nearest := int64(math.MaxInt64)
|
|
if index < rightEnd-rightStart {
|
|
nearest = absInt64(right[rightStart+index].TimestampUS - row.TimestampUS)
|
|
}
|
|
if index > 0 {
|
|
nearest = min(nearest, absInt64(right[rightStart+index-1].TimestampUS-row.TimestampUS))
|
|
}
|
|
maximum = max(maximum, nearest)
|
|
}
|
|
return maximum
|
|
}
|
|
|
|
func absInt64(value int64) int64 {
|
|
if value < 0 {
|
|
return -value
|
|
}
|
|
return value
|
|
}
|
|
|
|
func (server *server) materializeCombinedSliceSubmission(ctx context.Context, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, payload combinedAnnotationSubmissionInput, revision int) (*humanSliceMaterialization, error) {
|
|
run, err := server.store.GetRun(ctx, task.RunID)
|
|
if err != nil || run == nil {
|
|
return nil, fmt.Errorf("load materialization run: %w", err)
|
|
}
|
|
artifactID := fmt.Sprintf("%s-%s-materialized-revision-%06d", task.RunID, task.NodeID, revision)
|
|
for _, existing := range run.Artifacts {
|
|
if existing.ID == artifactID {
|
|
manifestSHA := ""
|
|
if existing.Metadata != nil {
|
|
manifestSHA = existing.Metadata.SHA256
|
|
}
|
|
coverage := mapValue(existing.Data["coverage"])
|
|
slices := decodeMaterializationValue[domain.SliceSegment](existing.Data["normalizedSlices"])
|
|
segments := decodeMaterializationValue[domain.AnnotationSegment](existing.Data["normalizedSegments"])
|
|
sliceCount := intValue(existing.Data["sliceCount"])
|
|
if sliceCount == 0 {
|
|
sliceCount = intValue(coverage["sliceCount"])
|
|
}
|
|
if slices == nil {
|
|
slices = append([]domain.SliceSegment(nil), payload.SliceSegments...)
|
|
}
|
|
if segments == nil {
|
|
segments = append([]domain.AnnotationSegment(nil), payload.Segments...)
|
|
}
|
|
return &humanSliceMaterialization{Artifact: existing, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": sliceCount, "coverage": coverage, "runId": task.RunID, "artifactId": existing.ID, "manifestSha256": manifestSHA}, Slices: slices, Segments: segments, CoverageStartSecond: floatValue(coverage["startSecond"]), CoverageEndSecond: floatValue(coverage["endSecond"])}, nil
|
|
}
|
|
}
|
|
var execution *domain.NodeExecutionRecord
|
|
for index := range run.NodeExecutions {
|
|
if run.NodeExecutions[index].NodeID == task.NodeID {
|
|
copy := run.NodeExecutions[index]
|
|
execution = ©
|
|
break
|
|
}
|
|
}
|
|
if execution == nil {
|
|
return nil, fmt.Errorf("human gate execution is unavailable")
|
|
}
|
|
leftArtifact := artifactByID(artifactsForTarget(*execution, "fullLeftVideos"), task.ArtifactID)
|
|
if leftArtifact == nil {
|
|
leftArtifact = artifactByID(execution.InputArtifacts, task.ArtifactID)
|
|
}
|
|
if leftArtifact == nil {
|
|
return nil, fmt.Errorf("left source video is unavailable")
|
|
}
|
|
collectionID := artifactCollectionID(*leftArtifact)
|
|
rightArtifact := matchingArtifact(artifactsForTarget(*execution, "fullRightVideos"), collectionID, "right", ".mp4")
|
|
leftPTSArtifact := matchingArtifact(artifactsForTarget(*execution, "leftTimestamps"), collectionID, "left", "pts")
|
|
rightPTSArtifact := matchingArtifact(artifactsForTarget(*execution, "rightTimestamps"), collectionID, "right", "pts")
|
|
imuArtifact := matchingArtifact(artifactsForTarget(*execution, "imu"), collectionID, "imu", ".csv")
|
|
leftPath, err := server.safeMaterializationSourcePath(leftArtifact)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rightPath, err := server.safeMaterializationSourcePath(rightArtifact)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
leftPTSPath, err := server.safeMaterializationSourcePath(leftPTSArtifact)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rightPTSPath, err := server.safeMaterializationSourcePath(rightPTSArtifact)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
imuPath, err := server.safeMaterializationSourcePath(imuArtifact)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
leftPTS, err := readHardwarePTS(leftPTSPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rightPTS, err := readHardwarePTS(rightPTSPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
imuRows, err := readHardwareIMU(imuPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
leftFrames, err := ffprobeFrameCount(ctx, leftPath)
|
|
if err != nil || leftFrames != len(leftPTS) {
|
|
return nil, fmt.Errorf("left video/PTS frame mismatch: video=%d pts=%d err=%v", leftFrames, len(leftPTS), err)
|
|
}
|
|
rightFrames, err := ffprobeFrameCount(ctx, rightPath)
|
|
if err != nil || rightFrames != len(rightPTS) {
|
|
return nil, fmt.Errorf("right video/PTS frame mismatch: video=%d pts=%d err=%v", rightFrames, len(rightPTS), err)
|
|
}
|
|
leftMediaDuration, err := ffprobeDurationSeconds(ctx, leftPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
leftCoverageEnd, err := ptsCoverageEnd(leftPTS)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rightCoverageEnd, err := ptsCoverageEnd(rightPTS)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
imuEnd, err := imuCoverageEnd(imuRows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
commonStart := max(leftPTS[0].TimestampUS, rightPTS[0].TimestampUS, imuRows[0].TimestampUS)
|
|
commonEnd := min(leftCoverageEnd, rightCoverageEnd, imuEnd)
|
|
if commonEnd <= commonStart {
|
|
return nil, fmt.Errorf("required streams have no common hardware-time range")
|
|
}
|
|
leftTimelineOrigin := leftPTS[0].TimestampUS
|
|
leftPTSDurationSeconds := float64(leftCoverageEnd-leftTimelineOrigin) / 1_000_000
|
|
ptsToMediaScale := leftMediaDuration / leftPTSDurationSeconds
|
|
if math.IsNaN(ptsToMediaScale) || math.IsInf(ptsToMediaScale, 0) || ptsToMediaScale < 0.5 || ptsToMediaScale > 2 {
|
|
return nil, fmt.Errorf("left video and PTS timelines have an implausible scale mismatch")
|
|
}
|
|
timestampToMediaSecond := func(timestampUS int64) float64 {
|
|
return float64(timestampUS-leftTimelineOrigin) / 1_000_000 * ptsToMediaScale
|
|
}
|
|
mediaSecondToTimestamp := func(second float64) int64 {
|
|
return leftTimelineOrigin + int64(math.Round(second/ptsToMediaScale*1_000_000))
|
|
}
|
|
commonMediaStart := timestampToMediaSecond(commonStart)
|
|
commonMediaEnd := timestampToMediaSecond(commonEnd)
|
|
if commonMediaEnd <= commonMediaStart {
|
|
return nil, fmt.Errorf("common hardware-time range cannot be mapped to the media timeline")
|
|
}
|
|
root := filepath.Join(filepath.Clean(server.workspaceMountPath), "system", "runs", task.RunID, "human-materializations", item.ID, fmt.Sprintf("revision-%06d", revision))
|
|
if err := os.MkdirAll(filepath.Join(root, "slices"), 0o755); err != nil {
|
|
return nil, err
|
|
}
|
|
slices := append([]domain.SliceSegment(nil), payload.SliceSegments...)
|
|
sort.Slice(slices, func(left, right int) bool { return slices[left].Start < slices[right].Start })
|
|
if len(slices) == 0 {
|
|
return nil, fmt.Errorf("full synchronized coverage requires at least one slice")
|
|
}
|
|
requestedStartUS := mediaSecondToTimestamp(slices[0].Start)
|
|
requestedEndUS := mediaSecondToTimestamp(slices[len(slices)-1].End)
|
|
boundaryToleranceUS := int64(math.Ceil(sliceCoverageToleranceSeconds / ptsToMediaScale * 1_000_000))
|
|
if requestedStartUS > commonStart+boundaryToleranceUS || requestedEndUS < commonEnd-boundaryToleranceUS {
|
|
return nil, fmt.Errorf("submitted slices do not cover the full common hardware-time range")
|
|
}
|
|
clippedSlices := make([]domain.SliceSegment, 0, len(slices))
|
|
for _, slice := range slices {
|
|
slice.Start = max(slice.Start, commonMediaStart)
|
|
slice.End = min(slice.End, commonMediaEnd)
|
|
if slice.End > slice.Start {
|
|
clippedSlices = append(clippedSlices, slice)
|
|
}
|
|
}
|
|
if len(clippedSlices) == 0 {
|
|
return nil, fmt.Errorf("submitted slices do not intersect the common hardware-time range")
|
|
}
|
|
normalizedSlices := make([]domain.SliceSegment, 0, len(slices))
|
|
previousEndUS := commonStart
|
|
for index, slice := range clippedSlices {
|
|
startUS := mediaSecondToTimestamp(slice.Start)
|
|
endUS := mediaSecondToTimestamp(slice.End)
|
|
if index == 0 {
|
|
startUS = commonStart
|
|
} else if absInt64(startUS-previousEndUS) <= boundaryToleranceUS {
|
|
startUS = previousEndUS
|
|
}
|
|
if index == len(clippedSlices)-1 {
|
|
endUS = commonEnd
|
|
}
|
|
if startUS != previousEndUS || endUS <= startUS || endUS > commonEnd {
|
|
return nil, fmt.Errorf("slice %s leaves a gap or overlap in the common hardware-time range", slice.ID)
|
|
}
|
|
normalized := slice
|
|
normalized.Start = timestampToMediaSecond(startUS)
|
|
normalized.End = timestampToMediaSecond(endUS)
|
|
normalizedSlices = append(normalizedSlices, normalized)
|
|
previousEndUS = endUS
|
|
}
|
|
if previousEndUS != commonEnd {
|
|
return nil, fmt.Errorf("submitted slices do not reach the common hardware-time end")
|
|
}
|
|
normalizedSegments := make([]domain.AnnotationSegment, 0, len(payload.Segments))
|
|
for _, segment := range payload.Segments {
|
|
segment.Start = max(segment.Start, commonMediaStart)
|
|
segment.End = min(segment.End, commonMediaEnd)
|
|
if segment.End > segment.Start {
|
|
normalizedSegments = append(normalizedSegments, segment)
|
|
}
|
|
}
|
|
leftCommonStart := sort.Search(len(leftPTS), func(index int) bool { return leftPTS[index].TimestampUS >= commonStart })
|
|
leftCommonEnd := sort.Search(len(leftPTS), func(index int) bool { return leftPTS[index].TimestampUS >= commonEnd })
|
|
rightCommonStart := sort.Search(len(rightPTS), func(index int) bool { return rightPTS[index].TimestampUS >= commonStart })
|
|
rightCommonEnd := sort.Search(len(rightPTS), func(index int) bool { return rightPTS[index].TimestampUS >= commonEnd })
|
|
imuCommonCount := 0
|
|
for _, row := range imuRows {
|
|
if row.TimestampUS >= commonStart && row.TimestampUS < commonEnd {
|
|
imuCommonCount++
|
|
}
|
|
}
|
|
deliveredLeftFrames, deliveredRightFrames, deliveredIMURows := 0, 0, 0
|
|
manifestItems := make([]map[string]any, 0, len(slices))
|
|
for index, slice := range normalizedSlices {
|
|
startUS := mediaSecondToTimestamp(slice.Start)
|
|
endUS := mediaSecondToTimestamp(slice.End)
|
|
if index == 0 {
|
|
startUS = commonStart
|
|
}
|
|
if index == len(normalizedSlices)-1 {
|
|
endUS = commonEnd
|
|
}
|
|
leftStart := sort.Search(len(leftPTS), func(i int) bool { return leftPTS[i].TimestampUS >= startUS })
|
|
leftEnd := sort.Search(len(leftPTS), func(i int) bool { return leftPTS[i].TimestampUS >= endUS })
|
|
rightStart := sort.Search(len(rightPTS), func(i int) bool { return rightPTS[i].TimestampUS >= startUS })
|
|
rightEnd := sort.Search(len(rightPTS), func(i int) bool { return rightPTS[i].TimestampUS >= endUS })
|
|
sliceID := fmt.Sprintf("slice-%06d", index+1)
|
|
directory := filepath.Join(root, "slices", sliceID)
|
|
if err := os.MkdirAll(directory, 0o755); err != nil {
|
|
return nil, err
|
|
}
|
|
leftOutput, rightOutput := filepath.Join(directory, "left.mp4"), filepath.Join(directory, "right.mp4")
|
|
var leftSliceErr, rightSliceErr error
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
go func() {
|
|
defer wg.Done()
|
|
leftSliceErr = writeFrameAccurateSlice(ctx, leftPath, leftOutput, leftStart, leftEnd)
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
rightSliceErr = writeFrameAccurateSlice(ctx, rightPath, rightOutput, rightStart, rightEnd)
|
|
}()
|
|
wg.Wait()
|
|
if leftSliceErr != nil {
|
|
return nil, leftSliceErr
|
|
}
|
|
if rightSliceErr != nil {
|
|
return nil, rightSliceErr
|
|
}
|
|
leftCount, leftErr := ffprobeFrameCount(ctx, leftOutput)
|
|
rightCount, rightErr := ffprobeFrameCount(ctx, rightOutput)
|
|
if leftErr != nil || rightErr != nil || leftCount != leftEnd-leftStart || rightCount != rightEnd-rightStart {
|
|
return nil, fmt.Errorf("materialized video frame count does not match sliced PTS rows")
|
|
}
|
|
deliveredLeftFrames += leftCount
|
|
deliveredRightFrames += rightCount
|
|
leftManifest, err := writePTSSlice(filepath.Join(directory, "left_pts.csv"), leftPTS, leftStart, leftEnd, startUS)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rightManifest, err := writePTSSlice(filepath.Join(directory, "right_pts.csv"), rightPTS, rightStart, rightEnd, startUS)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
imuManifest, err := writeIMUSlice(filepath.Join(directory, "imu.csv"), imuRows, startUS, endUS)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
deliveredIMURows += int(imuManifest["sampleCount"].(int))
|
|
actions := make([]map[string]any, 0)
|
|
for _, segment := range normalizedSegments {
|
|
segmentStart := math.Max(segment.Start, slice.Start)
|
|
segmentEnd := math.Min(segment.End, slice.End)
|
|
if segmentEnd <= segmentStart {
|
|
continue
|
|
}
|
|
relativeStart := segmentStart - slice.Start
|
|
relativeEnd := segmentEnd - slice.Start
|
|
actions = append(actions, map[string]any{"id": segment.ID, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": relativeStart * 1000, "clipEndMs": relativeEnd * 1000, "startTimestampUs": mediaSecondToTimestamp(segmentStart), "endTimestampUs": mediaSecondToTimestamp(segmentEnd)})
|
|
}
|
|
annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "sliceId": sliceID, "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions}
|
|
annotationSHA, annotationBytes, err := writeJSONFile(filepath.Join(directory, "annotation.json"), annotation)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
leftVideo := artifactForMaterializationPath(leftOutput, "leftVideo", domain.PortTypeVideo)
|
|
rightVideo := artifactForMaterializationPath(rightOutput, "rightVideo", domain.PortTypeVideo)
|
|
maximumAlignmentUS := maximumNearestAlignmentUS(leftPTS, leftStart, leftEnd, rightPTS, rightStart, rightEnd)
|
|
synchronization := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "collectionId": collectionID, "sliceIndex": index + 1, "sliceCount": len(normalizedSlices), "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "commonRange": map[string]any{"startTimestampUs": commonStart, "endTimestampUs": commonEnd, "startSecond": commonMediaStart, "endSecond": commonMediaEnd}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true, "previousEndTimestampUs": func() any {
|
|
if index == 0 {
|
|
return nil
|
|
}
|
|
return manifestItems[index-1]["endTimestampUs"]
|
|
}(), "nextStartTimestampUs": func() any {
|
|
if index == len(normalizedSlices)-1 {
|
|
return nil
|
|
}
|
|
return endUS
|
|
}()}, "maxStereoAlignmentErrorUs": maximumAlignmentUS, "left": map[string]any{"video": leftVideo, "pts": leftManifest}, "right": map[string]any{"video": rightVideo, "pts": rightManifest}, "imu": imuManifest, "lineage": map[string]any{"leftVideoArtifactId": leftArtifact.ID, "rightVideoArtifactId": rightArtifact.ID, "leftPtsArtifactId": leftPTSArtifact.ID, "rightPtsArtifactId": rightPTSArtifact.ID, "imuArtifactId": imuArtifact.ID}}
|
|
syncSHA, syncBytes, err := writeJSONFile(filepath.Join(directory, "synchronized-slice.json"), synchronization)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
manifestItems = append(manifestItems, map[string]any{"sliceId": sliceID, "sourceSliceId": slice.ID, "sliceName": slice.Name, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": startUS, "endTimestampUs": endUS, "directory": filepath.ToSlash(filepath.Join("slices", sliceID)), "files": map[string]any{"leftVideo": leftVideo, "rightVideo": rightVideo, "leftPts": leftManifest, "rightPts": rightManifest, "imu": imuManifest, "annotation": map[string]any{"name": "annotation.json", "sha256": annotationSHA, "bytes": annotationBytes}, "synchronization": map[string]any{"name": "synchronized-slice.json", "sha256": syncSHA, "bytes": syncBytes}}})
|
|
}
|
|
expectedLeftFrames := leftCommonEnd - leftCommonStart
|
|
expectedRightFrames := rightCommonEnd - rightCommonStart
|
|
if deliveredLeftFrames != expectedLeftFrames || deliveredRightFrames != expectedRightFrames || deliveredIMURows != imuCommonCount {
|
|
return nil, fmt.Errorf("slice reconstruction coverage mismatch: left=%d/%d right=%d/%d imu=%d/%d", deliveredLeftFrames, expectedLeftFrames, deliveredRightFrames, expectedRightFrames, deliveredIMURows, imuCommonCount)
|
|
}
|
|
coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": commonMediaStart, "endSecond": commonMediaEnd, "durationSeconds": commonMediaEnd - commonMediaStart, "startTimestampUs": commonStart, "endTimestampUs": commonEnd, "ptsToMediaScale": ptsToMediaScale, "sliceCount": len(manifestItems), "reconstruction": map[string]any{"leftFrameCount": deliveredLeftFrames, "rightFrameCount": deliveredRightFrames, "imuRowCount": deliveredIMURows}}
|
|
captureEvidence := map[string]any{}
|
|
if capture, captureErr := server.store.GetCaptureRecordByRunID(ctx, task.RunID); captureErr != nil {
|
|
return nil, captureErr
|
|
} else if capture != nil {
|
|
captureEvidence = map[string]any{
|
|
"captureRecordId": capture.ID, "captureId": capture.CaptureID, "prefix": capture.Prefix,
|
|
"fingerprint": capture.Fingerprint, "contractRevision": capture.ContractRevision,
|
|
}
|
|
}
|
|
revisionManifest := map[string]any{"schemaVersion": "droboticflow.slice-materialization/v2", "revision": revision, "humanItemId": item.ID, "taskId": task.ID, "runId": task.RunID, "collectionId": collectionID, "capture": captureEvidence, "status": "materialized", "sliceCount": len(manifestItems), "coverage": coverage, "items": manifestItems}
|
|
manifestPath := filepath.Join(root, "revision-manifest.json")
|
|
manifestSHA, manifestBytes, err := writeJSONFile(manifestPath, revisionManifest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
artifact := domain.ExecutionArtifact{ID: artifactID, RunID: task.RunID, NodeID: task.NodeID, PortID: "materializedSliceDirectory", PortName: "materializedSliceDirectory", Name: fmt.Sprintf("revision-%06d.directory.index", revision), Type: domain.PortTypeFile, Size: fmt.Sprintf("%d files", len(manifestItems)*7+1), SizeBytes: manifestBytes, Path: root, Metadata: &domain.ArtifactMetadata{SHA256: manifestSHA, MimeType: "application/json"}, Data: map[string]any{"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": root, "manifestPath": manifestPath, "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "normalizedSlices": normalizedSlices, "normalizedSegments": normalizedSegments}}
|
|
run.Artifacts = append(run.Artifacts, artifact)
|
|
for index := range run.NodeExecutions {
|
|
if run.NodeExecutions[index].NodeID == task.NodeID {
|
|
run.NodeExecutions[index].Artifacts = append(run.NodeExecutions[index].Artifacts, artifact)
|
|
}
|
|
}
|
|
run.UpdatedAt = nowISO()
|
|
if err := server.store.UpsertRun(ctx, *run); err != nil {
|
|
return nil, err
|
|
}
|
|
return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(manifestItems), "coverage": coverage, "runId": task.RunID, "artifactId": artifact.ID, "manifestSha256": manifestSHA}, Slices: normalizedSlices, Segments: normalizedSegments, CoverageStartSecond: commonMediaStart, CoverageEndSecond: commonMediaEnd}, nil
|
|
}
|