forked from longtao.wu/droboticflow
166 lines
11 KiB
Go
166 lines
11 KiB
Go
package api
|
||
|
||
import (
|
||
"context"
|
||
"droboticflow/internal/domain"
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
)
|
||
|
||
func isSourceIndexArtifact(a *domain.ExecutionArtifact) bool {
|
||
return a != nil && stringValue(a.Data["sourceIndexSchema"]) == sourceIndexSchema
|
||
}
|
||
|
||
func (s *server) materializeSourceIndexRevision(ctx context.Context, t domain.AnnotationTaskRecord, h domain.HumanItemRecord, run *domain.RunRecord, input combinedAnnotationSubmissionInput, revision int, fingerprint string) (*humanSliceMaterialization, error) {
|
||
cert, e := s.annotationSourceCertificate(ctx, t, *run, fingerprint)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
grid := annotationBoundaryGrid{Seconds: cert.Timeline.Seconds, Timestamps: cert.Timeline.Stamps, LeftFrames: cert.Timeline.Left, RightFrames: cert.Timeline.Right}
|
||
slices := append([]domain.SliceSegment(nil), input.SliceSegments...)
|
||
sort.Slice(slices, func(i, j int) bool { return slices[i].Start < slices[j].Start })
|
||
if len(slices) == 0 || len(slices) > 500 {
|
||
return nil, fmt.Errorf("切片数量必须为 1–500")
|
||
}
|
||
segments := append([]domain.AnnotationSegment{}, input.Segments...)
|
||
for i := range segments {
|
||
a, e := grid.resolve(segments[i].Start)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
b, e := grid.resolve(segments[i].End)
|
||
if e != nil || b <= a {
|
||
return nil, fmt.Errorf("invalid action frame interval")
|
||
}
|
||
segments[i].Start, segments[i].End = grid.Seconds[a], grid.Seconds[b]
|
||
segments[i].StartTimestampUS, segments[i].EndTimestampUS = grid.Timestamps[a], grid.Timestamps[b]
|
||
}
|
||
stage, e := os.MkdirTemp("", "annotation-source-revision-")
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
defer os.RemoveAll(stage)
|
||
items := []map[string]any{}
|
||
kept, discarded := 0, 0
|
||
keepDuration, discardDuration := 0.0, 0.0
|
||
previous := 0
|
||
for i := range slices {
|
||
slice := &slices[i]
|
||
a, e := grid.resolve(slice.Start)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
b, e := grid.resolve(slice.End)
|
||
if e != nil || b <= a || a != previous {
|
||
return nil, fmt.Errorf("切片必须无重叠、无缺口地覆盖统一有效范围")
|
||
}
|
||
previous = b
|
||
slice.Start, slice.End = grid.Seconds[a], grid.Seconds[b]
|
||
slice.StartTimestampUS, slice.EndTimestampUS = grid.Timestamps[a], grid.Timestamps[b]
|
||
id := fmt.Sprintf("slice-%06d", i+1)
|
||
dir := filepath.Join(stage, "slices", id)
|
||
if e = os.MkdirAll(dir, 0755); e != nil {
|
||
return nil, e
|
||
}
|
||
if slice.IsDiscarded() {
|
||
discarded++
|
||
discardDuration += slice.End - slice.Start
|
||
} else {
|
||
kept++
|
||
keepDuration += slice.End - slice.Start
|
||
}
|
||
leftStart, leftEnd := grid.LeftFrames[a], grid.LeftFrames[b]
|
||
rightStart, rightEnd := grid.RightFrames[a], grid.RightFrames[b]
|
||
if leftEnd <= leftStart || rightEnd <= rightStart {
|
||
return nil, fmt.Errorf("双目切片必须包含有效帧")
|
||
}
|
||
actions := []map[string]any{}
|
||
for _, segment := range segments {
|
||
if slice.IsDiscarded() || segment.End <= slice.Start || segment.Start >= slice.End {
|
||
continue
|
||
}
|
||
if segment.Start < slice.Start || segment.End > slice.End {
|
||
return nil, fmt.Errorf("动作不能跨越切片边界")
|
||
}
|
||
actions = append(actions, map[string]any{"id": segment.ID, "status": effectiveActionStatus(segment), "isInvalid": segment.IsInvalid, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "dictionary": segment.Dictionary, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": (segment.Start - slice.Start) * 1000, "clipEndMs": (segment.End - slice.Start) * 1000, "startTimestampUs": segment.StartTimestampUS, "endTimestampUs": segment.EndTimestampUS})
|
||
}
|
||
window := map[string]any{"startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "semantics": "half-open"}
|
||
annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": id, "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": window, "actions": actions, "descriptions": actions}
|
||
seconds, rightSeconds, stamps := []float64{}, []float64{}, []int64{}
|
||
for j := a; j <= b; j++ {
|
||
seconds = append(seconds, grid.Seconds[j]-slice.Start)
|
||
rightSeconds = append(rightSeconds, cert.Timeline.RightSeconds[j]-cert.Timeline.RightSeconds[a])
|
||
stamps = append(stamps, grid.Timestamps[j])
|
||
}
|
||
frames := map[string]any{"left": []int{leftStart, leftEnd}, "right": []int{rightStart, rightEnd}}
|
||
sync := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "sourceIndexSchema": sourceIndexSchema, "boundaryContract": "hardware-frame-v1", "collectionId": cert.CollectionID, "sliceIndex": i + 1, "sliceCount": len(slices), "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": window, "boundarySeconds": seconds, "rightBoundarySeconds": rightSeconds, "boundaryTimestampUs": stamps, "sourceFrames": frames, "sourceMediaOffsets": map[string]any{"left": slice.Start, "right": cert.Timeline.RightSeconds[a]}, "deliveryDisposition": sliceDisposition(*slice), "left": map[string]any{"pts": map[string]any{"rowCount": leftEnd - leftStart}}, "right": map[string]any{"pts": map[string]any{"rowCount": rightEnd - rightStart}}, "imu": map[string]any{"sampleCount": cert.IMURowsBeforeBoundary[b] - cert.IMURowsBeforeBoundary[a]}, "commonRange": map[string]any{"startTimestampUs": grid.Timestamps[0], "endTimestampUs": grid.Timestamps[len(grid.Timestamps)-1], "startSecond": grid.Seconds[0], "endSecond": grid.Seconds[len(grid.Seconds)-1]}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true}}
|
||
entry := map[string]any{"sliceId": id, "sourceSliceId": slice.ID, "sliceName": slice.Name, "dictionary": slice.Dictionary, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "directory": filepath.ToSlash(filepath.Join("slices", id)), "sourceFrames": frames, "deliveryDisposition": sliceDisposition(*slice)}
|
||
if slice.IsDiscarded() {
|
||
entry["discardEvidence"] = discardReasonEvidence(*slice)
|
||
sync["discardEvidence"] = discardReasonEvidence(*slice)
|
||
}
|
||
files := map[string]any{}
|
||
for kind, value := range map[string]map[string]any{"annotation": annotation, "synchronization": sync} {
|
||
name := "annotation.json"
|
||
if kind == "synchronization" {
|
||
name = "synchronized-slice.json"
|
||
}
|
||
sha, size, e := writeJSONFile(filepath.Join(dir, name), value)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
files[kind] = map[string]any{"name": name, "sha256": sha, "bytes": size}
|
||
}
|
||
entry["files"] = files
|
||
items = append(items, entry)
|
||
}
|
||
if previous != len(grid.Timestamps)-1 {
|
||
return nil, fmt.Errorf("切片未完整覆盖统一有效范围")
|
||
}
|
||
if issues := validateDescriptionIntervals(slices, segments, (!input.sliceOnly && s.continuousActionsForItem(ctx, &h)) || t.TemplateVersion < 6); len(issues) > 0 {
|
||
return nil, fmt.Errorf("%s", issues[0].Message)
|
||
}
|
||
coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": grid.Seconds[0], "endSecond": grid.Seconds[previous], "durationSeconds": grid.Seconds[previous] - grid.Seconds[0], "startTimestampUs": grid.Timestamps[0], "endTimestampUs": grid.Timestamps[previous], "ptsToMediaScale": cert.Timeline.Scale, "sliceCount": len(slices), "reconstruction": map[string]any{"leftFrameCount": grid.LeftFrames[previous] - grid.LeftFrames[0], "rightFrameCount": grid.RightFrames[previous] - grid.RightFrames[0], "imuRowCount": cert.IMURowsBeforeBoundary[previous] - cert.IMURowsBeforeBoundary[0]}}
|
||
selection := map[string]any{"keptSliceCount": kept, "discardedSliceCount": discarded, "keptDurationSeconds": keepDuration, "discardedDurationSeconds": discardDuration}
|
||
capture := map[string]any{}
|
||
var originalSource map[string]any
|
||
if record, e := s.store.GetCaptureRecordByRunID(ctx, t.RunID); e != nil {
|
||
return nil, e
|
||
} else if record != nil {
|
||
originalSource, e = annotationRevisionOriginalSource(run, record)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
capture = map[string]any{"captureRecordId": record.ID, "captureId": record.CaptureID, "prefix": record.Prefix, "fingerprint": record.Fingerprint, "contractRevision": record.ContractRevision}
|
||
}
|
||
manifest := map[string]any{"schemaVersion": sourceIndexSchema, "revision": revision, "taskId": t.ID, "humanItemId": h.ID, "runId": t.RunID, "collectionId": cert.CollectionID, "capture": capture, "sourceFingerprint": fingerprint, "sources": cert.Sources, "sourceDurations": map[string]any{"left": cert.LeftDuration, "right": cert.RightDuration}, "status": "materialized", "sliceCount": len(items), "coverage": coverage, "deliverySelection": selection, "items": items}
|
||
if originalSource != nil {
|
||
manifest["originalSource"] = originalSource
|
||
}
|
||
manifest["sourceFrameCounts"] = map[string]any{"left": len(cert.LeftPTS), "right": len(cert.RightPTS)}
|
||
encoded, e := json.Marshal(manifest)
|
||
if e != nil || len(encoded)+1 > maxRunArtifactManifestBytes {
|
||
return nil, fmt.Errorf("源引用修订索引过大")
|
||
}
|
||
after, e := s.annotationSourceFingerprint(ctx, t, *run)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
if after != fingerprint {
|
||
return nil, fmt.Errorf("源文件在确认期间发生变化")
|
||
}
|
||
root := filepath.Join(s.workspaceMountPath, "system", "runs", t.RunID, "human-materializations", h.ID, t.ID, fmt.Sprintf("revision-%06d", revision))
|
||
sha, size, e := s.publishAnnotationDirectory(ctx, t.OrganizationID, stage, root, manifest)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
artifact := domain.ExecutionArtifact{ID: fmt.Sprintf("%s-%s-%s-materialized-revision-%06d", t.RunID, t.NodeID, t.ID, revision), RunID: t.RunID, NodeID: t.NodeID, PortID: "materializedSliceDirectory", PortName: "materializedSliceDirectory", Name: fmt.Sprintf("revision-%06d.directory.index", revision), Type: domain.PortTypeFile, Path: root, SizeBytes: size, Metadata: &domain.ArtifactMetadata{SHA256: sha, MimeType: "application/json"}, Data: map[string]any{"schemaVersion": "droboticflow.directory-index/v1", "sourceIndexSchema": sourceIndexSchema, "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": root, "manifestPath": filepath.Join(root, "revision-manifest.json"), "revision": revision, "sliceCount": len(slices), "coverage": coverage, "deliverySelection": selection, "normalizedSlices": slices, "normalizedSegments": segments, "boundaryContract": "hardware-frame-v1", "sourceFingerprint": fingerprint}}
|
||
if e = s.storeAnnotationMaterializationArtifact(ctx, t, run, artifact); e != nil {
|
||
return nil, e
|
||
}
|
||
return &humanSliceMaterialization{Artifact: artifact, Payload: map[string]any{"status": "completed", "revision": revision, "sliceCount": len(slices), "coverage": coverage, "deliverySelection": selection, "runId": t.RunID, "artifactId": artifact.ID, "manifestSha256": sha, "sourceIndex": true}, Slices: slices, Segments: segments, CoverageStartSecond: grid.Seconds[0], CoverageEndSecond: grid.Seconds[previous]}, nil
|
||
}
|