droboticflow/internal/worker/capture_layout.go

245 lines
8.5 KiB
Go

package worker
import (
"fmt"
"path"
"regexp"
"sort"
"strings"
"droboticflow/internal/domain"
)
var capturePartDirectory = regexp.MustCompile(`(?i)^part[0-9]+$`)
// Existing admissions keep their original unit, including discarded parents.
func preserveCaptureParent(record *domain.CaptureRecord) bool {
return record != nil && (record.RunID != "" || record.Status == domain.CaptureRecordDiscarded || record.Status == domain.CaptureRecordLaunching)
}
func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
parents, err := discoverFlatCaptureGroups(objects, config)
if err != nil {
return nil, err
}
result := []captureGroup{}
for _, parent := range parents {
rootObjects := []captureObject{}
parts := map[string][]captureObject{}
for _, object := range parent.Objects {
relative := strings.TrimPrefix(object.Key, parent.Prefix+"/")
segments := strings.Split(relative, "/")
if len(segments) == 1 {
rootObjects = append(rootObjects, object)
}
if len(segments) >= 2 && capturePartDirectory.MatchString(segments[0]) {
if _, ok := parts[segments[0]]; !ok {
parts[segments[0]] = []captureObject{}
}
if len(segments) == 2 {
parts[segments[0]] = append(parts[segments[0]], object)
}
}
}
if len(parts) == 0 {
// Legacy rules remain valid, but nested files cannot fill root requirements.
flat, e := discoverFlatCaptureGroups(rootObjects, config)
if e != nil {
return nil, e
}
if len(flat) == 0 {
parent.Missing = []string{"根目录缺少必需数据"}
result = append(result, parent)
continue
}
group := flat[0]
for _, requirement := range captureRequirements(config) {
re := regexp.MustCompile(requirement.Pattern)
count := 0
for _, object := range rootObjects {
if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) {
count++
}
}
if count > 1 {
group.Missing = append(group.Missing, "匹配冲突 "+requirement.Name)
}
}
group.SourcePrefix = group.Prefix
for _, object := range rootObjects {
group.SourceKeys = append(group.SourceKeys, object.Key)
}
result = append(result, group)
continue
}
mixed := false
for _, object := range rootObjects {
name := strings.TrimPrefix(object.Key, parent.Prefix+"/")
for _, requirement := range defaultOrbbecCaptureRequirements[:5] {
if regexp.MustCompile(requirement.Pattern).MatchString(name) {
mixed = true
}
}
}
if mixed {
parent.Missing = []string{"目录结构冲突:根目录数据与 part 数据不能混合接入"}
result = append(result, parent)
continue
}
names := make([]string, 0, len(parts))
for name := range parts {
names = append(names, name)
}
sort.Strings(names)
for _, part := range names {
group := captureGroup{CaptureID: parent.CaptureID + "/" + part, Prefix: parent.Prefix + "/" + part, SourcePrefix: parent.Prefix}
captureName := regexp.QuoteMeta(parent.CaptureID)
partName := regexp.QuoteMeta(part)
requirements := []captureRequirement{
{"左相机视频", `(?i)^` + captureName + `_camera_left_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`},
{"右相机视频", `(?i)^` + captureName + `_camera_right_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`},
{"左相机 PTS", `(?i)^` + captureName + `_camera_left_` + partName + `_pts\.csv$`},
{"右相机 PTS", `(?i)^` + captureName + `_camera_right_` + partName + `_pts\.csv$`},
{"IMU", `(?i)^` + captureName + `_imu_` + partName + `\.csv$`},
{"相机标定", `(?i)^` + captureName + `_calibration_camera\.yaml$`},
{"IMU 标定", `(?i)^` + captureName + `_calibration_imu\.yaml$`},
}
for i, requirement := range requirements {
candidates := parts[part]
base := group.Prefix + "/"
if i >= 5 {
candidates = rootObjects
base = parent.Prefix + "/"
}
re := regexp.MustCompile(requirement.Pattern)
// Prefer PTS explicitly named for the selected video variant. Raw
// part PTS is still discoverable, but is not proof of transcode alignment.
if i == 2 || i == 3 {
videoRE := regexp.MustCompile(requirements[i-2].Pattern)
for _, selected := range group.Objects {
name := strings.TrimPrefix(selected.Key, group.Prefix+"/")
if !videoRE.MatchString(name) {
continue
}
stem := name[:strings.LastIndex(name, ".")]
exact := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(stem+"_pts.csv") + `$`)
for _, candidate := range candidates {
if exact.MatchString(strings.TrimPrefix(candidate.Key, base)) {
re = exact
break
}
}
}
}
matches := []captureObject{}
for _, object := range candidates {
if re.MatchString(strings.TrimPrefix(object.Key, base)) {
matches = append(matches, object)
}
}
if len(matches) == 0 {
group.Missing = append(group.Missing, part+" 缺少 "+requirement.Name)
}
if len(matches) > 1 {
group.Missing = append(group.Missing, part+" 匹配冲突 "+requirement.Name)
}
for _, object := range matches {
if object.Size <= 0 {
group.Missing = append(group.Missing, "空文件 "+object.Key)
}
group.Objects = append(group.Objects, object)
}
}
optional := regexp.MustCompile(`(?i)^` + captureName + `_audio_` + partName + `(?:_pts\.csv|\.wav)$`)
for _, object := range parts[part] {
if optional.MatchString(strings.TrimPrefix(object.Key, group.Prefix+"/")) {
group.Objects = append(group.Objects, object)
}
}
for _, object := range rootObjects {
if strings.EqualFold(object.Key, parent.Prefix+"/meta_"+parent.CaptureID+".json") {
group.Objects = append(group.Objects, object)
}
}
// Custom requirements are additional constraints; never bypass part isolation.
for _, pattern := range config.RequiredPatterns {
if strings.TrimSpace(pattern) == "" {
continue
}
re := regexp.MustCompile(pattern)
found := false
for _, object := range group.Objects {
if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) {
found = true
}
}
if !found {
group.Missing = append(group.Missing, part+" 缺少自定义必需数据 "+pattern)
}
}
for _, object := range group.Objects {
group.SourceKeys = append(group.SourceKeys, object.Key)
group.TotalBytes += object.Size
}
sort.Strings(group.SourceKeys)
group.Fingerprint = captureFingerprint(group.Objects)
result = append(result, group)
}
}
return result, nil
}
func workflowUsesRawHumanCapture(workflow domain.WorkflowRecord) bool {
for _, node := range workflow.Nodes {
if node.Data.OperatorID == "operator-builtin-orbbec-data-normalize" && nodeHyperparamsMap(node)["PROCESSING_PROFILE"] == "raw_human" {
return true
}
}
return false
}
func validateRawHumanCaptureMetadataSelection(workflow domain.WorkflowRecord, record *domain.CaptureRecord) error {
if !workflowUsesRawHumanCapture(workflow) {
return nil
}
if record == nil || record.SourcePrefix == "" {
return fmt.Errorf("原始数据人工版缺少采集根目录")
}
key := record.SourcePrefix + "/meta_" + path.Base(record.SourcePrefix) + ".json"
count := 0
for _, selected := range record.SourceKeys {
if selected == key {
count++
}
}
if count != 1 {
return fmt.Errorf("原始数据人工版需要唯一的 %s", path.Base(key))
}
return nil
}
func discoverCaptureGroupsForWorkflow(objects []captureObject, config domain.CaptureAutomationConfig, workflow domain.WorkflowRecord) ([]captureGroup, error) {
groups, err := discoverCaptureGroups(objects, config)
if err != nil || (!workflowUsesV4CaptureDelivery(workflow) && !workflowUsesRawHumanCapture(workflow)) {
return groups, err
}
for i := range groups {
group := &groups[i]
record := domain.CaptureRecord{Bucket: config.Bucket, CaptureID: group.CaptureID, Prefix: group.Prefix,
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys, Fingerprint: group.Fingerprint}
if err := validateV4CaptureSourceSelection(workflow, &record); err != nil {
group.Missing = append(group.Missing, err.Error())
}
if err := validateRawHumanCaptureMetadataSelection(workflow, &record); err != nil {
group.Missing = append(group.Missing, err.Error())
}
metadataKey := group.SourcePrefix + "/meta_" + path.Base(group.SourcePrefix) + ".json"
for _, object := range group.Objects {
if strings.EqualFold(object.Key, metadataKey) && object.Size <= 0 {
group.Missing = append(group.Missing, "采集 meta 文件为空 "+path.Base(object.Key))
}
}
}
return groups, nil
}