245 lines
8.5 KiB
Go
245 lines
8.5 KiB
Go
package worker
|
|
|
|
import (
|
|
"fmt"
|
|
"path"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
var capturePartDirectory = regexp.MustCompile(`(?i)^part[0-9]+$`)
|
|
|
|
// Existing admissions keep their original unit, including discarded parents.
|
|
func preserveCaptureParent(record *domain.CaptureRecord) bool {
|
|
return record != nil && (record.RunID != "" || record.Status == domain.CaptureRecordDiscarded || record.Status == domain.CaptureRecordLaunching)
|
|
}
|
|
|
|
func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
|
|
parents, err := discoverFlatCaptureGroups(objects, config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result := []captureGroup{}
|
|
for _, parent := range parents {
|
|
rootObjects := []captureObject{}
|
|
parts := map[string][]captureObject{}
|
|
for _, object := range parent.Objects {
|
|
relative := strings.TrimPrefix(object.Key, parent.Prefix+"/")
|
|
segments := strings.Split(relative, "/")
|
|
if len(segments) == 1 {
|
|
rootObjects = append(rootObjects, object)
|
|
}
|
|
if len(segments) >= 2 && capturePartDirectory.MatchString(segments[0]) {
|
|
if _, ok := parts[segments[0]]; !ok {
|
|
parts[segments[0]] = []captureObject{}
|
|
}
|
|
if len(segments) == 2 {
|
|
parts[segments[0]] = append(parts[segments[0]], object)
|
|
}
|
|
}
|
|
}
|
|
if len(parts) == 0 {
|
|
// Legacy rules remain valid, but nested files cannot fill root requirements.
|
|
flat, e := discoverFlatCaptureGroups(rootObjects, config)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if len(flat) == 0 {
|
|
parent.Missing = []string{"根目录缺少必需数据"}
|
|
result = append(result, parent)
|
|
continue
|
|
}
|
|
group := flat[0]
|
|
for _, requirement := range captureRequirements(config) {
|
|
re := regexp.MustCompile(requirement.Pattern)
|
|
count := 0
|
|
for _, object := range rootObjects {
|
|
if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) {
|
|
count++
|
|
}
|
|
}
|
|
if count > 1 {
|
|
group.Missing = append(group.Missing, "匹配冲突 "+requirement.Name)
|
|
}
|
|
}
|
|
group.SourcePrefix = group.Prefix
|
|
for _, object := range rootObjects {
|
|
group.SourceKeys = append(group.SourceKeys, object.Key)
|
|
}
|
|
result = append(result, group)
|
|
continue
|
|
}
|
|
mixed := false
|
|
for _, object := range rootObjects {
|
|
name := strings.TrimPrefix(object.Key, parent.Prefix+"/")
|
|
for _, requirement := range defaultOrbbecCaptureRequirements[:5] {
|
|
if regexp.MustCompile(requirement.Pattern).MatchString(name) {
|
|
mixed = true
|
|
}
|
|
}
|
|
}
|
|
if mixed {
|
|
parent.Missing = []string{"目录结构冲突:根目录数据与 part 数据不能混合接入"}
|
|
result = append(result, parent)
|
|
continue
|
|
}
|
|
names := make([]string, 0, len(parts))
|
|
for name := range parts {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
for _, part := range names {
|
|
group := captureGroup{CaptureID: parent.CaptureID + "/" + part, Prefix: parent.Prefix + "/" + part, SourcePrefix: parent.Prefix}
|
|
captureName := regexp.QuoteMeta(parent.CaptureID)
|
|
partName := regexp.QuoteMeta(part)
|
|
requirements := []captureRequirement{
|
|
{"左相机视频", `(?i)^` + captureName + `_camera_left_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`},
|
|
{"右相机视频", `(?i)^` + captureName + `_camera_right_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`},
|
|
{"左相机 PTS", `(?i)^` + captureName + `_camera_left_` + partName + `_pts\.csv$`},
|
|
{"右相机 PTS", `(?i)^` + captureName + `_camera_right_` + partName + `_pts\.csv$`},
|
|
{"IMU", `(?i)^` + captureName + `_imu_` + partName + `\.csv$`},
|
|
{"相机标定", `(?i)^` + captureName + `_calibration_camera\.yaml$`},
|
|
{"IMU 标定", `(?i)^` + captureName + `_calibration_imu\.yaml$`},
|
|
}
|
|
for i, requirement := range requirements {
|
|
candidates := parts[part]
|
|
base := group.Prefix + "/"
|
|
if i >= 5 {
|
|
candidates = rootObjects
|
|
base = parent.Prefix + "/"
|
|
}
|
|
re := regexp.MustCompile(requirement.Pattern)
|
|
// Prefer PTS explicitly named for the selected video variant. Raw
|
|
// part PTS is still discoverable, but is not proof of transcode alignment.
|
|
if i == 2 || i == 3 {
|
|
videoRE := regexp.MustCompile(requirements[i-2].Pattern)
|
|
for _, selected := range group.Objects {
|
|
name := strings.TrimPrefix(selected.Key, group.Prefix+"/")
|
|
if !videoRE.MatchString(name) {
|
|
continue
|
|
}
|
|
stem := name[:strings.LastIndex(name, ".")]
|
|
exact := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(stem+"_pts.csv") + `$`)
|
|
for _, candidate := range candidates {
|
|
if exact.MatchString(strings.TrimPrefix(candidate.Key, base)) {
|
|
re = exact
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
matches := []captureObject{}
|
|
for _, object := range candidates {
|
|
if re.MatchString(strings.TrimPrefix(object.Key, base)) {
|
|
matches = append(matches, object)
|
|
}
|
|
}
|
|
if len(matches) == 0 {
|
|
group.Missing = append(group.Missing, part+" 缺少 "+requirement.Name)
|
|
}
|
|
if len(matches) > 1 {
|
|
group.Missing = append(group.Missing, part+" 匹配冲突 "+requirement.Name)
|
|
}
|
|
for _, object := range matches {
|
|
if object.Size <= 0 {
|
|
group.Missing = append(group.Missing, "空文件 "+object.Key)
|
|
}
|
|
group.Objects = append(group.Objects, object)
|
|
}
|
|
}
|
|
optional := regexp.MustCompile(`(?i)^` + captureName + `_audio_` + partName + `(?:_pts\.csv|\.wav)$`)
|
|
for _, object := range parts[part] {
|
|
if optional.MatchString(strings.TrimPrefix(object.Key, group.Prefix+"/")) {
|
|
group.Objects = append(group.Objects, object)
|
|
}
|
|
}
|
|
for _, object := range rootObjects {
|
|
if strings.EqualFold(object.Key, parent.Prefix+"/meta_"+parent.CaptureID+".json") {
|
|
group.Objects = append(group.Objects, object)
|
|
}
|
|
}
|
|
// Custom requirements are additional constraints; never bypass part isolation.
|
|
for _, pattern := range config.RequiredPatterns {
|
|
if strings.TrimSpace(pattern) == "" {
|
|
continue
|
|
}
|
|
re := regexp.MustCompile(pattern)
|
|
found := false
|
|
for _, object := range group.Objects {
|
|
if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
group.Missing = append(group.Missing, part+" 缺少自定义必需数据 "+pattern)
|
|
}
|
|
}
|
|
for _, object := range group.Objects {
|
|
group.SourceKeys = append(group.SourceKeys, object.Key)
|
|
group.TotalBytes += object.Size
|
|
}
|
|
sort.Strings(group.SourceKeys)
|
|
group.Fingerprint = captureFingerprint(group.Objects)
|
|
result = append(result, group)
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func workflowUsesRawHumanCapture(workflow domain.WorkflowRecord) bool {
|
|
for _, node := range workflow.Nodes {
|
|
if node.Data.OperatorID == "operator-builtin-orbbec-data-normalize" && nodeHyperparamsMap(node)["PROCESSING_PROFILE"] == "raw_human" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func validateRawHumanCaptureMetadataSelection(workflow domain.WorkflowRecord, record *domain.CaptureRecord) error {
|
|
if !workflowUsesRawHumanCapture(workflow) {
|
|
return nil
|
|
}
|
|
if record == nil || record.SourcePrefix == "" {
|
|
return fmt.Errorf("原始数据人工版缺少采集根目录")
|
|
}
|
|
key := record.SourcePrefix + "/meta_" + path.Base(record.SourcePrefix) + ".json"
|
|
count := 0
|
|
for _, selected := range record.SourceKeys {
|
|
if selected == key {
|
|
count++
|
|
}
|
|
}
|
|
if count != 1 {
|
|
return fmt.Errorf("原始数据人工版需要唯一的 %s", path.Base(key))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func discoverCaptureGroupsForWorkflow(objects []captureObject, config domain.CaptureAutomationConfig, workflow domain.WorkflowRecord) ([]captureGroup, error) {
|
|
groups, err := discoverCaptureGroups(objects, config)
|
|
if err != nil || (!workflowUsesV4CaptureDelivery(workflow) && !workflowUsesRawHumanCapture(workflow)) {
|
|
return groups, err
|
|
}
|
|
for i := range groups {
|
|
group := &groups[i]
|
|
record := domain.CaptureRecord{Bucket: config.Bucket, CaptureID: group.CaptureID, Prefix: group.Prefix,
|
|
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys, Fingerprint: group.Fingerprint}
|
|
if err := validateV4CaptureSourceSelection(workflow, &record); err != nil {
|
|
group.Missing = append(group.Missing, err.Error())
|
|
}
|
|
if err := validateRawHumanCaptureMetadataSelection(workflow, &record); err != nil {
|
|
group.Missing = append(group.Missing, err.Error())
|
|
}
|
|
metadataKey := group.SourcePrefix + "/meta_" + path.Base(group.SourcePrefix) + ".json"
|
|
for _, object := range group.Objects {
|
|
if strings.EqualFold(object.Key, metadataKey) && object.Size <= 0 {
|
|
group.Missing = append(group.Missing, "采集 meta 文件为空 "+path.Base(object.Key))
|
|
}
|
|
}
|
|
}
|
|
return groups, nil
|
|
}
|