From cd5bfdb4098f66a7c6c99cfd022d1d2ae338fbaa Mon Sep 17 00:00:00 2001 From: eust-w Date: Thu, 10 Sep 2026 20:46:38 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Support=20legacy=20and=20isolated?= =?UTF-8?q?=20part=20Capture=20admission?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../builtin/directory_scan/operator.py | 15 +- .../builtin/orbbec_data_normalize/operator.py | 29 ++- docs/capture-layout-compatibility.md | 20 ++ internal/api/capture_automation.go | 5 + internal/api/capture_part_test.go | 23 +++ internal/domain/business_display.go | 4 + internal/domain/capture_automation.go | 2 + internal/domain/types.go | 2 + internal/worker/capture_automation.go | 53 ++++- internal/worker/capture_automation_test.go | 50 +++-- internal/worker/capture_layout.go | 188 ++++++++++++++++++ internal/worker/capture_layout_test.go | 138 +++++++++++++ .../builtin/directory_scan/src/operator.py | 15 +- .../orbbec_data_normalize/src/operator.py | 29 ++- tests/backend/test_capture_part_selection.py | 80 ++++++++ 15 files changed, 616 insertions(+), 37 deletions(-) create mode 100644 docs/capture-layout-compatibility.md create mode 100644 internal/api/capture_part_test.go create mode 100644 internal/worker/capture_layout.go create mode 100644 internal/worker/capture_layout_test.go create mode 100644 tests/backend/test_capture_part_selection.py diff --git a/backend/operators/builtin/directory_scan/operator.py b/backend/operators/builtin/directory_scan/operator.py index f5b4aa65..4359851e 100644 --- a/backend/operators/builtin/directory_scan/operator.py +++ b/backend/operators/builtin/directory_scan/operator.py @@ -65,6 +65,7 @@ def _bos_entries( pattern: str, params: dict[str, str], helpers: HelperMap, + object_keys: list[str] | None = None, ) -> list[dict[str, Any]]: endpoint = str(params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "")).strip() access_key_id = str(params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "")).strip() @@ -84,6 +85,15 @@ def _bos_entries( for item in listed_entries if "/_droboticflow/manifests/" not in "/" + str(item.get("path", "")).lstrip("/") ] + if object_keys is not None: + base = prefix.rstrip("/") + "/" + if not object_keys or any(not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in object_keys): + raise RuntimeError("目录文件选择超出 Capture 范围或为空") + selected = {key[len(base):] for key in object_keys} + entries = [item for item in entries if str(item.get("path", "")).lstrip("/") in selected] + missing = selected - {str(item.get("path", "")).lstrip("/") for item in entries} + if missing: + raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}") if not entries: raise RuntimeError(f"目录读取未在 bos://{bucket}/{prefix} 找到真实文件") client, _ = helpers["build_bos_client"](endpoint, access_key_id, secret_access_key) @@ -122,7 +132,8 @@ def _run_legacy( directory_ref = _directory_ref(inputs) if directory_ref is not None: bucket, prefix = directory_ref - entries = _bos_entries(bucket, prefix, pattern, params, helpers) + selected_ref = (_directory_input(inputs) or {}).get("payload", {}).get("ref", {}) + entries = _bos_entries(bucket, prefix, pattern, params, helpers, selected_ref.get("objectKeys")) source_path = helpers["canonical_bos_path"](bucket, prefix) payload = { "schemaVersion": "droboticflow.directory-read/v1", @@ -248,7 +259,7 @@ def _run_multiport( bucket = str(ref.get("bucket", "")).strip() prefix = str(ref.get("prefix", "")).strip().strip("/") if bucket: - entries = _bos_entries(bucket, prefix, pattern, params, helpers) + entries = _bos_entries(bucket, prefix, pattern, params, helpers, ref.get("objectKeys")) root_path = str(helpers["canonical_bos_path"](bucket, prefix)) else: root_path = str(directory_input.get("path", "")).strip() diff --git a/backend/operators/builtin/orbbec_data_normalize/operator.py b/backend/operators/builtin/orbbec_data_normalize/operator.py index 023308f1..180bf71e 100644 --- a/backend/operators/builtin/orbbec_data_normalize/operator.py +++ b/backend/operators/builtin/orbbec_data_normalize/operator.py @@ -18,7 +18,12 @@ def _entry_path(entry: dict[str, Any]) -> str: def _capture_id(path: str) -> str: - matched = CAPTURE_PATTERN.search(path) + # The Capture name is not required to contain a date (legacy business names). + stem = pathlib.PurePosixPath(path).stem + named = re.match(r"^(.+?)_(?:camera_(?:left|right)_part\d+(?:_.*)?|imu(?:_part\d+)?|audio(?:_part\d+)?(?:_pts)?|calibration_(?:camera|imu))$", stem, re.IGNORECASE) + if named: + return named.group(1) + matched = CAPTURE_PATTERN.search(path) or CAPTURE_PATTERN.search(pathlib.PurePosixPath(path).name) if matched: return matched.group("capture") parts = pathlib.PurePosixPath(path).parts @@ -33,13 +38,17 @@ def _camera_side(path: str) -> str: def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, str]: """Match sidecars within one Capture and segment, never the first segment.""" video = pathlib.PurePosixPath(video_path) - part = re.match(r"^(?P.+)_camera_(?:left|right)_(?Ppart\d+)(?P.*)$", video.stem) + part = re.match(r"^(?P.+)_camera_(?:left|right)_(?Ppart\d+)(?P.*)$", video.stem, re.IGNORECASE) def find(name: str) -> str: candidates = [_entry_path(item) for item in entries - if pathlib.PurePosixPath(_entry_path(item)).name == name] + if pathlib.PurePosixPath(_entry_path(item)).name.lower() == name.lower()] local = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent] - matches = local or candidates + # Only calibration files may live at the Capture root above this part. + shared = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent.parent] + matches = local or (shared if "_calibration_" in name else []) + if candidates and not matches: + raise RuntimeError(f"配套文件不唯一或不在当前 part: {video_path} -> {name}") if len(matches) > 1: raise RuntimeError(f"配套文件不唯一: {video_path} -> {name}: {matches}") return matches[0] if matches else "" @@ -59,7 +68,7 @@ def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, s "imu": f"{capture}_imu_{segment}.csv"} result = {key: find(name) for key, name in names.items()} for key, suffix in (("audio", "audio.wav"), ("audioPts", "audio_pts.csv"), ("imu", "imu.csv")): - if not result[key]: + if not result[key] and not re.fullmatch(r"part\d+", video.parent.name, re.IGNORECASE): result[key] = find(f"{capture}_{suffix}") result.update({"cameraCalibration": find(f"{capture}_calibration_camera.yaml"), "imuCalibration": find(f"{capture}_calibration_imu.yaml")}) @@ -204,6 +213,16 @@ def _directory_entries(inputs: list[dict[str, Any]], helpers: HelperMap) -> tupl if not endpoint or not access_key_id or not secret_access_key: raise RuntimeError("Orbbec 多端口标准化需要目录引用对应的 BOS 凭证") entries = helpers["list_bos_entries"](bucket, prefix, "", endpoint, access_key_id, secret_access_key) + if "objectKeys" in ref: + base = prefix.rstrip("/") + "/" + keys = ref["objectKeys"] + if not isinstance(keys, list) or not keys or any(not isinstance(key, str) or not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in keys): + raise RuntimeError("目录文件选择超出 Capture 范围或为空") + selected = {key[len(base):] for key in keys} + entries = [item for item in entries if _entry_path(item) in selected] + missing = selected - {_entry_path(item) for item in entries} + if missing: + raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}") source_path = str(helpers["canonical_bos_path"](bucket, prefix)) return [ { diff --git a/docs/capture-layout-compatibility.md b/docs/capture-layout-compatibility.md new file mode 100644 index 00000000..76ec1164 --- /dev/null +++ b/docs/capture-layout-compatibility.md @@ -0,0 +1,20 @@ +# Capture layout compatibility + +- Flat legacy directories keep the existing filename requirements and one Run per directory. Nested files cannot satisfy root requirements. Multiple candidates for a required type block admission rather than selecting the first file. +- A direct `part[0-9]+` child selects segmented layout. Each part is an independent unit, identified by workflow + parent prefix + part. Capture names are not restricted to FYLX or a date pattern. +- Part video names must contain the actual Capture name, camera side and actual part name. Original `.mp4/.mov/.mkv` and suffixed variants such as `_silent_480p_wm.mp4` are recognized. Multiple variants block intake as ambiguous. +- Part PTS and IMU names must match the Capture and part exactly. Video-specific `_pts.csv` takes precedence over raw part PTS when present. Only the parent's two `_calibration_camera.yaml` / `_calibration_imu.yaml` files are shared. Audio and meta are optional. +- Root sensor data plus part directories is a structural conflict. Missing/empty files block only their unit; missing shared calibration blocks all affected parts. Custom required patterns remain supported for flat layouts; for part layouts they add constraints within the selected unit, never permit cross-part matching. +- A directory reference records the parent `prefix`, explicit `objectKeys` and display `captureUnit`. Both directory reading and normalization enforce the selection, including on retry. Missing selected files fail closed. Each part's fingerprint includes only its selected files and shared calibration/meta. +- Existing admitted parent records retain their Run association and are not split/relaunched by scanning. Discarded or launching parents also prevent child admissions. No completion manifest is required; non-empty required files and the existing stability window still apply. +- Once a part has been admitted, the parent cannot later be admitted as a flat unit by moving/removing source files. + +## Data-quality boundary + +Filename compatibility does not prove frame correspondence. In particular, a transcoded `_silent_480p_wm.mp4` plus raw `_part0001_pts.csv` is recognized by intake but remains subject to the existing normalizer rejection when a matching transformed-video PTS/frame mapping is absent. This change must not invent timestamps or silently bypass that safety check. + +## Validation + +Go tests cover flat/part grouping, independent identities/fingerprints, shared calibration, missing/empty/wrong-part files, duplicate videos, mixed layouts, historical-parent protection, Run binding/de-duplication and retry references. Python tests cover scope enforcement in both readers, parent calibration pairing and unchanged PTS safety. + +Deploy worker, API and both directory-scan / Orbbec-normalization operator images together. Do not enable a new scanner with old operators that ignore `objectKeys`. No live deployment or business replay is performed by unit tests. diff --git a/internal/api/capture_automation.go b/internal/api/capture_automation.go index 78afa2ea..694db2dd 100644 --- a/internal/api/capture_automation.go +++ b/internal/api/capture_automation.go @@ -641,6 +641,11 @@ func captureReprocessInput(workflow *domain.WorkflowRecord, record *domain.Captu Provider: domain.StorageProviderBOS, ConnectionID: config.ConnectionID, Bucket: config.Bucket, Prefix: record.Prefix, VersionID: record.Fingerprint, Kind: domain.ArtifactRefKindDirectory, } + if record.SourcePrefix != "" { + ref.Prefix = record.SourcePrefix + ref.ObjectKeys = append([]string(nil), record.SourceKeys...) + ref.CaptureUnit = record.CaptureID + } return map[string]domain.ArtifactPayload{ config.WorkflowInputID: {Mode: domain.ArtifactTransferModeRef, Type: input.Type, Ref: ref}, }, nil diff --git a/internal/api/capture_part_test.go b/internal/api/capture_part_test.go new file mode 100644 index 00000000..6f6cab9b --- /dev/null +++ b/internal/api/capture_part_test.go @@ -0,0 +1,23 @@ +package api + +import ( + "droboticflow/internal/domain" + "testing" +) + +func TestCapturePartReprocessPreservesSelectedFiles(t *testing.T) { + workflow := &domain.WorkflowRecord{CaptureAutomation: &domain.CaptureAutomationConfig{ConnectionID: "connection", Bucket: "bucket", WorkflowInputID: "sourceDirectory"}, WorkflowInputs: []domain.WorkflowInputDefinition{{ID: "sourceDirectory", Type: domain.PortTypeFile}}} + record := &domain.CaptureRecord{Prefix: "incoming/capture/part0001", SourcePrefix: "incoming/capture", CaptureID: "capture/part0001", SourceKeys: []string{"incoming/capture/part0001/left.mp4", "incoming/capture/calibration_camera.yaml"}, Fingerprint: "snapshot"} + inputs, err := captureReprocessInput(workflow, record) + if err != nil { + t.Fatal(err) + } + ref := inputs["sourceDirectory"].Ref + if ref.Prefix != record.SourcePrefix || ref.CaptureUnit != record.CaptureID || len(ref.ObjectKeys) != 2 { + t.Fatalf("selection lost during retry: %+v", ref) + } + record.SourceKeys[0] = "changed" + if ref.ObjectKeys[0] == "changed" { + t.Fatal("retry input aliases mutable record") + } +} diff --git a/internal/domain/business_display.go b/internal/domain/business_display.go index 716daa86..132e0ca7 100644 --- a/internal/domain/business_display.go +++ b/internal/domain/business_display.go @@ -131,6 +131,10 @@ func CaptureDirectoriesFromRun(run RunRecord) []string { if input.Ref == nil { continue } + if input.Ref.CaptureUnit != "" { + values = append(values, input.Ref.CaptureUnit) + continue + } for _, candidate := range []string{input.Ref.Prefix, input.Ref.Key, input.Ref.ManifestKey} { if directory := captureDirectoryFromDirectoryValue(candidate); directory != "" { values = append(values, directory) diff --git a/internal/domain/capture_automation.go b/internal/domain/capture_automation.go index 6f346f69..f7261ab2 100644 --- a/internal/domain/capture_automation.go +++ b/internal/domain/capture_automation.go @@ -57,6 +57,8 @@ type CaptureActionRecord struct { } type CaptureRecord struct { + SourcePrefix string `json:"sourcePrefix,omitempty" bson:"sourcePrefix,omitempty"` + SourceKeys []string `json:"sourceKeys,omitempty" bson:"sourceKeys,omitempty"` ID string `json:"id" bson:"id"` OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` ProjectID string `json:"projectId" bson:"projectId"` diff --git a/internal/domain/types.go b/internal/domain/types.go index 6d49c564..3e9ef9cc 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -210,6 +210,8 @@ const ( // ArtifactReference is a credential-free pointer to durable object storage. type ArtifactReference struct { + ObjectKeys []string `json:"objectKeys,omitempty" bson:"objectKeys,omitempty"` + CaptureUnit string `json:"captureUnit,omitempty" bson:"captureUnit,omitempty"` Provider StorageProvider `json:"provider" bson:"provider"` ConnectionID string `json:"connectionId" bson:"connectionId"` Bucket string `json:"bucket" bson:"bucket"` diff --git a/internal/worker/capture_automation.go b/internal/worker/capture_automation.go index 2eb8ff61..33df6eea 100644 --- a/internal/worker/capture_automation.go +++ b/internal/worker/capture_automation.go @@ -30,13 +30,15 @@ type captureObject struct { } type captureGroup struct { - CaptureID string - Prefix string - ManifestKey string - Fingerprint string - Missing []string - Objects []captureObject - TotalBytes int64 + SourcePrefix string + SourceKeys []string + CaptureID string + Prefix string + ManifestKey string + Fingerprint string + Missing []string + Objects []captureObject + TotalBytes int64 } type captureRequirement struct { @@ -122,7 +124,7 @@ func captureAutomationFingerprint(workflowID string, config domain.CaptureAutoma return hex.EncodeToString(sum[:]) } -func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) { +func discoverFlatCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) { requirements := captureRequirements(config) compiledRequirements := make([]*regexp.Regexp, 0, len(requirements)) for _, requirement := range requirements { @@ -462,6 +464,11 @@ func (runtime *Runtime) launchCaptureRun(ctx context.Context, workflow domain.Wo VersionID: record.Fingerprint, Kind: domain.ArtifactRefKindDirectory, }, } + if record.SourcePrefix != "" { + payload.Ref.Prefix = record.SourcePrefix + payload.Ref.ObjectKeys = append([]string(nil), record.SourceKeys...) + payload.Ref.CaptureUnit = record.CaptureID + } run, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerScheduled, map[string]domain.ArtifactPayload{config.WorkflowInputID: payload}, record.ID) if err != nil { record.Status = domain.CaptureRecordReady @@ -530,8 +537,37 @@ func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow d if err != nil { return err } + priorRecords, err := runtime.store.ListCaptureRecords(ctx, workflow.OrganizationID, workflow.ID, 0) + if err != nil { + return err + } + admittedPartParents := map[string]bool{} + for _, prior := range priorRecords { + if prior.SourcePrefix != "" && prior.SourcePrefix != prior.Prefix && preserveCaptureParent(&prior) { + admittedPartParents[prior.SourcePrefix] = true + } + } newCount := 0 + historicalParents := map[string]bool{} for _, group := range groups { + if admittedPartParents[group.Prefix] { + group.Missing = append(group.Missing, "该目录已有 part 接入记录,不能重新作为整目录接入") + } + if group.SourcePrefix != "" && group.SourcePrefix != group.Prefix { + parent, e := runtime.store.GetCaptureRecordByWorkflowPrefix(ctx, workflow.ID, group.SourcePrefix) + if e != nil { + return e + } + if preserveCaptureParent(parent) { + if !historicalParents[parent.ID] && parent.RunID != "" { + if e := runtime.reconcileCaptureRun(ctx, parent); e != nil { + return e + } + } + historicalParents[parent.ID] = true + continue + } + } existing, err := runtime.store.GetCaptureRecordByWorkflowPrefix(ctx, workflow.ID, group.Prefix) if err != nil { return err @@ -564,6 +600,7 @@ func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow d missing := append([]string(nil), group.Missing...) nowText := nowISO() record := domain.CaptureRecord{ + SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys, ID: captureRecordID(workflow.ID, group.Prefix), OrganizationID: workflow.OrganizationID, ProjectID: workflow.ProjectID, WorkflowID: workflow.ID, ConnectionID: config.ConnectionID, Bucket: config.Bucket, Prefix: group.Prefix, CaptureID: group.CaptureID, diff --git a/internal/worker/capture_automation_test.go b/internal/worker/capture_automation_test.go index 92b97037..ece54598 100644 --- a/internal/worker/capture_automation_test.go +++ b/internal/worker/capture_automation_test.go @@ -2,6 +2,7 @@ package worker import ( "context" + "regexp" "testing" "time" @@ -54,20 +55,8 @@ func TestCaptureIMURecognizesLegacyAndSegmentNames(t *testing.T) { {"notimu.csv", false}, } { t.Run(tc.name, func(t *testing.T) { - objects := completeCaptureObjects("incoming", "capture") - objects[4].Key = "incoming/capture/" + tc.name - groups, err := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"})) - if err != nil || len(groups) != 1 { - t.Fatalf("groups=%v err=%v", groups, err) - } - missing := false - for _, name := range groups[0].Missing { - if name == "IMU" { - missing = true - } - } - if missing == tc.valid { - t.Fatalf("name=%s valid=%v missing=%v", tc.name, tc.valid, groups[0].Missing) + if regexp.MustCompile(defaultOrbbecCaptureRequirements[4].Pattern).MatchString(tc.name) != tc.valid { + t.Fatalf("unexpected IMU filename match: %s", tc.name) } }) } @@ -84,7 +73,7 @@ func TestCaptureSegmentIMUStillRejectsEmptyAndDoesNotFillOtherCapture(t *testing if err != nil || len(groups) != 2 { t.Fatalf("groups=%v err=%v", groups, err) } - if len(groups[0].Missing) != 1 || groups[0].Missing[0] != "空文件 part0001/a_imu_part0001.csv" { + if len(groups[0].Missing) != 1 || groups[0].Missing[0] != "目录结构冲突:根目录数据与 part 数据不能混合接入" { t.Fatalf("empty IMU accepted: %v", groups[0].Missing) } if len(groups[1].Missing) != 1 || groups[1].Missing[0] != "IMU" { @@ -341,6 +330,37 @@ func TestLaunchCaptureRunBindsOneCompleteDirectory(t *testing.T) { if input.Ref == nil || input.Ref.Prefix != record.Prefix || input.Ref.ManifestKey != "" || input.Ref.Kind != domain.ArtifactRefKindDirectory { t.Fatalf("unexpected directory binding: %#v", input) } + // New part units retain the parent source root plus an explicit file selection. + part := record + part.ID = "capture-record-part-1" + part.RunID = "" + part.Status = domain.CaptureRecordReady + part.SourcePrefix = record.Prefix + part.Prefix = record.Prefix + "/part0001" + part.CaptureID = record.CaptureID + "/part0001" + part.SourceKeys = []string{part.Prefix + "/video.mp4", record.Prefix + "/calibration_camera.yaml"} + if err := dataStore.UpsertCaptureRecord(ctx, part); err != nil { + t.Fatal(err) + } + if err := runtime.launchCaptureRun(ctx, *workflow, config, &part); err != nil { + t.Fatal(err) + } + partRun, err := dataStore.GetRun(ctx, part.RunID) + if err != nil || partRun == nil { + t.Fatal(err) + } + ref := partRun.Inputs["sourceDirectory"].Ref + if ref.Prefix != record.Prefix || len(ref.ObjectKeys) != 2 || ref.CaptureUnit != part.CaptureID { + t.Fatalf("part selection lost: %+v", ref) + } + if len(partRun.CaptureDirectories) != 1 || partRun.CaptureDirectories[0] != part.CaptureID { + t.Fatal(partRun.CaptureDirectories) + } + firstRunID := part.RunID + part.Status = domain.CaptureRecordReady + if err := runtime.launchCaptureRun(ctx, *workflow, config, &part); err != nil || part.RunID != firstRunID { + t.Fatal("duplicate part launch", err) + } } func TestNormalizedCaptureAutomationDefaultsToThirtySecondStabilityWindow(t *testing.T) { diff --git a/internal/worker/capture_layout.go b/internal/worker/capture_layout.go new file mode 100644 index 00000000..fc01593d --- /dev/null +++ b/internal/worker/capture_layout.go @@ -0,0 +1,188 @@ +package worker + +import ( + "regexp" + "sort" + "strings" + + "droboticflow/internal/domain" +) + +var capturePartDirectory = regexp.MustCompile(`(?i)^part[0-9]+$`) + +// Existing admissions keep their original unit, including discarded parents. +func preserveCaptureParent(record *domain.CaptureRecord) bool { + return record != nil && (record.RunID != "" || record.Status == domain.CaptureRecordDiscarded || record.Status == domain.CaptureRecordLaunching) +} + +func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) { + parents, err := discoverFlatCaptureGroups(objects, config) + if err != nil { + return nil, err + } + result := []captureGroup{} + for _, parent := range parents { + rootObjects := []captureObject{} + parts := map[string][]captureObject{} + for _, object := range parent.Objects { + relative := strings.TrimPrefix(object.Key, parent.Prefix+"/") + segments := strings.Split(relative, "/") + if len(segments) == 1 { + rootObjects = append(rootObjects, object) + } + if len(segments) >= 2 && capturePartDirectory.MatchString(segments[0]) { + if _, ok := parts[segments[0]]; !ok { + parts[segments[0]] = []captureObject{} + } + if len(segments) == 2 { + parts[segments[0]] = append(parts[segments[0]], object) + } + } + } + if len(parts) == 0 { + // Legacy rules remain valid, but nested files cannot fill root requirements. + flat, e := discoverFlatCaptureGroups(rootObjects, config) + if e != nil { + return nil, e + } + if len(flat) == 0 { + parent.Missing = []string{"根目录缺少必需数据"} + result = append(result, parent) + continue + } + group := flat[0] + for _, requirement := range captureRequirements(config) { + re := regexp.MustCompile(requirement.Pattern) + count := 0 + for _, object := range rootObjects { + if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) { + count++ + } + } + if count > 1 { + group.Missing = append(group.Missing, "匹配冲突 "+requirement.Name) + } + } + group.SourcePrefix = group.Prefix + for _, object := range rootObjects { + group.SourceKeys = append(group.SourceKeys, object.Key) + } + result = append(result, group) + continue + } + mixed := false + for _, object := range rootObjects { + name := strings.TrimPrefix(object.Key, parent.Prefix+"/") + for _, requirement := range defaultOrbbecCaptureRequirements[:5] { + if regexp.MustCompile(requirement.Pattern).MatchString(name) { + mixed = true + } + } + } + if mixed { + parent.Missing = []string{"目录结构冲突:根目录数据与 part 数据不能混合接入"} + result = append(result, parent) + continue + } + names := make([]string, 0, len(parts)) + for name := range parts { + names = append(names, name) + } + sort.Strings(names) + for _, part := range names { + group := captureGroup{CaptureID: parent.CaptureID + "/" + part, Prefix: parent.Prefix + "/" + part, SourcePrefix: parent.Prefix} + captureName := regexp.QuoteMeta(parent.CaptureID) + partName := regexp.QuoteMeta(part) + requirements := []captureRequirement{ + {"左相机视频", `(?i)^` + captureName + `_camera_left_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`}, + {"右相机视频", `(?i)^` + captureName + `_camera_right_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`}, + {"左相机 PTS", `(?i)^` + captureName + `_camera_left_` + partName + `_pts\.csv$`}, + {"右相机 PTS", `(?i)^` + captureName + `_camera_right_` + partName + `_pts\.csv$`}, + {"IMU", `(?i)^` + captureName + `_imu_` + partName + `\.csv$`}, + {"相机标定", `(?i)^` + captureName + `_calibration_camera\.yaml$`}, + {"IMU 标定", `(?i)^` + captureName + `_calibration_imu\.yaml$`}, + } + for i, requirement := range requirements { + candidates := parts[part] + base := group.Prefix + "/" + if i >= 5 { + candidates = rootObjects + base = parent.Prefix + "/" + } + re := regexp.MustCompile(requirement.Pattern) + // Prefer PTS explicitly named for the selected video variant. Raw + // part PTS is still discoverable, but is not proof of transcode alignment. + if i == 2 || i == 3 { + videoRE := regexp.MustCompile(requirements[i-2].Pattern) + for _, selected := range group.Objects { + name := strings.TrimPrefix(selected.Key, group.Prefix+"/") + if !videoRE.MatchString(name) { + continue + } + stem := name[:strings.LastIndex(name, ".")] + exact := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(stem+"_pts.csv") + `$`) + for _, candidate := range candidates { + if exact.MatchString(strings.TrimPrefix(candidate.Key, base)) { + re = exact + break + } + } + } + } + matches := []captureObject{} + for _, object := range candidates { + if re.MatchString(strings.TrimPrefix(object.Key, base)) { + matches = append(matches, object) + } + } + if len(matches) == 0 { + group.Missing = append(group.Missing, part+" 缺少 "+requirement.Name) + } + if len(matches) > 1 { + group.Missing = append(group.Missing, part+" 匹配冲突 "+requirement.Name) + } + for _, object := range matches { + if object.Size <= 0 { + group.Missing = append(group.Missing, "空文件 "+object.Key) + } + group.Objects = append(group.Objects, object) + } + } + optional := regexp.MustCompile(`(?i)^` + captureName + `_audio_` + partName + `(?:_pts\.csv|\.wav)$`) + for _, object := range parts[part] { + if optional.MatchString(strings.TrimPrefix(object.Key, group.Prefix+"/")) { + group.Objects = append(group.Objects, object) + } + } + for _, object := range rootObjects { + if strings.EqualFold(object.Key, parent.Prefix+"/meta_"+parent.CaptureID+".json") { + group.Objects = append(group.Objects, object) + } + } + // Custom requirements are additional constraints; never bypass part isolation. + for _, pattern := range config.RequiredPatterns { + if strings.TrimSpace(pattern) == "" { + continue + } + re := regexp.MustCompile(pattern) + found := false + for _, object := range group.Objects { + if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) { + found = true + } + } + if !found { + group.Missing = append(group.Missing, part+" 缺少自定义必需数据 "+pattern) + } + } + for _, object := range group.Objects { + group.SourceKeys = append(group.SourceKeys, object.Key) + group.TotalBytes += object.Size + } + sort.Strings(group.SourceKeys) + group.Fingerprint = captureFingerprint(group.Objects) + result = append(result, group) + } + } + return result, nil +} diff --git a/internal/worker/capture_layout_test.go b/internal/worker/capture_layout_test.go new file mode 100644 index 00000000..8f652674 --- /dev/null +++ b/internal/worker/capture_layout_test.go @@ -0,0 +1,138 @@ +package worker + +import ( + "droboticflow/internal/domain" + "strings" + "testing" +) + +const layoutCapture = "FYLX_B10_AZEC86400GP_19700101_035516" + +func partObjects(part string) []captureObject { + items := []captureObject{} + for _, suffix := range []string{"camera_left_" + part + "_silent_480p_wm.mp4", "camera_right_" + part + "_silent_480p_wm.mp4", "camera_left_" + part + "_pts.csv", "camera_right_" + part + "_pts.csv", "imu_" + part + ".csv"} { + items = append(items, captureObject{Key: "incoming/" + layoutCapture + "/" + part + "/" + layoutCapture + "_" + suffix, Size: 10, ETag: "original"}) + } + return items +} + +func calibrationObjects() []captureObject { + return []captureObject{{Key: "incoming/" + layoutCapture + "/" + layoutCapture + "_calibration_camera.yaml", Size: 2}, {Key: "incoming/" + layoutCapture + "/" + layoutCapture + "_calibration_imu.yaml", Size: 2}} +} + +func discoverLayout(t *testing.T, objects []captureObject) []captureGroup { + t.Helper() + groups, err := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"})) + if err != nil { + t.Fatal(err) + } + return groups +} + +func TestPartCaptureIndependentGroupsAndFingerprint(t *testing.T) { + objects := append(calibrationObjects(), partObjects("part0001")...) + objects = append(objects, partObjects("part0002")...) + groups := discoverLayout(t, objects) + if len(groups) != 2 { + t.Fatal(groups) + } + for i, g := range groups { + part := []string{"part0001", "part0002"}[i] + if len(g.Missing) != 0 || len(g.SourceKeys) != 7 || g.CaptureID != layoutCapture+"/"+part || g.SourcePrefix != "incoming/"+layoutCapture { + t.Fatalf("bad unit: %+v", g) + } + for _, key := range g.SourceKeys { + if strings.Contains(key, "part") && !strings.Contains(key, part) { + t.Fatal("cross-part reference", key) + } + } + } + objects[len(objects)-1].ETag = "changed" + next := discoverLayout(t, objects) + if groups[0].Fingerprint != next[0].Fingerprint || groups[1].Fingerprint == next[1].Fingerprint { + t.Fatal("fingerprint not isolated") + } + objects[0].ETag = "new calibration" + if discoverLayout(t, objects)[0].Fingerprint == next[0].Fingerprint { + t.Fatal("shared calibration excluded from fingerprint") + } + if captureRecordID("workflow", groups[0].Prefix) == captureRecordID("workflow", groups[1].Prefix) { + t.Fatal("part identities collide") + } +} + +func TestPartCaptureMissingEmptyConflictAndWrongIdentity(t *testing.T) { + for _, kind := range []string{"missing", "empty", "wrong-part", "wrong-capture", "duplicate", "missing-calibration"} { + t.Run(kind, func(t *testing.T) { + objects := append(calibrationObjects(), partObjects("part0001")...) + switch kind { + case "missing": + objects = objects[:len(objects)-1] + case "empty": + objects[len(objects)-1].Size = 0 + case "wrong-part": + objects[len(objects)-1].Key = strings.Replace(objects[len(objects)-1].Key, "_imu_part0001", "_imu_part0002", 1) + case "wrong-capture": + objects[len(objects)-1].Key = "incoming/" + layoutCapture + "/part0001/other_imu_part0001.csv" + case "duplicate": + objects = append(objects, captureObject{Key: "incoming/" + layoutCapture + "/part0001/" + layoutCapture + "_camera_left_part0001.mp4", Size: 10}) + case "missing-calibration": + objects = objects[1:] + } + objects = append(objects, partObjects("part0002")...) + groups := discoverLayout(t, objects) + if len(groups) != 2 || len(groups[0].Missing) == 0 { + t.Fatal("invalid part admitted", groups) + } + if kind != "missing-calibration" && len(groups[1].Missing) != 0 { + t.Fatal("other part blocked", groups[1]) + } + }) + } +} + +func TestPartCaptureMixedAndLegacyLayouts(t *testing.T) { + legacy := completeCaptureObjects("incoming", "Orbbec_Ego_legacy") + groups := discoverLayout(t, legacy) + if len(groups) != 1 || len(groups[0].Missing) != 0 || groups[0].Prefix != "incoming/Orbbec_Ego_legacy" { + t.Fatal(groups) + } + mixed := append(completeCaptureObjects("incoming", layoutCapture), partObjects("part0001")...) + groups = discoverLayout(t, mixed) + if len(groups) != 1 || !strings.Contains(strings.Join(groups[0].Missing, ""), "结构冲突") { + t.Fatal(groups) + } +} + +func TestPartCaptureHistoricalParentGuard(t *testing.T) { + for _, record := range []*domain.CaptureRecord{{RunID: "old", Status: domain.CaptureRecordFailed}, {RunID: "old", Status: domain.CaptureRecordCompleted}, {Status: domain.CaptureRecordDiscarded}, {Status: domain.CaptureRecordLaunching}} { + if !preserveCaptureParent(record) { + t.Fatal("historical parent may relaunch", record) + } + } + if preserveCaptureParent(nil) || preserveCaptureParent(&domain.CaptureRecord{Status: domain.CaptureRecordIncomplete}) { + t.Fatal("unadmitted parent cannot upgrade") + } +} + +func TestPartCapturePrefersVideoSpecificPTS(t *testing.T) { + objects := append(calibrationObjects(), partObjects("part0001")...) + key := "incoming/" + layoutCapture + "/part0001/" + layoutCapture + "_camera_left_part0001_silent_480p_wm_pts.csv" + objects = append(objects, captureObject{Key: key, Size: 10}) + groups := discoverLayout(t, objects) + if len(groups) != 1 || len(groups[0].Missing) != 0 { + t.Fatal(groups) + } + found := false + for _, selected := range groups[0].SourceKeys { + if selected == key { + found = true + } + if strings.HasSuffix(selected, "_camera_left_part0001_pts.csv") { + t.Fatal("raw PTS selected despite video-specific PTS") + } + } + if !found { + t.Fatal("video-specific PTS missing") + } +} diff --git a/operator-repos/builtin/directory_scan/src/operator.py b/operator-repos/builtin/directory_scan/src/operator.py index f5b4aa65..4359851e 100644 --- a/operator-repos/builtin/directory_scan/src/operator.py +++ b/operator-repos/builtin/directory_scan/src/operator.py @@ -65,6 +65,7 @@ def _bos_entries( pattern: str, params: dict[str, str], helpers: HelperMap, + object_keys: list[str] | None = None, ) -> list[dict[str, Any]]: endpoint = str(params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "")).strip() access_key_id = str(params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "")).strip() @@ -84,6 +85,15 @@ def _bos_entries( for item in listed_entries if "/_droboticflow/manifests/" not in "/" + str(item.get("path", "")).lstrip("/") ] + if object_keys is not None: + base = prefix.rstrip("/") + "/" + if not object_keys or any(not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in object_keys): + raise RuntimeError("目录文件选择超出 Capture 范围或为空") + selected = {key[len(base):] for key in object_keys} + entries = [item for item in entries if str(item.get("path", "")).lstrip("/") in selected] + missing = selected - {str(item.get("path", "")).lstrip("/") for item in entries} + if missing: + raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}") if not entries: raise RuntimeError(f"目录读取未在 bos://{bucket}/{prefix} 找到真实文件") client, _ = helpers["build_bos_client"](endpoint, access_key_id, secret_access_key) @@ -122,7 +132,8 @@ def _run_legacy( directory_ref = _directory_ref(inputs) if directory_ref is not None: bucket, prefix = directory_ref - entries = _bos_entries(bucket, prefix, pattern, params, helpers) + selected_ref = (_directory_input(inputs) or {}).get("payload", {}).get("ref", {}) + entries = _bos_entries(bucket, prefix, pattern, params, helpers, selected_ref.get("objectKeys")) source_path = helpers["canonical_bos_path"](bucket, prefix) payload = { "schemaVersion": "droboticflow.directory-read/v1", @@ -248,7 +259,7 @@ def _run_multiport( bucket = str(ref.get("bucket", "")).strip() prefix = str(ref.get("prefix", "")).strip().strip("/") if bucket: - entries = _bos_entries(bucket, prefix, pattern, params, helpers) + entries = _bos_entries(bucket, prefix, pattern, params, helpers, ref.get("objectKeys")) root_path = str(helpers["canonical_bos_path"](bucket, prefix)) else: root_path = str(directory_input.get("path", "")).strip() diff --git a/operator-repos/builtin/orbbec_data_normalize/src/operator.py b/operator-repos/builtin/orbbec_data_normalize/src/operator.py index 023308f1..180bf71e 100644 --- a/operator-repos/builtin/orbbec_data_normalize/src/operator.py +++ b/operator-repos/builtin/orbbec_data_normalize/src/operator.py @@ -18,7 +18,12 @@ def _entry_path(entry: dict[str, Any]) -> str: def _capture_id(path: str) -> str: - matched = CAPTURE_PATTERN.search(path) + # The Capture name is not required to contain a date (legacy business names). + stem = pathlib.PurePosixPath(path).stem + named = re.match(r"^(.+?)_(?:camera_(?:left|right)_part\d+(?:_.*)?|imu(?:_part\d+)?|audio(?:_part\d+)?(?:_pts)?|calibration_(?:camera|imu))$", stem, re.IGNORECASE) + if named: + return named.group(1) + matched = CAPTURE_PATTERN.search(path) or CAPTURE_PATTERN.search(pathlib.PurePosixPath(path).name) if matched: return matched.group("capture") parts = pathlib.PurePosixPath(path).parts @@ -33,13 +38,17 @@ def _camera_side(path: str) -> str: def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, str]: """Match sidecars within one Capture and segment, never the first segment.""" video = pathlib.PurePosixPath(video_path) - part = re.match(r"^(?P.+)_camera_(?:left|right)_(?Ppart\d+)(?P.*)$", video.stem) + part = re.match(r"^(?P.+)_camera_(?:left|right)_(?Ppart\d+)(?P.*)$", video.stem, re.IGNORECASE) def find(name: str) -> str: candidates = [_entry_path(item) for item in entries - if pathlib.PurePosixPath(_entry_path(item)).name == name] + if pathlib.PurePosixPath(_entry_path(item)).name.lower() == name.lower()] local = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent] - matches = local or candidates + # Only calibration files may live at the Capture root above this part. + shared = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent.parent] + matches = local or (shared if "_calibration_" in name else []) + if candidates and not matches: + raise RuntimeError(f"配套文件不唯一或不在当前 part: {video_path} -> {name}") if len(matches) > 1: raise RuntimeError(f"配套文件不唯一: {video_path} -> {name}: {matches}") return matches[0] if matches else "" @@ -59,7 +68,7 @@ def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, s "imu": f"{capture}_imu_{segment}.csv"} result = {key: find(name) for key, name in names.items()} for key, suffix in (("audio", "audio.wav"), ("audioPts", "audio_pts.csv"), ("imu", "imu.csv")): - if not result[key]: + if not result[key] and not re.fullmatch(r"part\d+", video.parent.name, re.IGNORECASE): result[key] = find(f"{capture}_{suffix}") result.update({"cameraCalibration": find(f"{capture}_calibration_camera.yaml"), "imuCalibration": find(f"{capture}_calibration_imu.yaml")}) @@ -204,6 +213,16 @@ def _directory_entries(inputs: list[dict[str, Any]], helpers: HelperMap) -> tupl if not endpoint or not access_key_id or not secret_access_key: raise RuntimeError("Orbbec 多端口标准化需要目录引用对应的 BOS 凭证") entries = helpers["list_bos_entries"](bucket, prefix, "", endpoint, access_key_id, secret_access_key) + if "objectKeys" in ref: + base = prefix.rstrip("/") + "/" + keys = ref["objectKeys"] + if not isinstance(keys, list) or not keys or any(not isinstance(key, str) or not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in keys): + raise RuntimeError("目录文件选择超出 Capture 范围或为空") + selected = {key[len(base):] for key in keys} + entries = [item for item in entries if _entry_path(item) in selected] + missing = selected - {_entry_path(item) for item in entries} + if missing: + raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}") source_path = str(helpers["canonical_bos_path"](bucket, prefix)) return [ { diff --git a/tests/backend/test_capture_part_selection.py b/tests/backend/test_capture_part_selection.py new file mode 100644 index 00000000..d9e3d7f9 --- /dev/null +++ b/tests/backend/test_capture_part_selection.py @@ -0,0 +1,80 @@ +import pytest + +from backend.operators.builtin.directory_scan import operator as directory +from backend.operators.builtin.orbbec_data_normalize import operator as normalize + +C = "FYLX_B10_AZEC86400GP_19700101_035516" +PREFIX = "incoming/" + C + + +def test_part_pairing_shares_only_parent_calibration(): + video = f"part0001/{C}_camera_left_part0001.mp4" + paths = [video, f"part0001/{C}_camera_left_part0001_pts.csv", + f"part0001/{C}_imu_part0001.csv", f"{C}_calibration_camera.yaml", + f"{C}_calibration_imu.yaml"] + entries = [{"path": path} for path in paths] + result = normalize._paired_files(video, entries) + assert result["pts"] == paths[1] + assert result["imu"] == paths[2] + assert result["cameraCalibration"] == paths[3] + assert result["imuCalibration"] == paths[4] + assert normalize._capture_id(video) == C + assert normalize._capture_id(paths[3]) == C + entries[2]["path"] = f"part0002/{C}_imu_part0001.csv" + with pytest.raises(RuntimeError, match="不在当前 part"): + normalize._paired_files(video, entries) + + +def mock_helpers(monkeypatch): + for key in ("BOS_ENDPOINT", "BOS_ACCESS_KEY_ID", "BOS_SECRET_ACCESS_KEY"): + monkeypatch.setenv(key, "isolated-test") + paths = [f"part0001/{C}_camera_left_part0001_pts.csv", + f"part0002/{C}_camera_left_part0002_pts.csv", f"{C}_calibration_camera.yaml"] + reads = [] + + class Client: + def get_object_as_string(self, bucket, key): + reads.append(key) + assert "part0002" not in key + return b"timestamp_us\n1\n2\n" + + helpers = { + "list_bos_entries": lambda *args: [{"path": path, "sizeBytes": 20} for path in paths], + "build_bos_client": lambda *args: (Client(), None), + "canonical_bos_path": lambda bucket, prefix: f"bos://{bucket}/{prefix}", + } + return paths, reads, helpers + + +def test_directory_and_normalizer_honor_same_frozen_selection(monkeypatch): + paths, reads, helpers = mock_helpers(monkeypatch) + selected = [PREFIX + "/" + paths[i] for i in (0, 2)] + entries = directory._bos_entries("bucket", PREFIX, "", {}, helpers, selected) + assert {entry["path"] for entry in entries} == {paths[0], paths[2]} + assert reads == [selected[0]] + inputs = [{"payload": {"ref": {"kind": "directory", "bucket": "bucket", "prefix": PREFIX, "objectKeys": selected}}}] + entries, source = normalize._directory_entries(inputs, helpers) + assert {entry["path"] for entry in entries} == {paths[0], paths[2]} + assert source == "bos://bucket/" + PREFIX + + +@pytest.mark.parametrize("keys", [[], ["other/capture.csv"], [PREFIX + "/../other.csv"], [PREFIX + "/missing.csv"]]) +def test_missing_or_out_of_scope_selected_files_fail_closed(monkeypatch, keys): + _, _, helpers = mock_helpers(monkeypatch) + with pytest.raises(RuntimeError): + directory._bos_entries("bucket", PREFIX, "", {}, helpers, keys) + inputs = [{"payload": {"ref": {"kind": "directory", "bucket": "bucket", "prefix": PREFIX, "objectKeys": keys}}}] + with pytest.raises(RuntimeError): + normalize._directory_entries(inputs, helpers) + + +def test_current_transcoded_filename_does_not_bypass_pts_safety(): + video = f"part0001/{C}_camera_left_part0001_silent_480p_wm.mp4" + entries = [{"path": f"part0001/{C}_camera_left_part0001_pts.csv"}] + with pytest.raises(RuntimeError, match="转码视频缺少逐帧对应的 PTS"): + normalize._paired_files(video, entries) + + +def test_arbitrary_capture_names_keep_media_and_calibration_together(): + for path in ("part0001/business_camera_left_part0001.mp4", "business_calibration_camera.yaml", "part0001/business_imu_part0001.csv"): + assert normalize._capture_id(path) == "business"