✨ Support legacy and isolated part Capture admission

This commit is contained in:
eust-w 2026-09-10 20:46:38 +08:00
parent 22b16c8663
commit cd5bfdb409
15 changed files with 616 additions and 37 deletions

View File

@ -65,6 +65,7 @@ def _bos_entries(
pattern: str,
params: dict[str, str],
helpers: HelperMap,
object_keys: list[str] | None = None,
) -> list[dict[str, Any]]:
endpoint = str(params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "")).strip()
access_key_id = str(params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "")).strip()
@ -84,6 +85,15 @@ def _bos_entries(
for item in listed_entries
if "/_droboticflow/manifests/" not in "/" + str(item.get("path", "")).lstrip("/")
]
if object_keys is not None:
base = prefix.rstrip("/") + "/"
if not object_keys or any(not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in object_keys):
raise RuntimeError("目录文件选择超出 Capture 范围或为空")
selected = {key[len(base):] for key in object_keys}
entries = [item for item in entries if str(item.get("path", "")).lstrip("/") in selected]
missing = selected - {str(item.get("path", "")).lstrip("/") for item in entries}
if missing:
raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}")
if not entries:
raise RuntimeError(f"目录读取未在 bos://{bucket}/{prefix} 找到真实文件")
client, _ = helpers["build_bos_client"](endpoint, access_key_id, secret_access_key)
@ -122,7 +132,8 @@ def _run_legacy(
directory_ref = _directory_ref(inputs)
if directory_ref is not None:
bucket, prefix = directory_ref
entries = _bos_entries(bucket, prefix, pattern, params, helpers)
selected_ref = (_directory_input(inputs) or {}).get("payload", {}).get("ref", {})
entries = _bos_entries(bucket, prefix, pattern, params, helpers, selected_ref.get("objectKeys"))
source_path = helpers["canonical_bos_path"](bucket, prefix)
payload = {
"schemaVersion": "droboticflow.directory-read/v1",
@ -248,7 +259,7 @@ def _run_multiport(
bucket = str(ref.get("bucket", "")).strip()
prefix = str(ref.get("prefix", "")).strip().strip("/")
if bucket:
entries = _bos_entries(bucket, prefix, pattern, params, helpers)
entries = _bos_entries(bucket, prefix, pattern, params, helpers, ref.get("objectKeys"))
root_path = str(helpers["canonical_bos_path"](bucket, prefix))
else:
root_path = str(directory_input.get("path", "")).strip()

View File

@ -18,7 +18,12 @@ def _entry_path(entry: dict[str, Any]) -> str:
def _capture_id(path: str) -> str:
matched = CAPTURE_PATTERN.search(path)
# The Capture name is not required to contain a date (legacy business names).
stem = pathlib.PurePosixPath(path).stem
named = re.match(r"^(.+?)_(?:camera_(?:left|right)_part\d+(?:_.*)?|imu(?:_part\d+)?|audio(?:_part\d+)?(?:_pts)?|calibration_(?:camera|imu))$", stem, re.IGNORECASE)
if named:
return named.group(1)
matched = CAPTURE_PATTERN.search(path) or CAPTURE_PATTERN.search(pathlib.PurePosixPath(path).name)
if matched:
return matched.group("capture")
parts = pathlib.PurePosixPath(path).parts
@ -33,13 +38,17 @@ def _camera_side(path: str) -> str:
def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, str]:
"""Match sidecars within one Capture and segment, never the first segment."""
video = pathlib.PurePosixPath(video_path)
part = re.match(r"^(?P<capture>.+)_camera_(?:left|right)_(?P<part>part\d+)(?P<suffix>.*)$", video.stem)
part = re.match(r"^(?P<capture>.+)_camera_(?:left|right)_(?P<part>part\d+)(?P<suffix>.*)$", video.stem, re.IGNORECASE)
def find(name: str) -> str:
candidates = [_entry_path(item) for item in entries
if pathlib.PurePosixPath(_entry_path(item)).name == name]
if pathlib.PurePosixPath(_entry_path(item)).name.lower() == name.lower()]
local = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent]
matches = local or candidates
# Only calibration files may live at the Capture root above this part.
shared = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent.parent]
matches = local or (shared if "_calibration_" in name else [])
if candidates and not matches:
raise RuntimeError(f"配套文件不唯一或不在当前 part: {video_path} -> {name}")
if len(matches) > 1:
raise RuntimeError(f"配套文件不唯一: {video_path} -> {name}: {matches}")
return matches[0] if matches else ""
@ -59,7 +68,7 @@ def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, s
"imu": f"{capture}_imu_{segment}.csv"}
result = {key: find(name) for key, name in names.items()}
for key, suffix in (("audio", "audio.wav"), ("audioPts", "audio_pts.csv"), ("imu", "imu.csv")):
if not result[key]:
if not result[key] and not re.fullmatch(r"part\d+", video.parent.name, re.IGNORECASE):
result[key] = find(f"{capture}_{suffix}")
result.update({"cameraCalibration": find(f"{capture}_calibration_camera.yaml"),
"imuCalibration": find(f"{capture}_calibration_imu.yaml")})
@ -204,6 +213,16 @@ def _directory_entries(inputs: list[dict[str, Any]], helpers: HelperMap) -> tupl
if not endpoint or not access_key_id or not secret_access_key:
raise RuntimeError("Orbbec 多端口标准化需要目录引用对应的 BOS 凭证")
entries = helpers["list_bos_entries"](bucket, prefix, "", endpoint, access_key_id, secret_access_key)
if "objectKeys" in ref:
base = prefix.rstrip("/") + "/"
keys = ref["objectKeys"]
if not isinstance(keys, list) or not keys or any(not isinstance(key, str) or not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in keys):
raise RuntimeError("目录文件选择超出 Capture 范围或为空")
selected = {key[len(base):] for key in keys}
entries = [item for item in entries if _entry_path(item) in selected]
missing = selected - {_entry_path(item) for item in entries}
if missing:
raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}")
source_path = str(helpers["canonical_bos_path"](bucket, prefix))
return [
{

View File

@ -0,0 +1,20 @@
# Capture layout compatibility
- Flat legacy directories keep the existing filename requirements and one Run per directory. Nested files cannot satisfy root requirements. Multiple candidates for a required type block admission rather than selecting the first file.
- A direct `part[0-9]+` child selects segmented layout. Each part is an independent unit, identified by workflow + parent prefix + part. Capture names are not restricted to FYLX or a date pattern.
- Part video names must contain the actual Capture name, camera side and actual part name. Original `.mp4/.mov/.mkv` and suffixed variants such as `_silent_480p_wm.mp4` are recognized. Multiple variants block intake as ambiguous.
- Part PTS and IMU names must match the Capture and part exactly. Video-specific `<video-stem>_pts.csv` takes precedence over raw part PTS when present. Only the parent's two `<capture>_calibration_camera.yaml` / `<capture>_calibration_imu.yaml` files are shared. Audio and meta are optional.
- Root sensor data plus part directories is a structural conflict. Missing/empty files block only their unit; missing shared calibration blocks all affected parts. Custom required patterns remain supported for flat layouts; for part layouts they add constraints within the selected unit, never permit cross-part matching.
- A directory reference records the parent `prefix`, explicit `objectKeys` and display `captureUnit`. Both directory reading and normalization enforce the selection, including on retry. Missing selected files fail closed. Each part's fingerprint includes only its selected files and shared calibration/meta.
- Existing admitted parent records retain their Run association and are not split/relaunched by scanning. Discarded or launching parents also prevent child admissions. No completion manifest is required; non-empty required files and the existing stability window still apply.
- Once a part has been admitted, the parent cannot later be admitted as a flat unit by moving/removing source files.
## Data-quality boundary
Filename compatibility does not prove frame correspondence. In particular, a transcoded `_silent_480p_wm.mp4` plus raw `_part0001_pts.csv` is recognized by intake but remains subject to the existing normalizer rejection when a matching transformed-video PTS/frame mapping is absent. This change must not invent timestamps or silently bypass that safety check.
## Validation
Go tests cover flat/part grouping, independent identities/fingerprints, shared calibration, missing/empty/wrong-part files, duplicate videos, mixed layouts, historical-parent protection, Run binding/de-duplication and retry references. Python tests cover scope enforcement in both readers, parent calibration pairing and unchanged PTS safety.
Deploy worker, API and both directory-scan / Orbbec-normalization operator images together. Do not enable a new scanner with old operators that ignore `objectKeys`. No live deployment or business replay is performed by unit tests.

View File

@ -641,6 +641,11 @@ func captureReprocessInput(workflow *domain.WorkflowRecord, record *domain.Captu
Provider: domain.StorageProviderBOS, ConnectionID: config.ConnectionID, Bucket: config.Bucket,
Prefix: record.Prefix, VersionID: record.Fingerprint, Kind: domain.ArtifactRefKindDirectory,
}
if record.SourcePrefix != "" {
ref.Prefix = record.SourcePrefix
ref.ObjectKeys = append([]string(nil), record.SourceKeys...)
ref.CaptureUnit = record.CaptureID
}
return map[string]domain.ArtifactPayload{
config.WorkflowInputID: {Mode: domain.ArtifactTransferModeRef, Type: input.Type, Ref: ref},
}, nil

View File

@ -0,0 +1,23 @@
package api
import (
"droboticflow/internal/domain"
"testing"
)
func TestCapturePartReprocessPreservesSelectedFiles(t *testing.T) {
workflow := &domain.WorkflowRecord{CaptureAutomation: &domain.CaptureAutomationConfig{ConnectionID: "connection", Bucket: "bucket", WorkflowInputID: "sourceDirectory"}, WorkflowInputs: []domain.WorkflowInputDefinition{{ID: "sourceDirectory", Type: domain.PortTypeFile}}}
record := &domain.CaptureRecord{Prefix: "incoming/capture/part0001", SourcePrefix: "incoming/capture", CaptureID: "capture/part0001", SourceKeys: []string{"incoming/capture/part0001/left.mp4", "incoming/capture/calibration_camera.yaml"}, Fingerprint: "snapshot"}
inputs, err := captureReprocessInput(workflow, record)
if err != nil {
t.Fatal(err)
}
ref := inputs["sourceDirectory"].Ref
if ref.Prefix != record.SourcePrefix || ref.CaptureUnit != record.CaptureID || len(ref.ObjectKeys) != 2 {
t.Fatalf("selection lost during retry: %+v", ref)
}
record.SourceKeys[0] = "changed"
if ref.ObjectKeys[0] == "changed" {
t.Fatal("retry input aliases mutable record")
}
}

View File

@ -131,6 +131,10 @@ func CaptureDirectoriesFromRun(run RunRecord) []string {
if input.Ref == nil {
continue
}
if input.Ref.CaptureUnit != "" {
values = append(values, input.Ref.CaptureUnit)
continue
}
for _, candidate := range []string{input.Ref.Prefix, input.Ref.Key, input.Ref.ManifestKey} {
if directory := captureDirectoryFromDirectoryValue(candidate); directory != "" {
values = append(values, directory)

View File

@ -57,6 +57,8 @@ type CaptureActionRecord struct {
}
type CaptureRecord struct {
SourcePrefix string `json:"sourcePrefix,omitempty" bson:"sourcePrefix,omitempty"`
SourceKeys []string `json:"sourceKeys,omitempty" bson:"sourceKeys,omitempty"`
ID string `json:"id" bson:"id"`
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
ProjectID string `json:"projectId" bson:"projectId"`

View File

@ -210,6 +210,8 @@ const (
// ArtifactReference is a credential-free pointer to durable object storage.
type ArtifactReference struct {
ObjectKeys []string `json:"objectKeys,omitempty" bson:"objectKeys,omitempty"`
CaptureUnit string `json:"captureUnit,omitempty" bson:"captureUnit,omitempty"`
Provider StorageProvider `json:"provider" bson:"provider"`
ConnectionID string `json:"connectionId" bson:"connectionId"`
Bucket string `json:"bucket" bson:"bucket"`

View File

@ -30,13 +30,15 @@ type captureObject struct {
}
type captureGroup struct {
CaptureID string
Prefix string
ManifestKey string
Fingerprint string
Missing []string
Objects []captureObject
TotalBytes int64
SourcePrefix string
SourceKeys []string
CaptureID string
Prefix string
ManifestKey string
Fingerprint string
Missing []string
Objects []captureObject
TotalBytes int64
}
type captureRequirement struct {
@ -122,7 +124,7 @@ func captureAutomationFingerprint(workflowID string, config domain.CaptureAutoma
return hex.EncodeToString(sum[:])
}
func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
func discoverFlatCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
requirements := captureRequirements(config)
compiledRequirements := make([]*regexp.Regexp, 0, len(requirements))
for _, requirement := range requirements {
@ -462,6 +464,11 @@ func (runtime *Runtime) launchCaptureRun(ctx context.Context, workflow domain.Wo
VersionID: record.Fingerprint, Kind: domain.ArtifactRefKindDirectory,
},
}
if record.SourcePrefix != "" {
payload.Ref.Prefix = record.SourcePrefix
payload.Ref.ObjectKeys = append([]string(nil), record.SourceKeys...)
payload.Ref.CaptureUnit = record.CaptureID
}
run, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerScheduled, map[string]domain.ArtifactPayload{config.WorkflowInputID: payload}, record.ID)
if err != nil {
record.Status = domain.CaptureRecordReady
@ -530,8 +537,37 @@ func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow d
if err != nil {
return err
}
priorRecords, err := runtime.store.ListCaptureRecords(ctx, workflow.OrganizationID, workflow.ID, 0)
if err != nil {
return err
}
admittedPartParents := map[string]bool{}
for _, prior := range priorRecords {
if prior.SourcePrefix != "" && prior.SourcePrefix != prior.Prefix && preserveCaptureParent(&prior) {
admittedPartParents[prior.SourcePrefix] = true
}
}
newCount := 0
historicalParents := map[string]bool{}
for _, group := range groups {
if admittedPartParents[group.Prefix] {
group.Missing = append(group.Missing, "该目录已有 part 接入记录,不能重新作为整目录接入")
}
if group.SourcePrefix != "" && group.SourcePrefix != group.Prefix {
parent, e := runtime.store.GetCaptureRecordByWorkflowPrefix(ctx, workflow.ID, group.SourcePrefix)
if e != nil {
return e
}
if preserveCaptureParent(parent) {
if !historicalParents[parent.ID] && parent.RunID != "" {
if e := runtime.reconcileCaptureRun(ctx, parent); e != nil {
return e
}
}
historicalParents[parent.ID] = true
continue
}
}
existing, err := runtime.store.GetCaptureRecordByWorkflowPrefix(ctx, workflow.ID, group.Prefix)
if err != nil {
return err
@ -564,6 +600,7 @@ func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow d
missing := append([]string(nil), group.Missing...)
nowText := nowISO()
record := domain.CaptureRecord{
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys,
ID: captureRecordID(workflow.ID, group.Prefix), OrganizationID: workflow.OrganizationID,
ProjectID: workflow.ProjectID, WorkflowID: workflow.ID, ConnectionID: config.ConnectionID,
Bucket: config.Bucket, Prefix: group.Prefix, CaptureID: group.CaptureID,

View File

@ -2,6 +2,7 @@ package worker
import (
"context"
"regexp"
"testing"
"time"
@ -54,20 +55,8 @@ func TestCaptureIMURecognizesLegacyAndSegmentNames(t *testing.T) {
{"notimu.csv", false},
} {
t.Run(tc.name, func(t *testing.T) {
objects := completeCaptureObjects("incoming", "capture")
objects[4].Key = "incoming/capture/" + tc.name
groups, err := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"}))
if err != nil || len(groups) != 1 {
t.Fatalf("groups=%v err=%v", groups, err)
}
missing := false
for _, name := range groups[0].Missing {
if name == "IMU" {
missing = true
}
}
if missing == tc.valid {
t.Fatalf("name=%s valid=%v missing=%v", tc.name, tc.valid, groups[0].Missing)
if regexp.MustCompile(defaultOrbbecCaptureRequirements[4].Pattern).MatchString(tc.name) != tc.valid {
t.Fatalf("unexpected IMU filename match: %s", tc.name)
}
})
}
@ -84,7 +73,7 @@ func TestCaptureSegmentIMUStillRejectsEmptyAndDoesNotFillOtherCapture(t *testing
if err != nil || len(groups) != 2 {
t.Fatalf("groups=%v err=%v", groups, err)
}
if len(groups[0].Missing) != 1 || groups[0].Missing[0] != "空文件 part0001/a_imu_part0001.csv" {
if len(groups[0].Missing) != 1 || groups[0].Missing[0] != "目录结构冲突:根目录数据与 part 数据不能混合接入" {
t.Fatalf("empty IMU accepted: %v", groups[0].Missing)
}
if len(groups[1].Missing) != 1 || groups[1].Missing[0] != "IMU" {
@ -341,6 +330,37 @@ func TestLaunchCaptureRunBindsOneCompleteDirectory(t *testing.T) {
if input.Ref == nil || input.Ref.Prefix != record.Prefix || input.Ref.ManifestKey != "" || input.Ref.Kind != domain.ArtifactRefKindDirectory {
t.Fatalf("unexpected directory binding: %#v", input)
}
// New part units retain the parent source root plus an explicit file selection.
part := record
part.ID = "capture-record-part-1"
part.RunID = ""
part.Status = domain.CaptureRecordReady
part.SourcePrefix = record.Prefix
part.Prefix = record.Prefix + "/part0001"
part.CaptureID = record.CaptureID + "/part0001"
part.SourceKeys = []string{part.Prefix + "/video.mp4", record.Prefix + "/calibration_camera.yaml"}
if err := dataStore.UpsertCaptureRecord(ctx, part); err != nil {
t.Fatal(err)
}
if err := runtime.launchCaptureRun(ctx, *workflow, config, &part); err != nil {
t.Fatal(err)
}
partRun, err := dataStore.GetRun(ctx, part.RunID)
if err != nil || partRun == nil {
t.Fatal(err)
}
ref := partRun.Inputs["sourceDirectory"].Ref
if ref.Prefix != record.Prefix || len(ref.ObjectKeys) != 2 || ref.CaptureUnit != part.CaptureID {
t.Fatalf("part selection lost: %+v", ref)
}
if len(partRun.CaptureDirectories) != 1 || partRun.CaptureDirectories[0] != part.CaptureID {
t.Fatal(partRun.CaptureDirectories)
}
firstRunID := part.RunID
part.Status = domain.CaptureRecordReady
if err := runtime.launchCaptureRun(ctx, *workflow, config, &part); err != nil || part.RunID != firstRunID {
t.Fatal("duplicate part launch", err)
}
}
func TestNormalizedCaptureAutomationDefaultsToThirtySecondStabilityWindow(t *testing.T) {

View File

@ -0,0 +1,188 @@
package worker
import (
"regexp"
"sort"
"strings"
"droboticflow/internal/domain"
)
var capturePartDirectory = regexp.MustCompile(`(?i)^part[0-9]+$`)
// Existing admissions keep their original unit, including discarded parents.
func preserveCaptureParent(record *domain.CaptureRecord) bool {
return record != nil && (record.RunID != "" || record.Status == domain.CaptureRecordDiscarded || record.Status == domain.CaptureRecordLaunching)
}
func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
parents, err := discoverFlatCaptureGroups(objects, config)
if err != nil {
return nil, err
}
result := []captureGroup{}
for _, parent := range parents {
rootObjects := []captureObject{}
parts := map[string][]captureObject{}
for _, object := range parent.Objects {
relative := strings.TrimPrefix(object.Key, parent.Prefix+"/")
segments := strings.Split(relative, "/")
if len(segments) == 1 {
rootObjects = append(rootObjects, object)
}
if len(segments) >= 2 && capturePartDirectory.MatchString(segments[0]) {
if _, ok := parts[segments[0]]; !ok {
parts[segments[0]] = []captureObject{}
}
if len(segments) == 2 {
parts[segments[0]] = append(parts[segments[0]], object)
}
}
}
if len(parts) == 0 {
// Legacy rules remain valid, but nested files cannot fill root requirements.
flat, e := discoverFlatCaptureGroups(rootObjects, config)
if e != nil {
return nil, e
}
if len(flat) == 0 {
parent.Missing = []string{"根目录缺少必需数据"}
result = append(result, parent)
continue
}
group := flat[0]
for _, requirement := range captureRequirements(config) {
re := regexp.MustCompile(requirement.Pattern)
count := 0
for _, object := range rootObjects {
if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) {
count++
}
}
if count > 1 {
group.Missing = append(group.Missing, "匹配冲突 "+requirement.Name)
}
}
group.SourcePrefix = group.Prefix
for _, object := range rootObjects {
group.SourceKeys = append(group.SourceKeys, object.Key)
}
result = append(result, group)
continue
}
mixed := false
for _, object := range rootObjects {
name := strings.TrimPrefix(object.Key, parent.Prefix+"/")
for _, requirement := range defaultOrbbecCaptureRequirements[:5] {
if regexp.MustCompile(requirement.Pattern).MatchString(name) {
mixed = true
}
}
}
if mixed {
parent.Missing = []string{"目录结构冲突:根目录数据与 part 数据不能混合接入"}
result = append(result, parent)
continue
}
names := make([]string, 0, len(parts))
for name := range parts {
names = append(names, name)
}
sort.Strings(names)
for _, part := range names {
group := captureGroup{CaptureID: parent.CaptureID + "/" + part, Prefix: parent.Prefix + "/" + part, SourcePrefix: parent.Prefix}
captureName := regexp.QuoteMeta(parent.CaptureID)
partName := regexp.QuoteMeta(part)
requirements := []captureRequirement{
{"左相机视频", `(?i)^` + captureName + `_camera_left_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`},
{"右相机视频", `(?i)^` + captureName + `_camera_right_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`},
{"左相机 PTS", `(?i)^` + captureName + `_camera_left_` + partName + `_pts\.csv$`},
{"右相机 PTS", `(?i)^` + captureName + `_camera_right_` + partName + `_pts\.csv$`},
{"IMU", `(?i)^` + captureName + `_imu_` + partName + `\.csv$`},
{"相机标定", `(?i)^` + captureName + `_calibration_camera\.yaml$`},
{"IMU 标定", `(?i)^` + captureName + `_calibration_imu\.yaml$`},
}
for i, requirement := range requirements {
candidates := parts[part]
base := group.Prefix + "/"
if i >= 5 {
candidates = rootObjects
base = parent.Prefix + "/"
}
re := regexp.MustCompile(requirement.Pattern)
// Prefer PTS explicitly named for the selected video variant. Raw
// part PTS is still discoverable, but is not proof of transcode alignment.
if i == 2 || i == 3 {
videoRE := regexp.MustCompile(requirements[i-2].Pattern)
for _, selected := range group.Objects {
name := strings.TrimPrefix(selected.Key, group.Prefix+"/")
if !videoRE.MatchString(name) {
continue
}
stem := name[:strings.LastIndex(name, ".")]
exact := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(stem+"_pts.csv") + `$`)
for _, candidate := range candidates {
if exact.MatchString(strings.TrimPrefix(candidate.Key, base)) {
re = exact
break
}
}
}
}
matches := []captureObject{}
for _, object := range candidates {
if re.MatchString(strings.TrimPrefix(object.Key, base)) {
matches = append(matches, object)
}
}
if len(matches) == 0 {
group.Missing = append(group.Missing, part+" 缺少 "+requirement.Name)
}
if len(matches) > 1 {
group.Missing = append(group.Missing, part+" 匹配冲突 "+requirement.Name)
}
for _, object := range matches {
if object.Size <= 0 {
group.Missing = append(group.Missing, "空文件 "+object.Key)
}
group.Objects = append(group.Objects, object)
}
}
optional := regexp.MustCompile(`(?i)^` + captureName + `_audio_` + partName + `(?:_pts\.csv|\.wav)$`)
for _, object := range parts[part] {
if optional.MatchString(strings.TrimPrefix(object.Key, group.Prefix+"/")) {
group.Objects = append(group.Objects, object)
}
}
for _, object := range rootObjects {
if strings.EqualFold(object.Key, parent.Prefix+"/meta_"+parent.CaptureID+".json") {
group.Objects = append(group.Objects, object)
}
}
// Custom requirements are additional constraints; never bypass part isolation.
for _, pattern := range config.RequiredPatterns {
if strings.TrimSpace(pattern) == "" {
continue
}
re := regexp.MustCompile(pattern)
found := false
for _, object := range group.Objects {
if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) {
found = true
}
}
if !found {
group.Missing = append(group.Missing, part+" 缺少自定义必需数据 "+pattern)
}
}
for _, object := range group.Objects {
group.SourceKeys = append(group.SourceKeys, object.Key)
group.TotalBytes += object.Size
}
sort.Strings(group.SourceKeys)
group.Fingerprint = captureFingerprint(group.Objects)
result = append(result, group)
}
}
return result, nil
}

View File

@ -0,0 +1,138 @@
package worker
import (
"droboticflow/internal/domain"
"strings"
"testing"
)
const layoutCapture = "FYLX_B10_AZEC86400GP_19700101_035516"
func partObjects(part string) []captureObject {
items := []captureObject{}
for _, suffix := range []string{"camera_left_" + part + "_silent_480p_wm.mp4", "camera_right_" + part + "_silent_480p_wm.mp4", "camera_left_" + part + "_pts.csv", "camera_right_" + part + "_pts.csv", "imu_" + part + ".csv"} {
items = append(items, captureObject{Key: "incoming/" + layoutCapture + "/" + part + "/" + layoutCapture + "_" + suffix, Size: 10, ETag: "original"})
}
return items
}
func calibrationObjects() []captureObject {
return []captureObject{{Key: "incoming/" + layoutCapture + "/" + layoutCapture + "_calibration_camera.yaml", Size: 2}, {Key: "incoming/" + layoutCapture + "/" + layoutCapture + "_calibration_imu.yaml", Size: 2}}
}
func discoverLayout(t *testing.T, objects []captureObject) []captureGroup {
t.Helper()
groups, err := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"}))
if err != nil {
t.Fatal(err)
}
return groups
}
func TestPartCaptureIndependentGroupsAndFingerprint(t *testing.T) {
objects := append(calibrationObjects(), partObjects("part0001")...)
objects = append(objects, partObjects("part0002")...)
groups := discoverLayout(t, objects)
if len(groups) != 2 {
t.Fatal(groups)
}
for i, g := range groups {
part := []string{"part0001", "part0002"}[i]
if len(g.Missing) != 0 || len(g.SourceKeys) != 7 || g.CaptureID != layoutCapture+"/"+part || g.SourcePrefix != "incoming/"+layoutCapture {
t.Fatalf("bad unit: %+v", g)
}
for _, key := range g.SourceKeys {
if strings.Contains(key, "part") && !strings.Contains(key, part) {
t.Fatal("cross-part reference", key)
}
}
}
objects[len(objects)-1].ETag = "changed"
next := discoverLayout(t, objects)
if groups[0].Fingerprint != next[0].Fingerprint || groups[1].Fingerprint == next[1].Fingerprint {
t.Fatal("fingerprint not isolated")
}
objects[0].ETag = "new calibration"
if discoverLayout(t, objects)[0].Fingerprint == next[0].Fingerprint {
t.Fatal("shared calibration excluded from fingerprint")
}
if captureRecordID("workflow", groups[0].Prefix) == captureRecordID("workflow", groups[1].Prefix) {
t.Fatal("part identities collide")
}
}
func TestPartCaptureMissingEmptyConflictAndWrongIdentity(t *testing.T) {
for _, kind := range []string{"missing", "empty", "wrong-part", "wrong-capture", "duplicate", "missing-calibration"} {
t.Run(kind, func(t *testing.T) {
objects := append(calibrationObjects(), partObjects("part0001")...)
switch kind {
case "missing":
objects = objects[:len(objects)-1]
case "empty":
objects[len(objects)-1].Size = 0
case "wrong-part":
objects[len(objects)-1].Key = strings.Replace(objects[len(objects)-1].Key, "_imu_part0001", "_imu_part0002", 1)
case "wrong-capture":
objects[len(objects)-1].Key = "incoming/" + layoutCapture + "/part0001/other_imu_part0001.csv"
case "duplicate":
objects = append(objects, captureObject{Key: "incoming/" + layoutCapture + "/part0001/" + layoutCapture + "_camera_left_part0001.mp4", Size: 10})
case "missing-calibration":
objects = objects[1:]
}
objects = append(objects, partObjects("part0002")...)
groups := discoverLayout(t, objects)
if len(groups) != 2 || len(groups[0].Missing) == 0 {
t.Fatal("invalid part admitted", groups)
}
if kind != "missing-calibration" && len(groups[1].Missing) != 0 {
t.Fatal("other part blocked", groups[1])
}
})
}
}
func TestPartCaptureMixedAndLegacyLayouts(t *testing.T) {
legacy := completeCaptureObjects("incoming", "Orbbec_Ego_legacy")
groups := discoverLayout(t, legacy)
if len(groups) != 1 || len(groups[0].Missing) != 0 || groups[0].Prefix != "incoming/Orbbec_Ego_legacy" {
t.Fatal(groups)
}
mixed := append(completeCaptureObjects("incoming", layoutCapture), partObjects("part0001")...)
groups = discoverLayout(t, mixed)
if len(groups) != 1 || !strings.Contains(strings.Join(groups[0].Missing, ""), "结构冲突") {
t.Fatal(groups)
}
}
func TestPartCaptureHistoricalParentGuard(t *testing.T) {
for _, record := range []*domain.CaptureRecord{{RunID: "old", Status: domain.CaptureRecordFailed}, {RunID: "old", Status: domain.CaptureRecordCompleted}, {Status: domain.CaptureRecordDiscarded}, {Status: domain.CaptureRecordLaunching}} {
if !preserveCaptureParent(record) {
t.Fatal("historical parent may relaunch", record)
}
}
if preserveCaptureParent(nil) || preserveCaptureParent(&domain.CaptureRecord{Status: domain.CaptureRecordIncomplete}) {
t.Fatal("unadmitted parent cannot upgrade")
}
}
func TestPartCapturePrefersVideoSpecificPTS(t *testing.T) {
objects := append(calibrationObjects(), partObjects("part0001")...)
key := "incoming/" + layoutCapture + "/part0001/" + layoutCapture + "_camera_left_part0001_silent_480p_wm_pts.csv"
objects = append(objects, captureObject{Key: key, Size: 10})
groups := discoverLayout(t, objects)
if len(groups) != 1 || len(groups[0].Missing) != 0 {
t.Fatal(groups)
}
found := false
for _, selected := range groups[0].SourceKeys {
if selected == key {
found = true
}
if strings.HasSuffix(selected, "_camera_left_part0001_pts.csv") {
t.Fatal("raw PTS selected despite video-specific PTS")
}
}
if !found {
t.Fatal("video-specific PTS missing")
}
}

View File

@ -65,6 +65,7 @@ def _bos_entries(
pattern: str,
params: dict[str, str],
helpers: HelperMap,
object_keys: list[str] | None = None,
) -> list[dict[str, Any]]:
endpoint = str(params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "")).strip()
access_key_id = str(params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "")).strip()
@ -84,6 +85,15 @@ def _bos_entries(
for item in listed_entries
if "/_droboticflow/manifests/" not in "/" + str(item.get("path", "")).lstrip("/")
]
if object_keys is not None:
base = prefix.rstrip("/") + "/"
if not object_keys or any(not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in object_keys):
raise RuntimeError("目录文件选择超出 Capture 范围或为空")
selected = {key[len(base):] for key in object_keys}
entries = [item for item in entries if str(item.get("path", "")).lstrip("/") in selected]
missing = selected - {str(item.get("path", "")).lstrip("/") for item in entries}
if missing:
raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}")
if not entries:
raise RuntimeError(f"目录读取未在 bos://{bucket}/{prefix} 找到真实文件")
client, _ = helpers["build_bos_client"](endpoint, access_key_id, secret_access_key)
@ -122,7 +132,8 @@ def _run_legacy(
directory_ref = _directory_ref(inputs)
if directory_ref is not None:
bucket, prefix = directory_ref
entries = _bos_entries(bucket, prefix, pattern, params, helpers)
selected_ref = (_directory_input(inputs) or {}).get("payload", {}).get("ref", {})
entries = _bos_entries(bucket, prefix, pattern, params, helpers, selected_ref.get("objectKeys"))
source_path = helpers["canonical_bos_path"](bucket, prefix)
payload = {
"schemaVersion": "droboticflow.directory-read/v1",
@ -248,7 +259,7 @@ def _run_multiport(
bucket = str(ref.get("bucket", "")).strip()
prefix = str(ref.get("prefix", "")).strip().strip("/")
if bucket:
entries = _bos_entries(bucket, prefix, pattern, params, helpers)
entries = _bos_entries(bucket, prefix, pattern, params, helpers, ref.get("objectKeys"))
root_path = str(helpers["canonical_bos_path"](bucket, prefix))
else:
root_path = str(directory_input.get("path", "")).strip()

View File

@ -18,7 +18,12 @@ def _entry_path(entry: dict[str, Any]) -> str:
def _capture_id(path: str) -> str:
matched = CAPTURE_PATTERN.search(path)
# The Capture name is not required to contain a date (legacy business names).
stem = pathlib.PurePosixPath(path).stem
named = re.match(r"^(.+?)_(?:camera_(?:left|right)_part\d+(?:_.*)?|imu(?:_part\d+)?|audio(?:_part\d+)?(?:_pts)?|calibration_(?:camera|imu))$", stem, re.IGNORECASE)
if named:
return named.group(1)
matched = CAPTURE_PATTERN.search(path) or CAPTURE_PATTERN.search(pathlib.PurePosixPath(path).name)
if matched:
return matched.group("capture")
parts = pathlib.PurePosixPath(path).parts
@ -33,13 +38,17 @@ def _camera_side(path: str) -> str:
def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, str]:
"""Match sidecars within one Capture and segment, never the first segment."""
video = pathlib.PurePosixPath(video_path)
part = re.match(r"^(?P<capture>.+)_camera_(?:left|right)_(?P<part>part\d+)(?P<suffix>.*)$", video.stem)
part = re.match(r"^(?P<capture>.+)_camera_(?:left|right)_(?P<part>part\d+)(?P<suffix>.*)$", video.stem, re.IGNORECASE)
def find(name: str) -> str:
candidates = [_entry_path(item) for item in entries
if pathlib.PurePosixPath(_entry_path(item)).name == name]
if pathlib.PurePosixPath(_entry_path(item)).name.lower() == name.lower()]
local = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent]
matches = local or candidates
# Only calibration files may live at the Capture root above this part.
shared = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent.parent]
matches = local or (shared if "_calibration_" in name else [])
if candidates and not matches:
raise RuntimeError(f"配套文件不唯一或不在当前 part: {video_path} -> {name}")
if len(matches) > 1:
raise RuntimeError(f"配套文件不唯一: {video_path} -> {name}: {matches}")
return matches[0] if matches else ""
@ -59,7 +68,7 @@ def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, s
"imu": f"{capture}_imu_{segment}.csv"}
result = {key: find(name) for key, name in names.items()}
for key, suffix in (("audio", "audio.wav"), ("audioPts", "audio_pts.csv"), ("imu", "imu.csv")):
if not result[key]:
if not result[key] and not re.fullmatch(r"part\d+", video.parent.name, re.IGNORECASE):
result[key] = find(f"{capture}_{suffix}")
result.update({"cameraCalibration": find(f"{capture}_calibration_camera.yaml"),
"imuCalibration": find(f"{capture}_calibration_imu.yaml")})
@ -204,6 +213,16 @@ def _directory_entries(inputs: list[dict[str, Any]], helpers: HelperMap) -> tupl
if not endpoint or not access_key_id or not secret_access_key:
raise RuntimeError("Orbbec 多端口标准化需要目录引用对应的 BOS 凭证")
entries = helpers["list_bos_entries"](bucket, prefix, "", endpoint, access_key_id, secret_access_key)
if "objectKeys" in ref:
base = prefix.rstrip("/") + "/"
keys = ref["objectKeys"]
if not isinstance(keys, list) or not keys or any(not isinstance(key, str) or not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in keys):
raise RuntimeError("目录文件选择超出 Capture 范围或为空")
selected = {key[len(base):] for key in keys}
entries = [item for item in entries if _entry_path(item) in selected]
missing = selected - {_entry_path(item) for item in entries}
if missing:
raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}")
source_path = str(helpers["canonical_bos_path"](bucket, prefix))
return [
{

View File

@ -0,0 +1,80 @@
import pytest
from backend.operators.builtin.directory_scan import operator as directory
from backend.operators.builtin.orbbec_data_normalize import operator as normalize
C = "FYLX_B10_AZEC86400GP_19700101_035516"
PREFIX = "incoming/" + C
def test_part_pairing_shares_only_parent_calibration():
video = f"part0001/{C}_camera_left_part0001.mp4"
paths = [video, f"part0001/{C}_camera_left_part0001_pts.csv",
f"part0001/{C}_imu_part0001.csv", f"{C}_calibration_camera.yaml",
f"{C}_calibration_imu.yaml"]
entries = [{"path": path} for path in paths]
result = normalize._paired_files(video, entries)
assert result["pts"] == paths[1]
assert result["imu"] == paths[2]
assert result["cameraCalibration"] == paths[3]
assert result["imuCalibration"] == paths[4]
assert normalize._capture_id(video) == C
assert normalize._capture_id(paths[3]) == C
entries[2]["path"] = f"part0002/{C}_imu_part0001.csv"
with pytest.raises(RuntimeError, match="不在当前 part"):
normalize._paired_files(video, entries)
def mock_helpers(monkeypatch):
for key in ("BOS_ENDPOINT", "BOS_ACCESS_KEY_ID", "BOS_SECRET_ACCESS_KEY"):
monkeypatch.setenv(key, "isolated-test")
paths = [f"part0001/{C}_camera_left_part0001_pts.csv",
f"part0002/{C}_camera_left_part0002_pts.csv", f"{C}_calibration_camera.yaml"]
reads = []
class Client:
def get_object_as_string(self, bucket, key):
reads.append(key)
assert "part0002" not in key
return b"timestamp_us\n1\n2\n"
helpers = {
"list_bos_entries": lambda *args: [{"path": path, "sizeBytes": 20} for path in paths],
"build_bos_client": lambda *args: (Client(), None),
"canonical_bos_path": lambda bucket, prefix: f"bos://{bucket}/{prefix}",
}
return paths, reads, helpers
def test_directory_and_normalizer_honor_same_frozen_selection(monkeypatch):
paths, reads, helpers = mock_helpers(monkeypatch)
selected = [PREFIX + "/" + paths[i] for i in (0, 2)]
entries = directory._bos_entries("bucket", PREFIX, "", {}, helpers, selected)
assert {entry["path"] for entry in entries} == {paths[0], paths[2]}
assert reads == [selected[0]]
inputs = [{"payload": {"ref": {"kind": "directory", "bucket": "bucket", "prefix": PREFIX, "objectKeys": selected}}}]
entries, source = normalize._directory_entries(inputs, helpers)
assert {entry["path"] for entry in entries} == {paths[0], paths[2]}
assert source == "bos://bucket/" + PREFIX
@pytest.mark.parametrize("keys", [[], ["other/capture.csv"], [PREFIX + "/../other.csv"], [PREFIX + "/missing.csv"]])
def test_missing_or_out_of_scope_selected_files_fail_closed(monkeypatch, keys):
_, _, helpers = mock_helpers(monkeypatch)
with pytest.raises(RuntimeError):
directory._bos_entries("bucket", PREFIX, "", {}, helpers, keys)
inputs = [{"payload": {"ref": {"kind": "directory", "bucket": "bucket", "prefix": PREFIX, "objectKeys": keys}}}]
with pytest.raises(RuntimeError):
normalize._directory_entries(inputs, helpers)
def test_current_transcoded_filename_does_not_bypass_pts_safety():
video = f"part0001/{C}_camera_left_part0001_silent_480p_wm.mp4"
entries = [{"path": f"part0001/{C}_camera_left_part0001_pts.csv"}]
with pytest.raises(RuntimeError, match="转码视频缺少逐帧对应的 PTS"):
normalize._paired_files(video, entries)
def test_arbitrary_capture_names_keep_media_and_calibration_together():
for path in ("part0001/business_camera_left_part0001.mp4", "business_calibration_camera.yaml", "part0001/business_imu_part0001.csv"):
assert normalize._capture_id(path) == "business"