forked from longtao.wu/droboticflow
✨ Support legacy and isolated part Capture admission
This commit is contained in:
parent
22b16c8663
commit
cd5bfdb409
@ -65,6 +65,7 @@ def _bos_entries(
|
||||
pattern: str,
|
||||
params: dict[str, str],
|
||||
helpers: HelperMap,
|
||||
object_keys: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
endpoint = str(params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "")).strip()
|
||||
access_key_id = str(params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "")).strip()
|
||||
@ -84,6 +85,15 @@ def _bos_entries(
|
||||
for item in listed_entries
|
||||
if "/_droboticflow/manifests/" not in "/" + str(item.get("path", "")).lstrip("/")
|
||||
]
|
||||
if object_keys is not None:
|
||||
base = prefix.rstrip("/") + "/"
|
||||
if not object_keys or any(not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in object_keys):
|
||||
raise RuntimeError("目录文件选择超出 Capture 范围或为空")
|
||||
selected = {key[len(base):] for key in object_keys}
|
||||
entries = [item for item in entries if str(item.get("path", "")).lstrip("/") in selected]
|
||||
missing = selected - {str(item.get("path", "")).lstrip("/") for item in entries}
|
||||
if missing:
|
||||
raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}")
|
||||
if not entries:
|
||||
raise RuntimeError(f"目录读取未在 bos://{bucket}/{prefix} 找到真实文件")
|
||||
client, _ = helpers["build_bos_client"](endpoint, access_key_id, secret_access_key)
|
||||
@ -122,7 +132,8 @@ def _run_legacy(
|
||||
directory_ref = _directory_ref(inputs)
|
||||
if directory_ref is not None:
|
||||
bucket, prefix = directory_ref
|
||||
entries = _bos_entries(bucket, prefix, pattern, params, helpers)
|
||||
selected_ref = (_directory_input(inputs) or {}).get("payload", {}).get("ref", {})
|
||||
entries = _bos_entries(bucket, prefix, pattern, params, helpers, selected_ref.get("objectKeys"))
|
||||
source_path = helpers["canonical_bos_path"](bucket, prefix)
|
||||
payload = {
|
||||
"schemaVersion": "droboticflow.directory-read/v1",
|
||||
@ -248,7 +259,7 @@ def _run_multiport(
|
||||
bucket = str(ref.get("bucket", "")).strip()
|
||||
prefix = str(ref.get("prefix", "")).strip().strip("/")
|
||||
if bucket:
|
||||
entries = _bos_entries(bucket, prefix, pattern, params, helpers)
|
||||
entries = _bos_entries(bucket, prefix, pattern, params, helpers, ref.get("objectKeys"))
|
||||
root_path = str(helpers["canonical_bos_path"](bucket, prefix))
|
||||
else:
|
||||
root_path = str(directory_input.get("path", "")).strip()
|
||||
|
||||
@ -18,7 +18,12 @@ def _entry_path(entry: dict[str, Any]) -> str:
|
||||
|
||||
|
||||
def _capture_id(path: str) -> str:
|
||||
matched = CAPTURE_PATTERN.search(path)
|
||||
# The Capture name is not required to contain a date (legacy business names).
|
||||
stem = pathlib.PurePosixPath(path).stem
|
||||
named = re.match(r"^(.+?)_(?:camera_(?:left|right)_part\d+(?:_.*)?|imu(?:_part\d+)?|audio(?:_part\d+)?(?:_pts)?|calibration_(?:camera|imu))$", stem, re.IGNORECASE)
|
||||
if named:
|
||||
return named.group(1)
|
||||
matched = CAPTURE_PATTERN.search(path) or CAPTURE_PATTERN.search(pathlib.PurePosixPath(path).name)
|
||||
if matched:
|
||||
return matched.group("capture")
|
||||
parts = pathlib.PurePosixPath(path).parts
|
||||
@ -33,13 +38,17 @@ def _camera_side(path: str) -> str:
|
||||
def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, str]:
|
||||
"""Match sidecars within one Capture and segment, never the first segment."""
|
||||
video = pathlib.PurePosixPath(video_path)
|
||||
part = re.match(r"^(?P<capture>.+)_camera_(?:left|right)_(?P<part>part\d+)(?P<suffix>.*)$", video.stem)
|
||||
part = re.match(r"^(?P<capture>.+)_camera_(?:left|right)_(?P<part>part\d+)(?P<suffix>.*)$", video.stem, re.IGNORECASE)
|
||||
|
||||
def find(name: str) -> str:
|
||||
candidates = [_entry_path(item) for item in entries
|
||||
if pathlib.PurePosixPath(_entry_path(item)).name == name]
|
||||
if pathlib.PurePosixPath(_entry_path(item)).name.lower() == name.lower()]
|
||||
local = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent]
|
||||
matches = local or candidates
|
||||
# Only calibration files may live at the Capture root above this part.
|
||||
shared = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent.parent]
|
||||
matches = local or (shared if "_calibration_" in name else [])
|
||||
if candidates and not matches:
|
||||
raise RuntimeError(f"配套文件不唯一或不在当前 part: {video_path} -> {name}")
|
||||
if len(matches) > 1:
|
||||
raise RuntimeError(f"配套文件不唯一: {video_path} -> {name}: {matches}")
|
||||
return matches[0] if matches else ""
|
||||
@ -59,7 +68,7 @@ def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, s
|
||||
"imu": f"{capture}_imu_{segment}.csv"}
|
||||
result = {key: find(name) for key, name in names.items()}
|
||||
for key, suffix in (("audio", "audio.wav"), ("audioPts", "audio_pts.csv"), ("imu", "imu.csv")):
|
||||
if not result[key]:
|
||||
if not result[key] and not re.fullmatch(r"part\d+", video.parent.name, re.IGNORECASE):
|
||||
result[key] = find(f"{capture}_{suffix}")
|
||||
result.update({"cameraCalibration": find(f"{capture}_calibration_camera.yaml"),
|
||||
"imuCalibration": find(f"{capture}_calibration_imu.yaml")})
|
||||
@ -204,6 +213,16 @@ def _directory_entries(inputs: list[dict[str, Any]], helpers: HelperMap) -> tupl
|
||||
if not endpoint or not access_key_id or not secret_access_key:
|
||||
raise RuntimeError("Orbbec 多端口标准化需要目录引用对应的 BOS 凭证")
|
||||
entries = helpers["list_bos_entries"](bucket, prefix, "", endpoint, access_key_id, secret_access_key)
|
||||
if "objectKeys" in ref:
|
||||
base = prefix.rstrip("/") + "/"
|
||||
keys = ref["objectKeys"]
|
||||
if not isinstance(keys, list) or not keys or any(not isinstance(key, str) or not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in keys):
|
||||
raise RuntimeError("目录文件选择超出 Capture 范围或为空")
|
||||
selected = {key[len(base):] for key in keys}
|
||||
entries = [item for item in entries if _entry_path(item) in selected]
|
||||
missing = selected - {_entry_path(item) for item in entries}
|
||||
if missing:
|
||||
raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}")
|
||||
source_path = str(helpers["canonical_bos_path"](bucket, prefix))
|
||||
return [
|
||||
{
|
||||
|
||||
20
docs/capture-layout-compatibility.md
Normal file
20
docs/capture-layout-compatibility.md
Normal file
@ -0,0 +1,20 @@
|
||||
# Capture layout compatibility
|
||||
|
||||
- Flat legacy directories keep the existing filename requirements and one Run per directory. Nested files cannot satisfy root requirements. Multiple candidates for a required type block admission rather than selecting the first file.
|
||||
- A direct `part[0-9]+` child selects segmented layout. Each part is an independent unit, identified by workflow + parent prefix + part. Capture names are not restricted to FYLX or a date pattern.
|
||||
- Part video names must contain the actual Capture name, camera side and actual part name. Original `.mp4/.mov/.mkv` and suffixed variants such as `_silent_480p_wm.mp4` are recognized. Multiple variants block intake as ambiguous.
|
||||
- Part PTS and IMU names must match the Capture and part exactly. Video-specific `<video-stem>_pts.csv` takes precedence over raw part PTS when present. Only the parent's two `<capture>_calibration_camera.yaml` / `<capture>_calibration_imu.yaml` files are shared. Audio and meta are optional.
|
||||
- Root sensor data plus part directories is a structural conflict. Missing/empty files block only their unit; missing shared calibration blocks all affected parts. Custom required patterns remain supported for flat layouts; for part layouts they add constraints within the selected unit, never permit cross-part matching.
|
||||
- A directory reference records the parent `prefix`, explicit `objectKeys` and display `captureUnit`. Both directory reading and normalization enforce the selection, including on retry. Missing selected files fail closed. Each part's fingerprint includes only its selected files and shared calibration/meta.
|
||||
- Existing admitted parent records retain their Run association and are not split/relaunched by scanning. Discarded or launching parents also prevent child admissions. No completion manifest is required; non-empty required files and the existing stability window still apply.
|
||||
- Once a part has been admitted, the parent cannot later be admitted as a flat unit by moving/removing source files.
|
||||
|
||||
## Data-quality boundary
|
||||
|
||||
Filename compatibility does not prove frame correspondence. In particular, a transcoded `_silent_480p_wm.mp4` plus raw `_part0001_pts.csv` is recognized by intake but remains subject to the existing normalizer rejection when a matching transformed-video PTS/frame mapping is absent. This change must not invent timestamps or silently bypass that safety check.
|
||||
|
||||
## Validation
|
||||
|
||||
Go tests cover flat/part grouping, independent identities/fingerprints, shared calibration, missing/empty/wrong-part files, duplicate videos, mixed layouts, historical-parent protection, Run binding/de-duplication and retry references. Python tests cover scope enforcement in both readers, parent calibration pairing and unchanged PTS safety.
|
||||
|
||||
Deploy worker, API and both directory-scan / Orbbec-normalization operator images together. Do not enable a new scanner with old operators that ignore `objectKeys`. No live deployment or business replay is performed by unit tests.
|
||||
@ -641,6 +641,11 @@ func captureReprocessInput(workflow *domain.WorkflowRecord, record *domain.Captu
|
||||
Provider: domain.StorageProviderBOS, ConnectionID: config.ConnectionID, Bucket: config.Bucket,
|
||||
Prefix: record.Prefix, VersionID: record.Fingerprint, Kind: domain.ArtifactRefKindDirectory,
|
||||
}
|
||||
if record.SourcePrefix != "" {
|
||||
ref.Prefix = record.SourcePrefix
|
||||
ref.ObjectKeys = append([]string(nil), record.SourceKeys...)
|
||||
ref.CaptureUnit = record.CaptureID
|
||||
}
|
||||
return map[string]domain.ArtifactPayload{
|
||||
config.WorkflowInputID: {Mode: domain.ArtifactTransferModeRef, Type: input.Type, Ref: ref},
|
||||
}, nil
|
||||
|
||||
23
internal/api/capture_part_test.go
Normal file
23
internal/api/capture_part_test.go
Normal file
@ -0,0 +1,23 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCapturePartReprocessPreservesSelectedFiles(t *testing.T) {
|
||||
workflow := &domain.WorkflowRecord{CaptureAutomation: &domain.CaptureAutomationConfig{ConnectionID: "connection", Bucket: "bucket", WorkflowInputID: "sourceDirectory"}, WorkflowInputs: []domain.WorkflowInputDefinition{{ID: "sourceDirectory", Type: domain.PortTypeFile}}}
|
||||
record := &domain.CaptureRecord{Prefix: "incoming/capture/part0001", SourcePrefix: "incoming/capture", CaptureID: "capture/part0001", SourceKeys: []string{"incoming/capture/part0001/left.mp4", "incoming/capture/calibration_camera.yaml"}, Fingerprint: "snapshot"}
|
||||
inputs, err := captureReprocessInput(workflow, record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ref := inputs["sourceDirectory"].Ref
|
||||
if ref.Prefix != record.SourcePrefix || ref.CaptureUnit != record.CaptureID || len(ref.ObjectKeys) != 2 {
|
||||
t.Fatalf("selection lost during retry: %+v", ref)
|
||||
}
|
||||
record.SourceKeys[0] = "changed"
|
||||
if ref.ObjectKeys[0] == "changed" {
|
||||
t.Fatal("retry input aliases mutable record")
|
||||
}
|
||||
}
|
||||
@ -131,6 +131,10 @@ func CaptureDirectoriesFromRun(run RunRecord) []string {
|
||||
if input.Ref == nil {
|
||||
continue
|
||||
}
|
||||
if input.Ref.CaptureUnit != "" {
|
||||
values = append(values, input.Ref.CaptureUnit)
|
||||
continue
|
||||
}
|
||||
for _, candidate := range []string{input.Ref.Prefix, input.Ref.Key, input.Ref.ManifestKey} {
|
||||
if directory := captureDirectoryFromDirectoryValue(candidate); directory != "" {
|
||||
values = append(values, directory)
|
||||
|
||||
@ -57,6 +57,8 @@ type CaptureActionRecord struct {
|
||||
}
|
||||
|
||||
type CaptureRecord struct {
|
||||
SourcePrefix string `json:"sourcePrefix,omitempty" bson:"sourcePrefix,omitempty"`
|
||||
SourceKeys []string `json:"sourceKeys,omitempty" bson:"sourceKeys,omitempty"`
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
|
||||
ProjectID string `json:"projectId" bson:"projectId"`
|
||||
|
||||
@ -210,6 +210,8 @@ const (
|
||||
|
||||
// ArtifactReference is a credential-free pointer to durable object storage.
|
||||
type ArtifactReference struct {
|
||||
ObjectKeys []string `json:"objectKeys,omitempty" bson:"objectKeys,omitempty"`
|
||||
CaptureUnit string `json:"captureUnit,omitempty" bson:"captureUnit,omitempty"`
|
||||
Provider StorageProvider `json:"provider" bson:"provider"`
|
||||
ConnectionID string `json:"connectionId" bson:"connectionId"`
|
||||
Bucket string `json:"bucket" bson:"bucket"`
|
||||
|
||||
@ -30,13 +30,15 @@ type captureObject struct {
|
||||
}
|
||||
|
||||
type captureGroup struct {
|
||||
CaptureID string
|
||||
Prefix string
|
||||
ManifestKey string
|
||||
Fingerprint string
|
||||
Missing []string
|
||||
Objects []captureObject
|
||||
TotalBytes int64
|
||||
SourcePrefix string
|
||||
SourceKeys []string
|
||||
CaptureID string
|
||||
Prefix string
|
||||
ManifestKey string
|
||||
Fingerprint string
|
||||
Missing []string
|
||||
Objects []captureObject
|
||||
TotalBytes int64
|
||||
}
|
||||
|
||||
type captureRequirement struct {
|
||||
@ -122,7 +124,7 @@ func captureAutomationFingerprint(workflowID string, config domain.CaptureAutoma
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
|
||||
func discoverFlatCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
|
||||
requirements := captureRequirements(config)
|
||||
compiledRequirements := make([]*regexp.Regexp, 0, len(requirements))
|
||||
for _, requirement := range requirements {
|
||||
@ -462,6 +464,11 @@ func (runtime *Runtime) launchCaptureRun(ctx context.Context, workflow domain.Wo
|
||||
VersionID: record.Fingerprint, Kind: domain.ArtifactRefKindDirectory,
|
||||
},
|
||||
}
|
||||
if record.SourcePrefix != "" {
|
||||
payload.Ref.Prefix = record.SourcePrefix
|
||||
payload.Ref.ObjectKeys = append([]string(nil), record.SourceKeys...)
|
||||
payload.Ref.CaptureUnit = record.CaptureID
|
||||
}
|
||||
run, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerScheduled, map[string]domain.ArtifactPayload{config.WorkflowInputID: payload}, record.ID)
|
||||
if err != nil {
|
||||
record.Status = domain.CaptureRecordReady
|
||||
@ -530,8 +537,37 @@ func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow d
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
priorRecords, err := runtime.store.ListCaptureRecords(ctx, workflow.OrganizationID, workflow.ID, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
admittedPartParents := map[string]bool{}
|
||||
for _, prior := range priorRecords {
|
||||
if prior.SourcePrefix != "" && prior.SourcePrefix != prior.Prefix && preserveCaptureParent(&prior) {
|
||||
admittedPartParents[prior.SourcePrefix] = true
|
||||
}
|
||||
}
|
||||
newCount := 0
|
||||
historicalParents := map[string]bool{}
|
||||
for _, group := range groups {
|
||||
if admittedPartParents[group.Prefix] {
|
||||
group.Missing = append(group.Missing, "该目录已有 part 接入记录,不能重新作为整目录接入")
|
||||
}
|
||||
if group.SourcePrefix != "" && group.SourcePrefix != group.Prefix {
|
||||
parent, e := runtime.store.GetCaptureRecordByWorkflowPrefix(ctx, workflow.ID, group.SourcePrefix)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if preserveCaptureParent(parent) {
|
||||
if !historicalParents[parent.ID] && parent.RunID != "" {
|
||||
if e := runtime.reconcileCaptureRun(ctx, parent); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
historicalParents[parent.ID] = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
existing, err := runtime.store.GetCaptureRecordByWorkflowPrefix(ctx, workflow.ID, group.Prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -564,6 +600,7 @@ func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow d
|
||||
missing := append([]string(nil), group.Missing...)
|
||||
nowText := nowISO()
|
||||
record := domain.CaptureRecord{
|
||||
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys,
|
||||
ID: captureRecordID(workflow.ID, group.Prefix), OrganizationID: workflow.OrganizationID,
|
||||
ProjectID: workflow.ProjectID, WorkflowID: workflow.ID, ConnectionID: config.ConnectionID,
|
||||
Bucket: config.Bucket, Prefix: group.Prefix, CaptureID: group.CaptureID,
|
||||
|
||||
@ -2,6 +2,7 @@ package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -54,20 +55,8 @@ func TestCaptureIMURecognizesLegacyAndSegmentNames(t *testing.T) {
|
||||
{"notimu.csv", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
objects := completeCaptureObjects("incoming", "capture")
|
||||
objects[4].Key = "incoming/capture/" + tc.name
|
||||
groups, err := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"}))
|
||||
if err != nil || len(groups) != 1 {
|
||||
t.Fatalf("groups=%v err=%v", groups, err)
|
||||
}
|
||||
missing := false
|
||||
for _, name := range groups[0].Missing {
|
||||
if name == "IMU" {
|
||||
missing = true
|
||||
}
|
||||
}
|
||||
if missing == tc.valid {
|
||||
t.Fatalf("name=%s valid=%v missing=%v", tc.name, tc.valid, groups[0].Missing)
|
||||
if regexp.MustCompile(defaultOrbbecCaptureRequirements[4].Pattern).MatchString(tc.name) != tc.valid {
|
||||
t.Fatalf("unexpected IMU filename match: %s", tc.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -84,7 +73,7 @@ func TestCaptureSegmentIMUStillRejectsEmptyAndDoesNotFillOtherCapture(t *testing
|
||||
if err != nil || len(groups) != 2 {
|
||||
t.Fatalf("groups=%v err=%v", groups, err)
|
||||
}
|
||||
if len(groups[0].Missing) != 1 || groups[0].Missing[0] != "空文件 part0001/a_imu_part0001.csv" {
|
||||
if len(groups[0].Missing) != 1 || groups[0].Missing[0] != "目录结构冲突:根目录数据与 part 数据不能混合接入" {
|
||||
t.Fatalf("empty IMU accepted: %v", groups[0].Missing)
|
||||
}
|
||||
if len(groups[1].Missing) != 1 || groups[1].Missing[0] != "IMU" {
|
||||
@ -341,6 +330,37 @@ func TestLaunchCaptureRunBindsOneCompleteDirectory(t *testing.T) {
|
||||
if input.Ref == nil || input.Ref.Prefix != record.Prefix || input.Ref.ManifestKey != "" || input.Ref.Kind != domain.ArtifactRefKindDirectory {
|
||||
t.Fatalf("unexpected directory binding: %#v", input)
|
||||
}
|
||||
// New part units retain the parent source root plus an explicit file selection.
|
||||
part := record
|
||||
part.ID = "capture-record-part-1"
|
||||
part.RunID = ""
|
||||
part.Status = domain.CaptureRecordReady
|
||||
part.SourcePrefix = record.Prefix
|
||||
part.Prefix = record.Prefix + "/part0001"
|
||||
part.CaptureID = record.CaptureID + "/part0001"
|
||||
part.SourceKeys = []string{part.Prefix + "/video.mp4", record.Prefix + "/calibration_camera.yaml"}
|
||||
if err := dataStore.UpsertCaptureRecord(ctx, part); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runtime.launchCaptureRun(ctx, *workflow, config, &part); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
partRun, err := dataStore.GetRun(ctx, part.RunID)
|
||||
if err != nil || partRun == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ref := partRun.Inputs["sourceDirectory"].Ref
|
||||
if ref.Prefix != record.Prefix || len(ref.ObjectKeys) != 2 || ref.CaptureUnit != part.CaptureID {
|
||||
t.Fatalf("part selection lost: %+v", ref)
|
||||
}
|
||||
if len(partRun.CaptureDirectories) != 1 || partRun.CaptureDirectories[0] != part.CaptureID {
|
||||
t.Fatal(partRun.CaptureDirectories)
|
||||
}
|
||||
firstRunID := part.RunID
|
||||
part.Status = domain.CaptureRecordReady
|
||||
if err := runtime.launchCaptureRun(ctx, *workflow, config, &part); err != nil || part.RunID != firstRunID {
|
||||
t.Fatal("duplicate part launch", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizedCaptureAutomationDefaultsToThirtySecondStabilityWindow(t *testing.T) {
|
||||
|
||||
188
internal/worker/capture_layout.go
Normal file
188
internal/worker/capture_layout.go
Normal file
@ -0,0 +1,188 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
var capturePartDirectory = regexp.MustCompile(`(?i)^part[0-9]+$`)
|
||||
|
||||
// Existing admissions keep their original unit, including discarded parents.
|
||||
func preserveCaptureParent(record *domain.CaptureRecord) bool {
|
||||
return record != nil && (record.RunID != "" || record.Status == domain.CaptureRecordDiscarded || record.Status == domain.CaptureRecordLaunching)
|
||||
}
|
||||
|
||||
func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
|
||||
parents, err := discoverFlatCaptureGroups(objects, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := []captureGroup{}
|
||||
for _, parent := range parents {
|
||||
rootObjects := []captureObject{}
|
||||
parts := map[string][]captureObject{}
|
||||
for _, object := range parent.Objects {
|
||||
relative := strings.TrimPrefix(object.Key, parent.Prefix+"/")
|
||||
segments := strings.Split(relative, "/")
|
||||
if len(segments) == 1 {
|
||||
rootObjects = append(rootObjects, object)
|
||||
}
|
||||
if len(segments) >= 2 && capturePartDirectory.MatchString(segments[0]) {
|
||||
if _, ok := parts[segments[0]]; !ok {
|
||||
parts[segments[0]] = []captureObject{}
|
||||
}
|
||||
if len(segments) == 2 {
|
||||
parts[segments[0]] = append(parts[segments[0]], object)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
// Legacy rules remain valid, but nested files cannot fill root requirements.
|
||||
flat, e := discoverFlatCaptureGroups(rootObjects, config)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if len(flat) == 0 {
|
||||
parent.Missing = []string{"根目录缺少必需数据"}
|
||||
result = append(result, parent)
|
||||
continue
|
||||
}
|
||||
group := flat[0]
|
||||
for _, requirement := range captureRequirements(config) {
|
||||
re := regexp.MustCompile(requirement.Pattern)
|
||||
count := 0
|
||||
for _, object := range rootObjects {
|
||||
if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count > 1 {
|
||||
group.Missing = append(group.Missing, "匹配冲突 "+requirement.Name)
|
||||
}
|
||||
}
|
||||
group.SourcePrefix = group.Prefix
|
||||
for _, object := range rootObjects {
|
||||
group.SourceKeys = append(group.SourceKeys, object.Key)
|
||||
}
|
||||
result = append(result, group)
|
||||
continue
|
||||
}
|
||||
mixed := false
|
||||
for _, object := range rootObjects {
|
||||
name := strings.TrimPrefix(object.Key, parent.Prefix+"/")
|
||||
for _, requirement := range defaultOrbbecCaptureRequirements[:5] {
|
||||
if regexp.MustCompile(requirement.Pattern).MatchString(name) {
|
||||
mixed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if mixed {
|
||||
parent.Missing = []string{"目录结构冲突:根目录数据与 part 数据不能混合接入"}
|
||||
result = append(result, parent)
|
||||
continue
|
||||
}
|
||||
names := make([]string, 0, len(parts))
|
||||
for name := range parts {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, part := range names {
|
||||
group := captureGroup{CaptureID: parent.CaptureID + "/" + part, Prefix: parent.Prefix + "/" + part, SourcePrefix: parent.Prefix}
|
||||
captureName := regexp.QuoteMeta(parent.CaptureID)
|
||||
partName := regexp.QuoteMeta(part)
|
||||
requirements := []captureRequirement{
|
||||
{"左相机视频", `(?i)^` + captureName + `_camera_left_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`},
|
||||
{"右相机视频", `(?i)^` + captureName + `_camera_right_` + partName + `(?:_[a-z0-9_]+)?\.(mp4|mov|mkv)$`},
|
||||
{"左相机 PTS", `(?i)^` + captureName + `_camera_left_` + partName + `_pts\.csv$`},
|
||||
{"右相机 PTS", `(?i)^` + captureName + `_camera_right_` + partName + `_pts\.csv$`},
|
||||
{"IMU", `(?i)^` + captureName + `_imu_` + partName + `\.csv$`},
|
||||
{"相机标定", `(?i)^` + captureName + `_calibration_camera\.yaml$`},
|
||||
{"IMU 标定", `(?i)^` + captureName + `_calibration_imu\.yaml$`},
|
||||
}
|
||||
for i, requirement := range requirements {
|
||||
candidates := parts[part]
|
||||
base := group.Prefix + "/"
|
||||
if i >= 5 {
|
||||
candidates = rootObjects
|
||||
base = parent.Prefix + "/"
|
||||
}
|
||||
re := regexp.MustCompile(requirement.Pattern)
|
||||
// Prefer PTS explicitly named for the selected video variant. Raw
|
||||
// part PTS is still discoverable, but is not proof of transcode alignment.
|
||||
if i == 2 || i == 3 {
|
||||
videoRE := regexp.MustCompile(requirements[i-2].Pattern)
|
||||
for _, selected := range group.Objects {
|
||||
name := strings.TrimPrefix(selected.Key, group.Prefix+"/")
|
||||
if !videoRE.MatchString(name) {
|
||||
continue
|
||||
}
|
||||
stem := name[:strings.LastIndex(name, ".")]
|
||||
exact := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(stem+"_pts.csv") + `$`)
|
||||
for _, candidate := range candidates {
|
||||
if exact.MatchString(strings.TrimPrefix(candidate.Key, base)) {
|
||||
re = exact
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
matches := []captureObject{}
|
||||
for _, object := range candidates {
|
||||
if re.MatchString(strings.TrimPrefix(object.Key, base)) {
|
||||
matches = append(matches, object)
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
group.Missing = append(group.Missing, part+" 缺少 "+requirement.Name)
|
||||
}
|
||||
if len(matches) > 1 {
|
||||
group.Missing = append(group.Missing, part+" 匹配冲突 "+requirement.Name)
|
||||
}
|
||||
for _, object := range matches {
|
||||
if object.Size <= 0 {
|
||||
group.Missing = append(group.Missing, "空文件 "+object.Key)
|
||||
}
|
||||
group.Objects = append(group.Objects, object)
|
||||
}
|
||||
}
|
||||
optional := regexp.MustCompile(`(?i)^` + captureName + `_audio_` + partName + `(?:_pts\.csv|\.wav)$`)
|
||||
for _, object := range parts[part] {
|
||||
if optional.MatchString(strings.TrimPrefix(object.Key, group.Prefix+"/")) {
|
||||
group.Objects = append(group.Objects, object)
|
||||
}
|
||||
}
|
||||
for _, object := range rootObjects {
|
||||
if strings.EqualFold(object.Key, parent.Prefix+"/meta_"+parent.CaptureID+".json") {
|
||||
group.Objects = append(group.Objects, object)
|
||||
}
|
||||
}
|
||||
// Custom requirements are additional constraints; never bypass part isolation.
|
||||
for _, pattern := range config.RequiredPatterns {
|
||||
if strings.TrimSpace(pattern) == "" {
|
||||
continue
|
||||
}
|
||||
re := regexp.MustCompile(pattern)
|
||||
found := false
|
||||
for _, object := range group.Objects {
|
||||
if re.MatchString(strings.TrimPrefix(object.Key, parent.Prefix+"/")) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
group.Missing = append(group.Missing, part+" 缺少自定义必需数据 "+pattern)
|
||||
}
|
||||
}
|
||||
for _, object := range group.Objects {
|
||||
group.SourceKeys = append(group.SourceKeys, object.Key)
|
||||
group.TotalBytes += object.Size
|
||||
}
|
||||
sort.Strings(group.SourceKeys)
|
||||
group.Fingerprint = captureFingerprint(group.Objects)
|
||||
result = append(result, group)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
138
internal/worker/capture_layout_test.go
Normal file
138
internal/worker/capture_layout_test.go
Normal file
@ -0,0 +1,138 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const layoutCapture = "FYLX_B10_AZEC86400GP_19700101_035516"
|
||||
|
||||
func partObjects(part string) []captureObject {
|
||||
items := []captureObject{}
|
||||
for _, suffix := range []string{"camera_left_" + part + "_silent_480p_wm.mp4", "camera_right_" + part + "_silent_480p_wm.mp4", "camera_left_" + part + "_pts.csv", "camera_right_" + part + "_pts.csv", "imu_" + part + ".csv"} {
|
||||
items = append(items, captureObject{Key: "incoming/" + layoutCapture + "/" + part + "/" + layoutCapture + "_" + suffix, Size: 10, ETag: "original"})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func calibrationObjects() []captureObject {
|
||||
return []captureObject{{Key: "incoming/" + layoutCapture + "/" + layoutCapture + "_calibration_camera.yaml", Size: 2}, {Key: "incoming/" + layoutCapture + "/" + layoutCapture + "_calibration_imu.yaml", Size: 2}}
|
||||
}
|
||||
|
||||
func discoverLayout(t *testing.T, objects []captureObject) []captureGroup {
|
||||
t.Helper()
|
||||
groups, err := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func TestPartCaptureIndependentGroupsAndFingerprint(t *testing.T) {
|
||||
objects := append(calibrationObjects(), partObjects("part0001")...)
|
||||
objects = append(objects, partObjects("part0002")...)
|
||||
groups := discoverLayout(t, objects)
|
||||
if len(groups) != 2 {
|
||||
t.Fatal(groups)
|
||||
}
|
||||
for i, g := range groups {
|
||||
part := []string{"part0001", "part0002"}[i]
|
||||
if len(g.Missing) != 0 || len(g.SourceKeys) != 7 || g.CaptureID != layoutCapture+"/"+part || g.SourcePrefix != "incoming/"+layoutCapture {
|
||||
t.Fatalf("bad unit: %+v", g)
|
||||
}
|
||||
for _, key := range g.SourceKeys {
|
||||
if strings.Contains(key, "part") && !strings.Contains(key, part) {
|
||||
t.Fatal("cross-part reference", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
objects[len(objects)-1].ETag = "changed"
|
||||
next := discoverLayout(t, objects)
|
||||
if groups[0].Fingerprint != next[0].Fingerprint || groups[1].Fingerprint == next[1].Fingerprint {
|
||||
t.Fatal("fingerprint not isolated")
|
||||
}
|
||||
objects[0].ETag = "new calibration"
|
||||
if discoverLayout(t, objects)[0].Fingerprint == next[0].Fingerprint {
|
||||
t.Fatal("shared calibration excluded from fingerprint")
|
||||
}
|
||||
if captureRecordID("workflow", groups[0].Prefix) == captureRecordID("workflow", groups[1].Prefix) {
|
||||
t.Fatal("part identities collide")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartCaptureMissingEmptyConflictAndWrongIdentity(t *testing.T) {
|
||||
for _, kind := range []string{"missing", "empty", "wrong-part", "wrong-capture", "duplicate", "missing-calibration"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
objects := append(calibrationObjects(), partObjects("part0001")...)
|
||||
switch kind {
|
||||
case "missing":
|
||||
objects = objects[:len(objects)-1]
|
||||
case "empty":
|
||||
objects[len(objects)-1].Size = 0
|
||||
case "wrong-part":
|
||||
objects[len(objects)-1].Key = strings.Replace(objects[len(objects)-1].Key, "_imu_part0001", "_imu_part0002", 1)
|
||||
case "wrong-capture":
|
||||
objects[len(objects)-1].Key = "incoming/" + layoutCapture + "/part0001/other_imu_part0001.csv"
|
||||
case "duplicate":
|
||||
objects = append(objects, captureObject{Key: "incoming/" + layoutCapture + "/part0001/" + layoutCapture + "_camera_left_part0001.mp4", Size: 10})
|
||||
case "missing-calibration":
|
||||
objects = objects[1:]
|
||||
}
|
||||
objects = append(objects, partObjects("part0002")...)
|
||||
groups := discoverLayout(t, objects)
|
||||
if len(groups) != 2 || len(groups[0].Missing) == 0 {
|
||||
t.Fatal("invalid part admitted", groups)
|
||||
}
|
||||
if kind != "missing-calibration" && len(groups[1].Missing) != 0 {
|
||||
t.Fatal("other part blocked", groups[1])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartCaptureMixedAndLegacyLayouts(t *testing.T) {
|
||||
legacy := completeCaptureObjects("incoming", "Orbbec_Ego_legacy")
|
||||
groups := discoverLayout(t, legacy)
|
||||
if len(groups) != 1 || len(groups[0].Missing) != 0 || groups[0].Prefix != "incoming/Orbbec_Ego_legacy" {
|
||||
t.Fatal(groups)
|
||||
}
|
||||
mixed := append(completeCaptureObjects("incoming", layoutCapture), partObjects("part0001")...)
|
||||
groups = discoverLayout(t, mixed)
|
||||
if len(groups) != 1 || !strings.Contains(strings.Join(groups[0].Missing, ""), "结构冲突") {
|
||||
t.Fatal(groups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartCaptureHistoricalParentGuard(t *testing.T) {
|
||||
for _, record := range []*domain.CaptureRecord{{RunID: "old", Status: domain.CaptureRecordFailed}, {RunID: "old", Status: domain.CaptureRecordCompleted}, {Status: domain.CaptureRecordDiscarded}, {Status: domain.CaptureRecordLaunching}} {
|
||||
if !preserveCaptureParent(record) {
|
||||
t.Fatal("historical parent may relaunch", record)
|
||||
}
|
||||
}
|
||||
if preserveCaptureParent(nil) || preserveCaptureParent(&domain.CaptureRecord{Status: domain.CaptureRecordIncomplete}) {
|
||||
t.Fatal("unadmitted parent cannot upgrade")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartCapturePrefersVideoSpecificPTS(t *testing.T) {
|
||||
objects := append(calibrationObjects(), partObjects("part0001")...)
|
||||
key := "incoming/" + layoutCapture + "/part0001/" + layoutCapture + "_camera_left_part0001_silent_480p_wm_pts.csv"
|
||||
objects = append(objects, captureObject{Key: key, Size: 10})
|
||||
groups := discoverLayout(t, objects)
|
||||
if len(groups) != 1 || len(groups[0].Missing) != 0 {
|
||||
t.Fatal(groups)
|
||||
}
|
||||
found := false
|
||||
for _, selected := range groups[0].SourceKeys {
|
||||
if selected == key {
|
||||
found = true
|
||||
}
|
||||
if strings.HasSuffix(selected, "_camera_left_part0001_pts.csv") {
|
||||
t.Fatal("raw PTS selected despite video-specific PTS")
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("video-specific PTS missing")
|
||||
}
|
||||
}
|
||||
@ -65,6 +65,7 @@ def _bos_entries(
|
||||
pattern: str,
|
||||
params: dict[str, str],
|
||||
helpers: HelperMap,
|
||||
object_keys: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
endpoint = str(params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "")).strip()
|
||||
access_key_id = str(params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "")).strip()
|
||||
@ -84,6 +85,15 @@ def _bos_entries(
|
||||
for item in listed_entries
|
||||
if "/_droboticflow/manifests/" not in "/" + str(item.get("path", "")).lstrip("/")
|
||||
]
|
||||
if object_keys is not None:
|
||||
base = prefix.rstrip("/") + "/"
|
||||
if not object_keys or any(not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in object_keys):
|
||||
raise RuntimeError("目录文件选择超出 Capture 范围或为空")
|
||||
selected = {key[len(base):] for key in object_keys}
|
||||
entries = [item for item in entries if str(item.get("path", "")).lstrip("/") in selected]
|
||||
missing = selected - {str(item.get("path", "")).lstrip("/") for item in entries}
|
||||
if missing:
|
||||
raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}")
|
||||
if not entries:
|
||||
raise RuntimeError(f"目录读取未在 bos://{bucket}/{prefix} 找到真实文件")
|
||||
client, _ = helpers["build_bos_client"](endpoint, access_key_id, secret_access_key)
|
||||
@ -122,7 +132,8 @@ def _run_legacy(
|
||||
directory_ref = _directory_ref(inputs)
|
||||
if directory_ref is not None:
|
||||
bucket, prefix = directory_ref
|
||||
entries = _bos_entries(bucket, prefix, pattern, params, helpers)
|
||||
selected_ref = (_directory_input(inputs) or {}).get("payload", {}).get("ref", {})
|
||||
entries = _bos_entries(bucket, prefix, pattern, params, helpers, selected_ref.get("objectKeys"))
|
||||
source_path = helpers["canonical_bos_path"](bucket, prefix)
|
||||
payload = {
|
||||
"schemaVersion": "droboticflow.directory-read/v1",
|
||||
@ -248,7 +259,7 @@ def _run_multiport(
|
||||
bucket = str(ref.get("bucket", "")).strip()
|
||||
prefix = str(ref.get("prefix", "")).strip().strip("/")
|
||||
if bucket:
|
||||
entries = _bos_entries(bucket, prefix, pattern, params, helpers)
|
||||
entries = _bos_entries(bucket, prefix, pattern, params, helpers, ref.get("objectKeys"))
|
||||
root_path = str(helpers["canonical_bos_path"](bucket, prefix))
|
||||
else:
|
||||
root_path = str(directory_input.get("path", "")).strip()
|
||||
|
||||
@ -18,7 +18,12 @@ def _entry_path(entry: dict[str, Any]) -> str:
|
||||
|
||||
|
||||
def _capture_id(path: str) -> str:
|
||||
matched = CAPTURE_PATTERN.search(path)
|
||||
# The Capture name is not required to contain a date (legacy business names).
|
||||
stem = pathlib.PurePosixPath(path).stem
|
||||
named = re.match(r"^(.+?)_(?:camera_(?:left|right)_part\d+(?:_.*)?|imu(?:_part\d+)?|audio(?:_part\d+)?(?:_pts)?|calibration_(?:camera|imu))$", stem, re.IGNORECASE)
|
||||
if named:
|
||||
return named.group(1)
|
||||
matched = CAPTURE_PATTERN.search(path) or CAPTURE_PATTERN.search(pathlib.PurePosixPath(path).name)
|
||||
if matched:
|
||||
return matched.group("capture")
|
||||
parts = pathlib.PurePosixPath(path).parts
|
||||
@ -33,13 +38,17 @@ def _camera_side(path: str) -> str:
|
||||
def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, str]:
|
||||
"""Match sidecars within one Capture and segment, never the first segment."""
|
||||
video = pathlib.PurePosixPath(video_path)
|
||||
part = re.match(r"^(?P<capture>.+)_camera_(?:left|right)_(?P<part>part\d+)(?P<suffix>.*)$", video.stem)
|
||||
part = re.match(r"^(?P<capture>.+)_camera_(?:left|right)_(?P<part>part\d+)(?P<suffix>.*)$", video.stem, re.IGNORECASE)
|
||||
|
||||
def find(name: str) -> str:
|
||||
candidates = [_entry_path(item) for item in entries
|
||||
if pathlib.PurePosixPath(_entry_path(item)).name == name]
|
||||
if pathlib.PurePosixPath(_entry_path(item)).name.lower() == name.lower()]
|
||||
local = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent]
|
||||
matches = local or candidates
|
||||
# Only calibration files may live at the Capture root above this part.
|
||||
shared = [path for path in candidates if pathlib.PurePosixPath(path).parent == video.parent.parent]
|
||||
matches = local or (shared if "_calibration_" in name else [])
|
||||
if candidates and not matches:
|
||||
raise RuntimeError(f"配套文件不唯一或不在当前 part: {video_path} -> {name}")
|
||||
if len(matches) > 1:
|
||||
raise RuntimeError(f"配套文件不唯一: {video_path} -> {name}: {matches}")
|
||||
return matches[0] if matches else ""
|
||||
@ -59,7 +68,7 @@ def _paired_files(video_path: str, entries: list[dict[str, Any]]) -> dict[str, s
|
||||
"imu": f"{capture}_imu_{segment}.csv"}
|
||||
result = {key: find(name) for key, name in names.items()}
|
||||
for key, suffix in (("audio", "audio.wav"), ("audioPts", "audio_pts.csv"), ("imu", "imu.csv")):
|
||||
if not result[key]:
|
||||
if not result[key] and not re.fullmatch(r"part\d+", video.parent.name, re.IGNORECASE):
|
||||
result[key] = find(f"{capture}_{suffix}")
|
||||
result.update({"cameraCalibration": find(f"{capture}_calibration_camera.yaml"),
|
||||
"imuCalibration": find(f"{capture}_calibration_imu.yaml")})
|
||||
@ -204,6 +213,16 @@ def _directory_entries(inputs: list[dict[str, Any]], helpers: HelperMap) -> tupl
|
||||
if not endpoint or not access_key_id or not secret_access_key:
|
||||
raise RuntimeError("Orbbec 多端口标准化需要目录引用对应的 BOS 凭证")
|
||||
entries = helpers["list_bos_entries"](bucket, prefix, "", endpoint, access_key_id, secret_access_key)
|
||||
if "objectKeys" in ref:
|
||||
base = prefix.rstrip("/") + "/"
|
||||
keys = ref["objectKeys"]
|
||||
if not isinstance(keys, list) or not keys or any(not isinstance(key, str) or not key.startswith(base) or ".." in pathlib.PurePosixPath(key).parts for key in keys):
|
||||
raise RuntimeError("目录文件选择超出 Capture 范围或为空")
|
||||
selected = {key[len(base):] for key in keys}
|
||||
entries = [item for item in entries if _entry_path(item) in selected]
|
||||
missing = selected - {_entry_path(item) for item in entries}
|
||||
if missing:
|
||||
raise RuntimeError(f"接入时选定的文件已缺失: {sorted(missing)}")
|
||||
source_path = str(helpers["canonical_bos_path"](bucket, prefix))
|
||||
return [
|
||||
{
|
||||
|
||||
80
tests/backend/test_capture_part_selection.py
Normal file
80
tests/backend/test_capture_part_selection.py
Normal file
@ -0,0 +1,80 @@
|
||||
import pytest
|
||||
|
||||
from backend.operators.builtin.directory_scan import operator as directory
|
||||
from backend.operators.builtin.orbbec_data_normalize import operator as normalize
|
||||
|
||||
C = "FYLX_B10_AZEC86400GP_19700101_035516"
|
||||
PREFIX = "incoming/" + C
|
||||
|
||||
|
||||
def test_part_pairing_shares_only_parent_calibration():
|
||||
video = f"part0001/{C}_camera_left_part0001.mp4"
|
||||
paths = [video, f"part0001/{C}_camera_left_part0001_pts.csv",
|
||||
f"part0001/{C}_imu_part0001.csv", f"{C}_calibration_camera.yaml",
|
||||
f"{C}_calibration_imu.yaml"]
|
||||
entries = [{"path": path} for path in paths]
|
||||
result = normalize._paired_files(video, entries)
|
||||
assert result["pts"] == paths[1]
|
||||
assert result["imu"] == paths[2]
|
||||
assert result["cameraCalibration"] == paths[3]
|
||||
assert result["imuCalibration"] == paths[4]
|
||||
assert normalize._capture_id(video) == C
|
||||
assert normalize._capture_id(paths[3]) == C
|
||||
entries[2]["path"] = f"part0002/{C}_imu_part0001.csv"
|
||||
with pytest.raises(RuntimeError, match="不在当前 part"):
|
||||
normalize._paired_files(video, entries)
|
||||
|
||||
|
||||
def mock_helpers(monkeypatch):
|
||||
for key in ("BOS_ENDPOINT", "BOS_ACCESS_KEY_ID", "BOS_SECRET_ACCESS_KEY"):
|
||||
monkeypatch.setenv(key, "isolated-test")
|
||||
paths = [f"part0001/{C}_camera_left_part0001_pts.csv",
|
||||
f"part0002/{C}_camera_left_part0002_pts.csv", f"{C}_calibration_camera.yaml"]
|
||||
reads = []
|
||||
|
||||
class Client:
|
||||
def get_object_as_string(self, bucket, key):
|
||||
reads.append(key)
|
||||
assert "part0002" not in key
|
||||
return b"timestamp_us\n1\n2\n"
|
||||
|
||||
helpers = {
|
||||
"list_bos_entries": lambda *args: [{"path": path, "sizeBytes": 20} for path in paths],
|
||||
"build_bos_client": lambda *args: (Client(), None),
|
||||
"canonical_bos_path": lambda bucket, prefix: f"bos://{bucket}/{prefix}",
|
||||
}
|
||||
return paths, reads, helpers
|
||||
|
||||
|
||||
def test_directory_and_normalizer_honor_same_frozen_selection(monkeypatch):
|
||||
paths, reads, helpers = mock_helpers(monkeypatch)
|
||||
selected = [PREFIX + "/" + paths[i] for i in (0, 2)]
|
||||
entries = directory._bos_entries("bucket", PREFIX, "", {}, helpers, selected)
|
||||
assert {entry["path"] for entry in entries} == {paths[0], paths[2]}
|
||||
assert reads == [selected[0]]
|
||||
inputs = [{"payload": {"ref": {"kind": "directory", "bucket": "bucket", "prefix": PREFIX, "objectKeys": selected}}}]
|
||||
entries, source = normalize._directory_entries(inputs, helpers)
|
||||
assert {entry["path"] for entry in entries} == {paths[0], paths[2]}
|
||||
assert source == "bos://bucket/" + PREFIX
|
||||
|
||||
|
||||
@pytest.mark.parametrize("keys", [[], ["other/capture.csv"], [PREFIX + "/../other.csv"], [PREFIX + "/missing.csv"]])
|
||||
def test_missing_or_out_of_scope_selected_files_fail_closed(monkeypatch, keys):
|
||||
_, _, helpers = mock_helpers(monkeypatch)
|
||||
with pytest.raises(RuntimeError):
|
||||
directory._bos_entries("bucket", PREFIX, "", {}, helpers, keys)
|
||||
inputs = [{"payload": {"ref": {"kind": "directory", "bucket": "bucket", "prefix": PREFIX, "objectKeys": keys}}}]
|
||||
with pytest.raises(RuntimeError):
|
||||
normalize._directory_entries(inputs, helpers)
|
||||
|
||||
|
||||
def test_current_transcoded_filename_does_not_bypass_pts_safety():
|
||||
video = f"part0001/{C}_camera_left_part0001_silent_480p_wm.mp4"
|
||||
entries = [{"path": f"part0001/{C}_camera_left_part0001_pts.csv"}]
|
||||
with pytest.raises(RuntimeError, match="转码视频缺少逐帧对应的 PTS"):
|
||||
normalize._paired_files(video, entries)
|
||||
|
||||
|
||||
def test_arbitrary_capture_names_keep_media_and_calibration_together():
|
||||
for path in ("part0001/business_camera_left_part0001.mp4", "business_calibration_camera.yaml", "part0001/business_imu_part0001.csv"):
|
||||
assert normalize._capture_id(path) == "business"
|
||||
Loading…
x
Reference in New Issue
Block a user