forked from longtao.wu/droboticflow
🐛 Fix part-index delivery: unbounded payload peek, no pose frames, explicit re-delivery for migrated parts
This commit is contained in:
parent
2ff6283a59
commit
0c5f34f79a
@ -840,23 +840,12 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
|
||||
return "described"
|
||||
|
||||
|
||||
def _pose_section(payload: dict[str, Any] | None, kind: str, start: float, end: float) -> dict[str, Any] | None:
|
||||
if payload is None:
|
||||
return None
|
||||
cropped = _crop_pose(payload, start, end, kind)
|
||||
if cropped.get("frameCount", 0) <= 0:
|
||||
return None
|
||||
return cropped
|
||||
|
||||
|
||||
def _part_index_slice(
|
||||
row: dict[str, Any],
|
||||
index: int,
|
||||
source_slice: pathlib.Path,
|
||||
calibration: list[dict[str, Any]],
|
||||
pose_policy: dict[str, Any],
|
||||
headpose: dict[str, Any] | None,
|
||||
handpose: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
declared = str(row.get("deliveryDisposition") or "")
|
||||
slice_id = str(row.get("sliceId") or f"slice-{index + 1:06d}")
|
||||
@ -955,16 +944,15 @@ def _part_index_slice(
|
||||
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
|
||||
"discardedAt": str(evidence.get("discardedAt") or ""),
|
||||
}
|
||||
if pose_policy.get("headpose") == "include":
|
||||
cropped = _pose_section(headpose, "headpose", start, end)
|
||||
if cropped is None:
|
||||
raise RuntimeError(f"HeadPose has no frames for accepted slice {slice_id}")
|
||||
entry.setdefault("pose", {})["headpose"] = cropped
|
||||
if pose_policy.get("handpose") == "include":
|
||||
cropped = _pose_section(handpose, "handpose", start, end)
|
||||
if cropped is None:
|
||||
raise RuntimeError(f"HandPose has no frames for accepted slice {slice_id}")
|
||||
entry.setdefault("pose", {})["handpose"] = cropped
|
||||
# Pose frames deliberately stay out of the index: the customer contract for
|
||||
# this mode is the split information, and embedded per-frame pose would
|
||||
# make a multi-megabyte part.json out of a kilobyte index. The per-slice
|
||||
# poseDelivery decision below tells downstream which segments carry pose
|
||||
# so it can resolve frames from the pinned materialization revision.
|
||||
entry["classification"]["poseDelivery"] = {
|
||||
"headpose": "include" if pose_policy.get("headpose") == "include" else "exclude",
|
||||
"handpose": "include" if pose_policy.get("handpose") == "include" else "exclude",
|
||||
}
|
||||
return entry
|
||||
|
||||
|
||||
@ -975,8 +963,6 @@ def _build_part_index(
|
||||
source_root: pathlib.Path,
|
||||
calibration: list[dict[str, Any]],
|
||||
pose_policy: dict[str, Any],
|
||||
headpose: dict[str, Any] | None,
|
||||
handpose: dict[str, Any] | None,
|
||||
summary_payload: dict[str, Any],
|
||||
final_acceptance: dict[str, Any],
|
||||
capture_record: dict[str, Any],
|
||||
@ -993,7 +979,7 @@ def _build_part_index(
|
||||
source_slice = source_root / pathlib.Path(*relative.parts)
|
||||
if not source_slice.is_dir():
|
||||
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
|
||||
slices.append(_part_index_slice(row, index, source_slice, calibration, pose_policy, headpose, handpose))
|
||||
slices.append(_part_index_slice(row, index, source_slice, calibration, pose_policy))
|
||||
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
|
||||
exported_reviews = [
|
||||
{key: review[key] for key in _QUALITY_REVIEW_EXPORT_FIELDS if key in review}
|
||||
@ -1545,7 +1531,7 @@ def _run_accepted_slice_directory(
|
||||
if _bool(params, "PART_INDEX_DELIVERY", False):
|
||||
calibration_refs = _part_index_calibration_refs(inputs)
|
||||
lineage = {"acceptedSliceDirectoryArtifactId": directory_artifact.get("id"), "humanItemId": revision_manifest.get("humanItemId"), "taskId": revision_manifest.get("taskId"), "sourceRunId": revision_manifest.get("runId")}
|
||||
_, body = _build_part_index(revision_manifest, coverage, kept_slices, source_root, calibration_refs, pose_policy, headpose, handpose, summary_payload, final_acceptance, capture_record, lineage, discarded_slices)
|
||||
_, body = _build_part_index(revision_manifest, coverage, kept_slices, source_root, calibration_refs, pose_policy, summary_payload, final_acceptance, capture_record, lineage, discarded_slices)
|
||||
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
|
||||
delivery_root = pathlib.Path(tempfile.mkdtemp(prefix="delivery-", dir=output)) if disposition_contract else output / "delivery"
|
||||
clips_root = delivery_root / "clips"
|
||||
|
||||
@ -6,11 +6,15 @@ status: accepted
|
||||
|
||||
来源数据在人工流程里已经按任务级切分(切片边界)与动作级切分(动作区间)组织完毕,平台也已经在物化修订里持有全部证据。逐片目录形态把这份信息再物化成一份目录树:为每个保留切片重新编码左右视频、切分 PTS 与 IMU、逐片裁剪姿态、复制标定文件,并为每个文件重算 SHA-256。交付的价值在于**索引与分类**,不在于再搬运一遍字节,因此交付物改为每个 part 一份 `part.json`(`droboticflow.delivery.part-index/v1`),媒体由下游流程按索引解析。打包节点用 `PART_INDEX_DELIVERY`(默认 `false`)在两种形态之间切换,历史形态与其契约保持不变。
|
||||
|
||||
索引只携带切分与分类信息,**不内嵌姿态帧数据**:每片记录 `poseDelivery` 的纳入/排除决定,帧数据由下游从固定的物化修订解析。内嵌逐帧姿态会把千字节的索引变成数兆字节,且与客户对「切分信息」的需求不符。
|
||||
|
||||
已确认的产品决定:
|
||||
|
||||
- 舍弃切片是**一等分类**,随索引交付并携带 `discardEvidence`。此前的对外交付不包含舍弃原因;这一条有意变更,因为下游必须知道某个区间为什么被排除。舍弃前的历史描述仍属于内部审核证据,不进入索引。
|
||||
- 索引不内嵌姿态帧数据(见上)。
|
||||
- 交付范围由已验收的物化修订固定。索引像逐片形态一样校验逐片 `deliveryDisposition` 与 `synchronized-slice.json` 一致,不一致即 fail-closed。
|
||||
- 已交付数据的迁移是**增量**的:只新增索引,不删除既有 `clips/` 与清单。因此两种形态会在同一交付根下共存,消费方按 `manifest.json` 的 `kind`(`part-index` / `directory`)分流。
|
||||
- 已交付数据的迁移是**离线候选产物**:迁移脚本只产出候选 `part.json`,不写对象存储。已带有完成清单的已交付 part **不能原地补发**(worker 的发布通道拒绝改写完成清单);需要索引的已交付 part 通过重新交付该 part 完成。
|
||||
- 因此两种形态会在同一交付根下共存,消费方按 `manifest.json` 的 `kind`(`part-index` / `directory`)分流。
|
||||
|
||||
## 保持不变的安全约束
|
||||
|
||||
|
||||
@ -15,20 +15,13 @@ The package node selects the delivery shape with the `PART_INDEX_DELIVERY` hyper
|
||||
|
||||
The index carries both split levels in one document: the task-level split (`slices[].taskSplit`) and the action-level split (`slices[].segments`). Every reviewed interval is classified exactly once — kept intervals in `slices[]`, discarded intervals in `discardedSlices[]` with their `discardEvidence`. Discarded intervals are now visible to the customer by design; earlier directory deliveries omitted discard reasons entirely.
|
||||
|
||||
`clips/` is absent in this mode, so consumers must branch on the completion manifest `kind` (`part-index` versus `directory`) instead of assuming a directory layout.
|
||||
`clips/` is absent in this mode, so consumers must branch on the completion manifest `kind` (`part-index` versus `directory`) instead of assuming a directory layout. The index carries the per-slice `poseDelivery` include/exclude decision, but never pose frame data itself — frames stay in the pinned materialization revision for downstream resolution.
|
||||
|
||||
## Migrating an already delivered part
|
||||
|
||||
An already published directory delivery keeps its `clips/`, calibration and evidence files. The migration is additive: it produces the missing `part.json` and publishes it through the same frozen destination. Nothing is deleted and no existing member is rewritten.
|
||||
An already published directory delivery keeps its `clips/`, calibration and evidence files. `scripts/part_index_migration.py` produces the **candidate** `part.json` for such a part from a local copy of the delivered directory plus the accepted materialization revision. It reuses the package operator's own builder, so a migrated index is byte-identical to what an online part-index package would have produced for the same evidence — apart from a `lineage.migratedFrom` provenance marker.
|
||||
|
||||
`scripts/part_index_migration.py` produces the candidate index for one part from a local copy of the delivered directory plus the accepted materialization revision. It reuses the package operator's own builder, so a migrated index is byte-identical to what an online part-index package would have produced for the same evidence — apart from a `lineage.migratedFrom` provenance marker.
|
||||
|
||||
Two constraints follow from the shared owner marker:
|
||||
|
||||
- The destination must already carry `.delivery-owner.json`. A directory without an owner marker is rejected rather than adopted, so an arbitrary prefix can never be claimed.
|
||||
- If the part already has a `part.json` with different bytes, the migration refuses instead of overwriting. Resolve the part's delivery state explicitly.
|
||||
|
||||
The tool never writes object storage itself. Publication has to go through the API/Worker path so the ownership marker, conditional create and no-overwrite guarantees stay in effect.
|
||||
The tool never writes object storage, and the worker publication path **cannot publish that candidate into a destination whose completion manifest already exists**: rewriting a customer's completion marker in place is refused (a same-owner, same-bytes retry of an existing part-index manifest is the one accepted idempotent case). A part that was already delivered in the directory form therefore gains the index through an explicit re-delivery of that part, never through an in-place patch of the delivered directory. If the part already has a `part.json` with different bytes, the candidate is refused as well.
|
||||
|
||||
## Safety and persistence
|
||||
|
||||
|
||||
@ -98,7 +98,7 @@ Schema `droboticflow.delivery.part-index/v1`。顶层字段:
|
||||
|
||||
标定只记录引用(名称、路径、SHA-256、字节数),不再复制标定文件。每个保留区间的 `synchronization` 里带同一组标定引用与边界证据(`maxStereoAlignmentErrorUs`、`boundaryContract`、`boundaryTimestampUs`)。
|
||||
|
||||
姿态结果按 `poseDelivery` 决定是否包含:`include` 时逐区间裁剪并挂在 `slices[].pose.headpose` / `slices[].pose.handpose`,`exclude` 时整个字段不出现。原始数据人工版(raw human)不生成姿态产物。
|
||||
姿态帧数据不进索引:每片的 `classification.poseDelivery` 记录头部/手部姿态的纳入/排除决定,帧数据由下游从固定的物化修订解析。纳入姿态不会把逐帧数据带进 `part.json`。原始数据人工版(raw human)不生成姿态产物。
|
||||
|
||||
### 校验
|
||||
|
||||
|
||||
@ -512,6 +512,7 @@ func (server *server) ensureFanyuDeliveryTemplateV58(ctx context.Context, organi
|
||||
"deliveryDirectory",
|
||||
"deliveryManifest",
|
||||
"deliverySummary",
|
||||
"deliveryPartIndex",
|
||||
)
|
||||
refreshed.Data.Inputs = fanyuSetRequiredPorts(refreshed.Data.Inputs, "acceptedSliceDirectory", "acceptanceSummary", "calibrations")
|
||||
refreshed.Data.Inputs = fanyuSetEvidenceCollectionPort(refreshed.Data.Inputs, "headpose", "按验收策略整理的 HeadPose 证据集合")
|
||||
|
||||
@ -7,7 +7,6 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@ -315,19 +314,32 @@ type partIndexDelivery struct {
|
||||
FileName string `json:"fileName"`
|
||||
}
|
||||
|
||||
// partIndexPayloadPeek reads only the routing block. Payload sizes are bounded
|
||||
// by the operator contract; the full document is verified by sha256 on upload.
|
||||
// partIndexPayloadPeek reads only the routing block. A real part index can
|
||||
// embed megabytes of data, so the probe must read the whole document rather
|
||||
// than a fixed prefix; the operator bounds the size through the workspace
|
||||
// file, and the full content is verified by sha256 on upload.
|
||||
const partIndexMaxPeekBytes = 256 << 20
|
||||
|
||||
func partIndexPayloadPeek(path string) (partIndexDelivery, error) {
|
||||
handle, err := os.Open(path)
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return partIndexDelivery{}, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return partIndexDelivery{}, fmt.Errorf("part index payload is not a regular file")
|
||||
}
|
||||
if info.Size() > partIndexMaxPeekBytes {
|
||||
return partIndexDelivery{}, fmt.Errorf("part index payload exceeds %d bytes", partIndexMaxPeekBytes)
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return partIndexDelivery{}, err
|
||||
}
|
||||
defer handle.Close()
|
||||
var probe struct {
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
Delivery *partIndexDelivery `json:"delivery"`
|
||||
}
|
||||
if err = json.NewDecoder(io.LimitReader(handle, 1<<16)).Decode(&probe); err != nil {
|
||||
if err = json.Unmarshal(body, &probe); err != nil {
|
||||
return partIndexDelivery{}, fmt.Errorf("part index payload is not readable JSON: %w", err)
|
||||
}
|
||||
if probe.Delivery == nil || probe.Delivery.Kind != "part_index" {
|
||||
@ -343,6 +355,26 @@ func partIndexPayloadPeek(path string) (partIndexDelivery, error) {
|
||||
return *probe.Delivery, nil
|
||||
}
|
||||
|
||||
// mergePartIndexIntoManifest builds the deterministic completion manifest body
|
||||
// for a single-file part delivery. The body depends only on the delivered
|
||||
// member, so publishBytes' conditional create already distinguishes the two
|
||||
// legitimate existing cases (identical retry → accepted; anything else,
|
||||
// including a legacy directory manifest or a foreign owner → rejected with a
|
||||
// conflict). No in-place manifest mutation is performed: the worker publisher
|
||||
// is write-only, and a completion marker that already describes a delivered
|
||||
// directory tree must never be rewritten into a single-file manifest.
|
||||
func mergePartIndexIntoManifest(_ context.Context, _ objectsync.Publisher, _ domain.StorageConnectionRecord, _ string, _ string, _ string, entries []directoryManifestEntry) ([]byte, error) {
|
||||
return json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "part-index", FileCount: len(entries), TotalBytes: sumManifestBytes(entries), Entries: entries})
|
||||
}
|
||||
|
||||
func sumManifestBytes(entries []directoryManifestEntry) int64 {
|
||||
total := int64(0)
|
||||
for _, entry := range entries {
|
||||
total += entry.SizeBytes
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// publishCaptureDeliveryFile places a single file into the frozen part
|
||||
// destination. It shares every protection of the directory publication — one
|
||||
// owner marker per destination, conditional create, sha256/size verification,
|
||||
@ -382,7 +414,10 @@ func (runtime *Runtime) publishCaptureDeliveryFile(ctx context.Context, run *dom
|
||||
}
|
||||
entries := []directoryManifestEntry{{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}}
|
||||
manifestKey := target.Prefix + "/manifest.json"
|
||||
body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "part-index", FileCount: 1, TotalBytes: meta.SizeBytes, Entries: entries})
|
||||
body, err := mergePartIndexIntoManifest(ctx, publisher, *connection, target.Bucket, manifestKey, owner, entries)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
if err = publishBytes(manifestKey, body); err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
@ -50,6 +51,7 @@ func newPartIndexRuntime(t *testing.T, prefix string) (*Runtime, *deliveryPublis
|
||||
ctx := context.Background()
|
||||
db := store.NewMemoryStore()
|
||||
p := &deliveryPublisher{}
|
||||
p.objects = map[string][]byte{}
|
||||
runtime := &Runtime{store: db, artifactPublisher: p}
|
||||
connection := domain.StorageConnectionRecord{ID: "c", OrganizationID: "org", Provider: domain.StorageProviderBOS, Bucket: "b"}
|
||||
if err := db.UpsertStorageConnection(ctx, connection); err != nil {
|
||||
@ -62,6 +64,22 @@ func newPartIndexRuntime(t *testing.T, prefix string) (*Runtime, *deliveryPublis
|
||||
return runtime, p, record, domain.RunRecord{ID: "run-1", OrganizationID: "org"}
|
||||
}
|
||||
|
||||
func TestPartIndexPayloadPeekAcceptsAMultiMegabyteDocument(t *testing.T) {
|
||||
// A real part index can hold thousands of split entries; the probe must
|
||||
// read the whole document instead of a fixed 64KB prefix, which used to
|
||||
// fail with "unexpected EOF" for any larger payload.
|
||||
filler := strings.Repeat("a", 9<<20) // 9 MiB of content after the routing block
|
||||
payload := "{\"filler\":\"" + filler + "\",\"schemaVersion\":\"droboticflow.delivery.part-index/v1\",\"delivery\":{\"kind\":\"part_index\",\"fileName\":\"part.json\"}}"
|
||||
path := partIndexFixture(t, "part.json", []byte(payload))
|
||||
routing, err := partIndexPayloadPeek(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if routing.Kind != "part_index" || routing.FileName != "part.json" {
|
||||
t.Fatalf("unexpected routing %+v", routing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryPublishesSingleFileAndRetries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
prefix := "deliveries/测试__project/capture/part0001"
|
||||
@ -168,6 +186,48 @@ func TestPartIndexDeliveryRejectsANewOwner(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryRefusesToRewriteADeliveredDirectoryManifest(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
prefix := "deliveries/p__1/capture/part0001"
|
||||
runtime, _, _, run := newPartIndexRuntime(t, prefix)
|
||||
// The destination already carries this owner's completion manifest from a
|
||||
// legacy directory delivery. Migrating that part to the index form in place
|
||||
// would rewrite a customer-visible completion marker, so it must be
|
||||
// rejected: such parts are re-delivered explicitly instead.
|
||||
markDelivered(t, runtime, prefix, run, directoryManifest{SchemaVersion: 1, Kind: "directory", FileCount: 2, TotalBytes: 10, Entries: []directoryManifestEntry{{Path: "clips/slice-000001/left.mp4", Key: prefix + "/clips/slice-000001/left.mp4", SizeBytes: 5, SHA256: strings.Repeat("0", 64)}}})
|
||||
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
|
||||
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
|
||||
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err == nil {
|
||||
t.Fatal("an already delivered directory manifest was rewritten for a part index")
|
||||
}
|
||||
}
|
||||
|
||||
func markDelivered(t *testing.T, runtime *Runtime, prefix string, run domain.RunRecord, manifest directoryManifest) {
|
||||
t.Helper()
|
||||
// Publish through the real code path so the manifest carries the delivery
|
||||
// owner metadata the conditional create later compares against.
|
||||
ctx := context.Background()
|
||||
record, err := runtime.store.GetCaptureRecord(ctx, "capture")
|
||||
if err != nil || record == nil {
|
||||
t.Fatalf("capture record: %v", err)
|
||||
}
|
||||
connection, err := runtime.store.GetStorageConnection(ctx, record.Delivery.ConnectionID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publisher := runtime.outputArtifactPublisher()
|
||||
body, _ := json.Marshal(manifest)
|
||||
sum := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(sum[:])
|
||||
stage := "system/runs/" + run.ID + "/delivery-staging/legacy/" + sha
|
||||
if _, err = publisher.PutInternal(ctx, *connection, record.Delivery.Bucket, stage, strings.NewReader(string(body)), int64(len(body)), "application/json", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = publishDeliveryObject(ctx, publisher, *connection, record.Delivery.Bucket, stage, prefix+"/manifest.json", "legacy-owner", sha, int64(len(body))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryRefusesToAdoptAnUnownedDirectory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
prefix := "deliveries/p__1/capture/part0001"
|
||||
|
||||
@ -840,23 +840,12 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
|
||||
return "described"
|
||||
|
||||
|
||||
def _pose_section(payload: dict[str, Any] | None, kind: str, start: float, end: float) -> dict[str, Any] | None:
|
||||
if payload is None:
|
||||
return None
|
||||
cropped = _crop_pose(payload, start, end, kind)
|
||||
if cropped.get("frameCount", 0) <= 0:
|
||||
return None
|
||||
return cropped
|
||||
|
||||
|
||||
def _part_index_slice(
|
||||
row: dict[str, Any],
|
||||
index: int,
|
||||
source_slice: pathlib.Path,
|
||||
calibration: list[dict[str, Any]],
|
||||
pose_policy: dict[str, Any],
|
||||
headpose: dict[str, Any] | None,
|
||||
handpose: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
declared = str(row.get("deliveryDisposition") or "")
|
||||
slice_id = str(row.get("sliceId") or f"slice-{index + 1:06d}")
|
||||
@ -955,16 +944,15 @@ def _part_index_slice(
|
||||
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
|
||||
"discardedAt": str(evidence.get("discardedAt") or ""),
|
||||
}
|
||||
if pose_policy.get("headpose") == "include":
|
||||
cropped = _pose_section(headpose, "headpose", start, end)
|
||||
if cropped is None:
|
||||
raise RuntimeError(f"HeadPose has no frames for accepted slice {slice_id}")
|
||||
entry.setdefault("pose", {})["headpose"] = cropped
|
||||
if pose_policy.get("handpose") == "include":
|
||||
cropped = _pose_section(handpose, "handpose", start, end)
|
||||
if cropped is None:
|
||||
raise RuntimeError(f"HandPose has no frames for accepted slice {slice_id}")
|
||||
entry.setdefault("pose", {})["handpose"] = cropped
|
||||
# Pose frames deliberately stay out of the index: the customer contract for
|
||||
# this mode is the split information, and embedded per-frame pose would
|
||||
# make a multi-megabyte part.json out of a kilobyte index. The per-slice
|
||||
# poseDelivery decision below tells downstream which segments carry pose
|
||||
# so it can resolve frames from the pinned materialization revision.
|
||||
entry["classification"]["poseDelivery"] = {
|
||||
"headpose": "include" if pose_policy.get("headpose") == "include" else "exclude",
|
||||
"handpose": "include" if pose_policy.get("handpose") == "include" else "exclude",
|
||||
}
|
||||
return entry
|
||||
|
||||
|
||||
@ -975,8 +963,6 @@ def _build_part_index(
|
||||
source_root: pathlib.Path,
|
||||
calibration: list[dict[str, Any]],
|
||||
pose_policy: dict[str, Any],
|
||||
headpose: dict[str, Any] | None,
|
||||
handpose: dict[str, Any] | None,
|
||||
summary_payload: dict[str, Any],
|
||||
final_acceptance: dict[str, Any],
|
||||
capture_record: dict[str, Any],
|
||||
@ -993,7 +979,7 @@ def _build_part_index(
|
||||
source_slice = source_root / pathlib.Path(*relative.parts)
|
||||
if not source_slice.is_dir():
|
||||
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
|
||||
slices.append(_part_index_slice(row, index, source_slice, calibration, pose_policy, headpose, handpose))
|
||||
slices.append(_part_index_slice(row, index, source_slice, calibration, pose_policy))
|
||||
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
|
||||
exported_reviews = [
|
||||
{key: review[key] for key in _QUALITY_REVIEW_EXPORT_FIELDS if key in review}
|
||||
@ -1545,7 +1531,7 @@ def _run_accepted_slice_directory(
|
||||
if _bool(params, "PART_INDEX_DELIVERY", False):
|
||||
calibration_refs = _part_index_calibration_refs(inputs)
|
||||
lineage = {"acceptedSliceDirectoryArtifactId": directory_artifact.get("id"), "humanItemId": revision_manifest.get("humanItemId"), "taskId": revision_manifest.get("taskId"), "sourceRunId": revision_manifest.get("runId")}
|
||||
_, body = _build_part_index(revision_manifest, coverage, kept_slices, source_root, calibration_refs, pose_policy, headpose, handpose, summary_payload, final_acceptance, capture_record, lineage, discarded_slices)
|
||||
_, body = _build_part_index(revision_manifest, coverage, kept_slices, source_root, calibration_refs, pose_policy, summary_payload, final_acceptance, capture_record, lineage, discarded_slices)
|
||||
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
|
||||
delivery_root = pathlib.Path(tempfile.mkdtemp(prefix="delivery-", dir=output)) if disposition_contract else output / "delivery"
|
||||
clips_root = delivery_root / "clips"
|
||||
|
||||
@ -168,8 +168,6 @@ def build_index(delivery_root: pathlib.Path, revision_root: pathlib.Path) -> tup
|
||||
revision_root,
|
||||
delivered_calibrations(delivery_root, operator),
|
||||
resolve_pose_policy(acceptance),
|
||||
None,
|
||||
None,
|
||||
summary_payload,
|
||||
acceptance.get("finalAcceptance") or {},
|
||||
capture_record,
|
||||
|
||||
@ -98,6 +98,41 @@ def test_part_index_is_a_single_json_delivery_with_both_split_levels(tmp_path):
|
||||
assert first["classification"]["descriptionStatus"] == "invalid_only"
|
||||
|
||||
|
||||
def test_pose_frames_stay_out_of_the_part_index(tmp_path):
|
||||
"""The index carries the pose delivery decision, never the frame bytes.
|
||||
|
||||
A include decision must not pull per-frame pose into part.json: a real
|
||||
segment would otherwise embed megabytes of frames into a kilobyte index.
|
||||
"""
|
||||
inputs = fixture(tmp_path, ["keep"])
|
||||
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "include", "handpose": "include"}
|
||||
pose = {
|
||||
"schemaVersion": "droboticflow.headpose/v1",
|
||||
"enabled": True,
|
||||
"frames": [
|
||||
{"timestampSeconds": 0.04, "frameIndex": 1, "values": [0.1, 0.2, 0.3, 0.4]},
|
||||
{"timestampSeconds": 0.08, "frameIndex": 2, "values": [0.1, 0.2, 0.3, 0.4]},
|
||||
],
|
||||
}
|
||||
hand = {
|
||||
"schemaVersion": "droboticflow.wilor.hand_pose_result/v1",
|
||||
"enabled": True,
|
||||
"result": {"frames": [{"timestampSeconds": 0.04, "sourceFrame": 1}]},
|
||||
}
|
||||
inputs.append({"id": "headpose", "portId": "headpose", "type": "json", "data": pose})
|
||||
inputs.append({"id": "handpose", "portId": "handpose", "type": "json", "data": hand})
|
||||
artifacts = package.run("r", "p", tmp_path / "output", PART_NODE, inputs, helpers())
|
||||
index = read_index(artifacts[0])
|
||||
entry = index["slices"][0]
|
||||
# The decision is exported for downstream resolution.
|
||||
assert entry["classification"]["poseDelivery"] == {"headpose": "include", "handpose": "include"}
|
||||
# The frames themselves never enter the document.
|
||||
assert "pose" not in entry
|
||||
body = Path(artifacts[0]["path"]).read_text(encoding="utf-8")
|
||||
assert "timestampSeconds" not in body and "frameIndex" not in body
|
||||
assert index["poseDelivery"] == {"headpose": "include", "handpose": "include"}
|
||||
|
||||
|
||||
def test_discarded_slices_are_delivered_as_their_own_classification(tmp_path):
|
||||
_, artifacts = run_part_index(tmp_path, ["keep", "discard"])
|
||||
index = read_index(artifacts[0])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user