🧾 Deliver one part index JSON instead of sliced directories

Add a PART_INDEX_DELIVERY mode to the package operator (default off): a part
delivers a single part.json carrying both split levels, the task-level slice
boundaries and the action-level segments, plus a classification for every
reviewed interval. Kept intervals land in slices[], discarded intervals in
discardedSlices[] with their discard evidence, which deliberately makes the
discard reason part of the customer deliverable. Media, PTS, IMU, pose and
calibration bytes are no longer published; downstream packaging resolves them
from the pinned materialization revision.

The worker publishes the index through the same frozen-destination path as the
directory form. Extract prepareDeliveryTarget so single-file and directory
publication share one owner marker, overlap guards, delegated-scope check,
conditional create, sha256 verification, idempotent retry and the refusal to
overwrite another delivery. A part index also refuses to adopt a destination
that holds files without an owner marker, so additive migration cannot claim a
legacy directory by accident.

The API and download control plane recognise the new schema, project the split
classification for the browser, and treat a part index as a first-class final
delivery. Cross-language contract tests read the operator's own schema and
field literals so a one-sided rename fails a test.

Verification: 10 operator tests, 8 worker tests and 3 API tests for the new
behaviour pass. Failure sets match the HEAD baseline name-for-name for
internal/api (75), internal/worker (4), tests/backend and the frontend suite;
no regression. The end-to-end local replica stack was not exercised: it needs
image builds and the opt-in Go materialization fixture, so the Go to operator
handoff is covered by the contract tests instead.
This commit is contained in:
d-robotics 2026-10-09 00:28:50 -07:00
parent dfe61d99bb
commit 7b5604f9e0
17 changed files with 1996 additions and 62 deletions

View File

@ -661,6 +661,392 @@ def _collect_unique_package_artifacts(
return artifacts
_PART_INDEX_SCHEMA = "droboticflow.delivery.part-index/v1"
_PART_INDEX_ARTIFACT = "delivery.part-index"
_PART_INDEX_FILE = "part.json"
# Only these review fields leave the platform. Reviewer identity, free-text
# quality comments and unfiltered acceptance statistics stay internal.
_QUALITY_REVIEW_EXPORT_FIELDS = ("taskId", "revision", "decision", "reviewerUserId", "createdAt")
_ACCEPTANCE_EXPORT_FIELDS = (
"confirmed",
"sampled",
"samplingDecision",
"samplingProbability",
"samplingPolicyRevision",
"samplingRandomValue",
"samplingDecidedAt",
"acceptedAt",
"confirmedByUserId",
"processingProfile",
"poseDelivery",
)
def _optional_float(value: Any) -> float | None:
if value is None or isinstance(value, bool):
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def _part_index_capture_record(summary_payload: dict[str, Any], revision_manifest: dict[str, Any]) -> dict[str, Any]:
if isinstance(summary_payload.get("capture"), dict):
return summary_payload["capture"]
if isinstance(revision_manifest.get("capture"), dict):
return revision_manifest["capture"]
return {}
def _part_index_discarded_slice(
row: dict[str, Any],
source_root: pathlib.Path,
) -> dict[str, Any]:
"""Classify a discarded interval without shipping its media or old text.
History of prior descriptions stays internal: the acceptance contract keeps
it as review evidence, not as part of the external deliverable.
"""
slice_id = str(row.get("sliceId") or "")
declared = str(row.get("deliveryDisposition") or "")
evidence = row.get("discardEvidence") if isinstance(row.get("discardEvidence"), dict) else {}
if not str(evidence.get("reason") or "").strip():
raise RuntimeError(f"discarded slice {slice_id} has no reason evidence")
relative = pathlib.PurePosixPath(str(row.get("directory") or ""))
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
raise RuntimeError("unsafe accepted slice directory")
source_slice = source_root / pathlib.Path(*relative.parts)
if not source_slice.is_dir():
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
synchronization_path = source_slice / "synchronized-slice.json"
if not synchronization_path.is_file():
raise RuntimeError(f"accepted slice {slice_id} has no synchronized-slice.json")
try:
synchronization = json.loads(synchronization_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError) as error:
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not valid JSON") from error
if not isinstance(synchronization, dict):
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not an object")
recorded = str(synchronization.get("deliveryDisposition") or declared)
if recorded != declared:
raise RuntimeError(f"accepted slice {slice_id} disposition disagrees with its synchronization evidence")
start = _optional_float(row.get("startSecond")) or 0.0
end = _optional_float(row.get("endSecond")) or 0.0
return {
"sliceId": slice_id,
"sliceName": str(row.get("sliceName") or ""),
"taskSplit": {
"startSecond": start,
"endSecond": end,
"startTimestampUs": int(row["startTimestampUs"]) if row.get("startTimestampUs") is not None else None,
"endTimestampUs": int(row["endTimestampUs"]) if row.get("endTimestampUs") is not None else None,
"semantics": "half-open",
},
"classification": {
"deliveryDisposition": declared,
"discardEvidence": {
"reason": str(evidence.get("reason") or ""),
"reasonLabelId": str(evidence.get("reasonLabelId") or ""),
"reasonName": str(evidence.get("reasonName") or ""),
"note": str(evidence.get("note") or ""),
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
"discardedAt": str(evidence.get("discardedAt") or ""),
},
},
"segments": [],
}
def _part_index_calibration_refs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Reference calibration honestly instead of copying it into a delivery tree.
The part index records where calibration lives and its checksum; the bytes
stay in the accepted materialization revision that this index pins.
"""
refs: list[dict[str, Any]] = []
seen: set[str] = set()
for artifact in _sidecar_artifacts(inputs, {"calibrations"}):
name = str(artifact.get("name") or "").strip()
if not name or name in seen:
continue
seen.add(name)
path = str(artifact.get("path") or "").strip()
digest = str(artifact.get("sha256") or (artifact.get("metadata") or {}).get("sha256") or "").strip().lower()
if path and not _SHA.match(digest):
try:
digest = _sha256_file(pathlib.Path(path))
except OSError as error:
raise RuntimeError(f"calibration file is unavailable: {name}") from error
if not _SHA.match(digest):
raise RuntimeError(f"calibration file has no verifiable checksum: {name}")
size_bytes = int(artifact.get("sizeBytes") or 0)
if path and not size_bytes:
try:
size_bytes = pathlib.Path(path).stat().st_size
except OSError:
size_bytes = 0
refs.append({"name": name, "path": path, "sha256": digest, "bytes": size_bytes})
if not refs:
raise RuntimeError("part index delivery requires camera/IMU calibration files")
return refs
def _emit_part_index(
run_id: str,
node_id: str,
output: pathlib.Path,
node_data: dict[str, Any],
helpers: HelperMap,
payload_bytes: bytes,
) -> list[dict[str, Any]]:
"""Emit the part index as a single file artifact.
The artifact path is the deliverable itself: the worker publishes exactly
this file into the frozen capture delivery destination. The directory is
only a private staging area so a concurrently written node output cannot be
captured by accident. Byte content never carries a build timestamp, so a
retry of the same accepted revision republishes identical bytes.
"""
staging = output / "part-index"
staging.mkdir(parents=True, exist_ok=True)
path = staging / _PART_INDEX_FILE
path.write_bytes(payload_bytes)
return [
helpers["build_artifact"](
run_id,
node_id,
path,
"json",
metadata={"mimeType": "application/json", "sha256": _sha256_file(path)},
output=_output(node_data, helpers, "deliveryPartIndex"),
)
]
def _description_status(actions: list[dict[str, Any]]) -> str:
"""Describe a kept slice without reintroducing the internal annotation state.
``empty_confirmed`` keeps its platform meaning: no description was written
and the submitter confirmed that as a deliberate semantic blank.
"""
if not actions:
return "empty_confirmed"
if all(bool(action.get("isInvalid")) for action in actions):
return "invalid_only"
return "described"
def _pose_section(payload: dict[str, Any] | None, kind: str, start: float, end: float) -> dict[str, Any] | None:
if payload is None:
return None
cropped = _crop_pose(payload, start, end, kind)
if cropped.get("frameCount", 0) <= 0:
return None
return cropped
def _part_index_slice(
row: dict[str, Any],
index: int,
source_slice: pathlib.Path,
calibration: list[dict[str, Any]],
pose_policy: dict[str, Any],
headpose: dict[str, Any] | None,
handpose: dict[str, Any] | None,
) -> dict[str, Any]:
declared = str(row.get("deliveryDisposition") or "")
slice_id = str(row.get("sliceId") or f"slice-{index + 1:06d}")
start = _optional_float(row.get("startSecond")) or 0.0
end = _optional_float(row.get("endSecond")) or 0.0
annotation_path = source_slice / "annotation.json"
if not annotation_path.is_file():
raise RuntimeError(f"accepted slice {slice_id} has no annotation.json")
try:
annotation = json.loads(annotation_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError) as error:
raise RuntimeError(f"accepted slice {slice_id} annotation.json is not valid JSON") from error
if not isinstance(annotation, dict):
raise RuntimeError(f"accepted slice {slice_id} annotation.json is not an object")
raw_actions = annotation.get("actions")
if raw_actions is None:
raw_actions = annotation.get("descriptions")
if raw_actions is None:
raw_actions = []
if not isinstance(raw_actions, list):
raise RuntimeError(f"accepted slice {slice_id} annotation actions are not a list")
actions: list[dict[str, Any]] = []
for action in raw_actions:
if not isinstance(action, dict):
raise RuntimeError(f"accepted slice {slice_id} has a malformed action segment")
start_us = action.get("startTimestampUs")
end_us = action.get("endTimestampUs")
actions.append({
"id": str(action.get("id") or ""),
"description": str(action.get("description") or ""),
"status": str(action.get("status") or ""),
"isInvalid": bool(action.get("isInvalid", False)),
"startTimestampUs": int(start_us) if start_us is not None else None,
"endTimestampUs": int(end_us) if end_us is not None else None,
"startSecond": _optional_float(action.get("clipStartMs")) / 1000.0 if _optional_float(action.get("clipStartMs")) is not None else None,
"endSecond": _optional_float(action.get("clipEndMs")) / 1000.0 if _optional_float(action.get("clipEndMs")) is not None else None,
"clipStartMs": _optional_float(action.get("clipStartMs")),
"clipEndMs": _optional_float(action.get("clipEndMs")),
"labelId": str(action.get("descriptionTemplateId") or ""),
"phrases": [str(item) for item in action.get("phrases", [])] if isinstance(action.get("phrases"), list) else [],
"source": str(action.get("source") or ""),
"confidence": _optional_float(action.get("confidence")),
})
synchronization_path = source_slice / "synchronized-slice.json"
if not synchronization_path.is_file():
raise RuntimeError(f"accepted slice {slice_id} has no synchronized-slice.json")
try:
synchronization = json.loads(synchronization_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError) as error:
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not valid JSON") from error
if not isinstance(synchronization, dict):
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not an object")
# A disposition that disagrees with the pinned review revision must fail
# closed instead of silently exporting the wrong classification.
recorded = str(synchronization.get("deliveryDisposition") or declared)
if recorded != declared:
raise RuntimeError(f"accepted slice {slice_id} disposition disagrees with its synchronization evidence")
entry: dict[str, Any] = {
"sliceId": slice_id,
"sliceName": str(row.get("sliceName") or ""),
"taskSplit": {
"startSecond": start,
"endSecond": end,
"startTimestampUs": int(row["startTimestampUs"]) if row.get("startTimestampUs") is not None else None,
"endTimestampUs": int(row["endTimestampUs"]) if row.get("endTimestampUs") is not None else None,
"semantics": "half-open",
},
"classification": {
"deliveryDisposition": declared,
"taskLabel": str(synchronization.get("taskLabel") or ""),
"taskLabelId": str(synchronization.get("taskLabelId") or ""),
"descriptionStatus": _description_status(actions),
},
"segments": actions,
"synchronization": {
"maxStereoAlignmentErrorUs": synchronization.get("maxStereoAlignmentErrorUs"),
"boundaryContract": str(synchronization.get("boundaryContract") or ""),
"boundaryTimestampUs": [int(value) for value in synchronization.get("boundaryTimestampUs", [])] if isinstance(synchronization.get("boundaryTimestampUs"), list) else [],
"calibration": [
{"name": item["name"], "path": item["path"], "sha256": item["sha256"]}
for item in calibration
],
},
}
if declared == "discard":
evidence = row.get("discardEvidence") if isinstance(row.get("discardEvidence"), dict) else {}
if not str(evidence.get("reason") or "").strip():
raise RuntimeError(f"discarded slice {slice_id} has no reason evidence")
# Discard classification is exported deliberately: downstream packaging
# must know why an interval is excluded.
entry["classification"]["discardEvidence"] = {
"reason": str(evidence.get("reason") or ""),
"reasonLabelId": str(evidence.get("reasonLabelId") or ""),
"reasonName": str(evidence.get("reasonName") or ""),
"note": str(evidence.get("note") or ""),
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
"discardedAt": str(evidence.get("discardedAt") or ""),
}
if pose_policy.get("headpose") == "include":
cropped = _pose_section(headpose, "headpose", start, end)
if cropped is None:
raise RuntimeError(f"HeadPose has no frames for accepted slice {slice_id}")
entry.setdefault("pose", {})["headpose"] = cropped
if pose_policy.get("handpose") == "include":
cropped = _pose_section(handpose, "handpose", start, end)
if cropped is None:
raise RuntimeError(f"HandPose has no frames for accepted slice {slice_id}")
entry.setdefault("pose", {})["handpose"] = cropped
return entry
def _build_part_index(
revision_manifest: dict[str, Any],
coverage: dict[str, Any],
kept_slices: list[dict[str, Any]],
source_root: pathlib.Path,
calibration: list[dict[str, Any]],
pose_policy: dict[str, Any],
headpose: dict[str, Any] | None,
handpose: dict[str, Any] | None,
summary_payload: dict[str, Any],
final_acceptance: dict[str, Any],
capture_record: dict[str, Any],
lineage: dict[str, Any],
discarded_slices: list[dict[str, Any]],
) -> tuple[dict[str, Any], bytes]:
slices: list[dict[str, Any]] = []
for index, row in enumerate(kept_slices):
if not isinstance(row, dict):
raise RuntimeError("invalid accepted slice entry")
relative = pathlib.PurePosixPath(str(row.get("directory") or ""))
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
raise RuntimeError("unsafe accepted slice directory")
source_slice = source_root / pathlib.Path(*relative.parts)
if not source_slice.is_dir():
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
slices.append(_part_index_slice(row, index, source_slice, calibration, pose_policy, headpose, handpose))
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
exported_reviews = [
{key: review[key] for key in _QUALITY_REVIEW_EXPORT_FIELDS if key in review}
for review in quality_reviews
if isinstance(review, dict)
]
payload: dict[str, Any] = {
"schemaVersion": _PART_INDEX_SCHEMA,
"datasetType": "fanyu-accepted-slice-part-index",
# The worker publishes this file unchanged into the part's frozen
# delivery destination and needs stable routing facts, not a filename
# convention it has to re-derive.
"delivery": {
"kind": "part_index",
"fileName": _PART_INDEX_FILE,
"artifactName": _PART_INDEX_ARTIFACT,
},
"capture": capture_record,
"revision": revision_manifest.get("revision"),
"coverage": coverage,
"slices": slices,
"discardedSlices": [
_part_index_discarded_slice(row, source_root)
for row in discarded_slices
if isinstance(row, dict)
],
"deliveryClassification": {
"keptSliceCount": len(slices),
"discardedSliceCount": len(discarded_slices),
"keptDurationSeconds": sum(float(item["taskSplit"]["endSecond"]) - float(item["taskSplit"]["startSecond"]) for item in slices),
"discardedDurationSeconds": sum(
(_optional_float(row.get("endSecond")) or 0.0) - (_optional_float(row.get("startSecond")) or 0.0)
for row in discarded_slices
if isinstance(row, dict)
),
},
"calibration": calibration,
"poseDelivery": pose_policy,
"acceptance": {
"sampling": {
"sampled": bool(final_acceptance.get("sampled", True)),
"decision": str(final_acceptance.get("samplingDecision") or "selected"),
"probability": final_acceptance.get("samplingProbability"),
"policyRevision": final_acceptance.get("samplingPolicyRevision"),
"randomValue": final_acceptance.get("samplingRandomValue"),
"decidedAt": final_acceptance.get("samplingDecidedAt"),
},
"finalAcceptance": {key: final_acceptance[key] for key in _ACCEPTANCE_EXPORT_FIELDS if key in final_acceptance},
"qualityReviews": exported_reviews,
},
"lineage": lineage,
}
body = (json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8")
return payload, body
def _copy_calibrations(inputs: list[dict[str, Any]], packages_root: pathlib.Path, output: pathlib.Path, helpers: HelperMap) -> list[dict[str, Any]]:
destination = packages_root / "calibration"
result: list[dict[str, Any]] = []
@ -1107,6 +1493,7 @@ def _run_accepted_slice_directory(
summary_artifact = next((item for item in inputs if _port(item) == "acceptanceSummary"), None)
summary_payload = summary_artifact.get("data") if isinstance((summary_artifact or {}).get("data"), dict) else {}
final_acceptance = summary_payload.get("finalAcceptance") if isinstance(summary_payload.get("finalAcceptance"), dict) else {}
capture_record = _part_index_capture_record(summary_payload, revision_manifest)
all_slices = revision_manifest.get("items", [])
if not isinstance(all_slices, list) or not all_slices:
raise RuntimeError("accepted slice manifest has no review evidence")
@ -1151,6 +1538,12 @@ def _run_accepted_slice_directory(
payloads = _payload_tree(_payloads_with_full_pose_manifests(inputs, output, helpers))
headpose = _first_payload(payloads, "droboticflow.headpose/v1")
handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1")
params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {}
if _bool(params, "PART_INDEX_DELIVERY", False):
calibration_refs = _part_index_calibration_refs(inputs)
lineage = {"acceptedSliceDirectoryArtifactId": directory_artifact.get("id"), "humanItemId": revision_manifest.get("humanItemId"), "taskId": revision_manifest.get("taskId"), "sourceRunId": revision_manifest.get("runId")}
_, body = _build_part_index(revision_manifest, coverage, kept_slices, source_root, calibration_refs, pose_policy, headpose, handpose, summary_payload, final_acceptance, capture_record, lineage, discarded_slices)
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
delivery_root = pathlib.Path(tempfile.mkdtemp(prefix="delivery-", dir=output)) if disposition_contract else output / "delivery"
clips_root = delivery_root / "clips"
clips_root.mkdir(parents=True, exist_ok=True)
@ -1241,7 +1634,6 @@ def _run_accepted_slice_directory(
"randomValue": final_acceptance.get("samplingRandomValue"),
"decidedAt": final_acceptance.get("samplingDecidedAt"),
}
capture_record = summary_payload.get("capture") if isinstance(summary_payload.get("capture"), dict) else revision_manifest.get("capture") if isinstance(revision_manifest.get("capture"), dict) else {}
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
if disposition_contract:
quality_reviews = [{k:r[k] for k in ("taskId","revision","decision","reviewerUserId","createdAt") if k in r} for r in quality_reviews if isinstance(r,dict)]
@ -1380,6 +1772,12 @@ def _run_multiport(
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
if _output(node_data, helpers, "packageDirectory") is not None or _output(node_data, helpers, "deliveryDirectory") is not None:
# A part-index delivery intentionally has no directory port; it must still
# reach the accepted-slice path instead of the legacy inline contract.
if (
_output(node_data, helpers, "packageDirectory") is not None
or _output(node_data, helpers, "deliveryDirectory") is not None
or _output(node_data, helpers, "deliveryPartIndex") is not None
):
return _run_multiport(run_id, node_id, output_dir, node_data, inputs, helpers)
return _run_existing_contract(run_id, node_id, output_dir, node_data, inputs, helpers)

View File

@ -0,0 +1,29 @@
---
status: accepted
---
# part 交付以单份总 JSON 索引为交付物,媒体交由下游流程按索引打包
来源数据在人工流程里已经按任务级切分(切片边界)与动作级切分(动作区间)组织完毕,平台也已经在物化修订里持有全部证据。逐片目录形态把这份信息再物化成一份目录树:为每个保留切片重新编码左右视频、切分 PTS 与 IMU、逐片裁剪姿态、复制标定文件,并为每个文件重算 SHA-256。交付的价值在于**索引与分类**,不在于再搬运一遍字节,因此交付物改为每个 part 一份 `part.json`(`droboticflow.delivery.part-index/v1`),媒体由下游流程按索引解析。打包节点用 `PART_INDEX_DELIVERY`(默认 `false`)在两种形态之间切换,历史形态与其契约保持不变。
已确认的产品决定:
- 舍弃切片是**一等分类**,随索引交付并携带 `discardEvidence`。此前的对外交付不包含舍弃原因;这一条有意变更,因为下游必须知道某个区间为什么被排除。舍弃前的历史描述仍属于内部审核证据,不进入索引。
- 交付范围由已验收的物化修订固定。索引像逐片形态一样校验逐片 `deliveryDisposition` 与 `synchronized-slice.json` 一致,不一致即 fail-closed。
- 已交付数据的迁移是**增量**的:只新增索引,不删除既有 `clips/` 与清单。因此两种形态会在同一交付根下共存,消费方按 `manifest.json` 的 `kind`(`part-index` / `directory`)分流。
## 保持不变的安全约束
单文件交付不降低交付安全性,`prepareDeliveryTarget` 是两种形态共用的前置校验:冻结目标校验、与其他 Capture 目标及启用监听目录的重叠检测、委派组织范围校验、`system/runs/` 暂存、`.delivery-owner.json` 条件创建与归属校验、SHA-256/大小元数据校验、幂等重试,以及「绝不覆盖他人交付」的冲突拒绝。两种形态共用同一份归属标记,因此一个 part 不会被两次不同交付写入。
索引内容不含构建时间,同一已验收修订重新打包产出逐字节相同的文件,使重试天然幂等。
## 被拒绝的替代方案
- **按区间列表声明交付范围、视频仍为整段 part 视频**:客户可见的交付视频里会含未交付区间,与「舍弃区间不出现在交付物里」的既有语义冲突。
- **索引与逐片媒体并存于同一 part 目录**:等于付出两次物化成本而收益不变;且逐片形态已能表达同一份信息。
- **让算子直接写冻结交付目标**:算子不持有交付连接与凭据,会绕过 Worker 的条件创建与归属校验。
## 审核层不受影响
物化修订里的逐片视频仍是质检与验收工作台的输入(`MaterializedSliceWorkbench`、`FinalAcceptanceWorkspace` 依赖 `slices/slice-*/left.mp4`)。本决定只改变**交付层**的输出形态,不改变内部审核层。

View File

@ -9,7 +9,7 @@
- 质检和验收均保留全部切片、视频、PTS/IMU和舍弃原因;按原流程打回和重新审核。不改变抽检比例或既有角色权限。最终验收继续保持匿名,操作人身份保存在内部审计记录中。
- “处理覆盖”包含保留和舍弃区间,仍必须完整、无重叠地覆盖统一帧范围。“交付范围”仅包含保留区间,不把正常舍弃造成的交付间隔误报成漏标。
- 当前描述型工作台(模板 v6 起)允许保留切片部分或全部语义留空。提交任务即确认空白,不要求逐片标记无效;没有描述的保留切片以 `empty_confirmed` 保存(仅表示确认没有语义描述),不改变交付用途。已有描述仍须有文本、位于所属切片、无重叠且对齐统一帧边界;未保存的编辑仍须保存或取消。旧版任务标签流程的完成规则不变。
- 留空仍经过原有质检、验收,保留全部视频及同步文件;`annotation.json` 只记录实际填写的描述,全片留空时 `actions` / `descriptions` 为 `[]`。不会自动补无效片段,不会把未标注区间从交付视频中裁掉。
- 留空仍经过原有质检、验收,保留全部视频及同步文件;`annotation.json` 只记录实际填写的描述,全片留空时 `actions` / `descriptions` 为 `[]`。不会自动补无效片段,不会把未标注区间从交付视频中裁掉。总 JSON 交付形态不发布视频,“不裁掉”体现为未交付区间的证据仍留在已验收的物化版本里。
- 新增描述从当前播放头开始,不自动回填前面的空白。已有动作描述(包括最后一条)可删除并保留空白,不移动相邻描述;这与“删除切片并合并范围”是两种不同操作。只有真正首尾相接的动作才显示共用分界拖动手柄。
- 全部舍弃仍需完成原有审核。完成后状态为“无可交付片段”,不创建客户交付目录或空包,交付包记录没有可导出文件;父/子 Run 正常结束。
- 有舍弃/恢复历史的任务保留此前内部审核版本和生成证据,增加相应存储占用;不新增历史版本选择器,不改变项目删除权限或既有项目删除范围。
@ -22,12 +22,14 @@
## 打包与兼容
- 含舍弃的内部清单使用 `droboticflow.slice-materialization/v3`,仍包含所有审核切片;交付使用 `droboticflow.delivery.manifest/v6`,只列保留项。
- Worker 将已审核的物化版本和清单 SHA-256传给打包算子。打包必须匹配该版本,不接受更改后的清单或未知用途。
- 每次新契约打包在独立临时目录构建,仅复制保留切片的白名单文件;左右视频、PTS、IMU、描述和姿态文件一致过滤。舍弃原因和历史描述不进入对外交付文件。
- 发布仍使用原有指定交付根目录和项目/采集目录结构;已有交付目标冲突不会自动覆盖或删除历史文件。
- 含舍弃的内部清单使用 `droboticflow.slice-materialization/v3`,仍包含所有审核切片;逐片目录交付使用 `droboticflow.delivery.manifest/v6`,只列保留项;总 JSON 交付使用 `droboticflow.delivery.part-index/v1`,保留与舍弃区间分别落在 `slices[]` 与 `discardedSlices[]`。
- Worker 将已审核的物化版本和清单 SHA-256传给打包算子。打包必须匹配该版本,不接受更改后的清单或未知用途。两种交付形态都校验逐片 `deliveryDisposition` 与 `synchronized-slice.json` 一致,不一致即 fail-closed。
- 逐片目录形态:每次新契约打包在独立临时目录构建,仅复制保留切片的白名单文件;左右视频、PTS、IMU、描述和姿态文件一致过滤。舍弃原因和历史描述不进入对外交付文件。
- 总 JSON 形态(`PART_INDEX_DELIVERY=true`):交付物只有一份 `part.json`,媒体不随交付发布。舍弃区间作为一等分类随索引交付,**携带 `discardEvidence`(原因、原因字典名、备注、操作人、时间)**;舍弃前的历史描述仍留在平台内部审核记录,不进入索引。
- 发布仍使用原有指定交付根目录和项目/采集目录结构;已有交付目标冲突不会自动覆盖或删除历史文件。两种形态共用同一份 `.delivery-owner.json` 归属标记,因此同一 part 不会被两次不同交付写入。
- 总 JSON 形态拒绝接管「有文件但没有归属标记」的目标目录:向已有历史目录追加索引前,该目录必须已带归属标记,否则拒绝发布而不是静默认领。
- 全保留的旧清单继续走原有 v1/v2兼容逻辑。原始人工和带前置处理工作流都保留原能力。
- Fanyu Worker 的部署配置 `FANYU_SLICE_DISCARD_PACKAGE_IMAGE` 指定兼容新旧契约的 digest-pinned 打包镜像,仅覆盖 builtin package 节点;实际镜像摘要写入执行快照,不改历史 Run 或其他前置算子镜像。
- Fanyu Worker 的部署配置 `FANYU_SLICE_DISCARD_PACKAGE_IMAGE` 指定兼容新旧契约的 digest-pinned 打包镜像,仅覆盖 builtin package 节点;实际镜像摘要写入执行快照,不改历史 Run 或其他前置算子镜像。总 JSON 形态还要求 Worker 识别 `part.json` 的路由块:旧 Worker 会把它当作普通运行 artifact 上传,而不写入冻结交付目标。
## 验证

View File

@ -6,20 +6,32 @@ The continuous-processing settings now include a required delivery root in the s
`clips/`, `calibration/` and package/evidence manifests are direct children of each part directory. Every part creates an independent Run using the same workflow instance, with its own annotation, quality review and acceptance sampling. A single part also retains its directory. Flat legacy sources omit the part layer. Run IDs are not added to the delivery path.
## Part-index delivery mode
The package node selects the delivery shape with the `PART_INDEX_DELIVERY` hyperparameter (default `false`).
- `false` — the directory form above: sliced media per kept slice, calibration copies, package/evidence manifests.
- `true` — one JSON document per part. The part directory holds `part.json` plus the shared completion `manifest.json`. No sliced media, PTS, IMU, pose or calibration bytes are delivered; downstream packaging reads the index and resolves media from the pinned materialization revision.
The index carries both split levels in one document: the task-level split (`slices[].taskSplit`) and the action-level split (`slices[].segments`). Every reviewed interval is classified exactly once — kept intervals in `slices[]`, discarded intervals in `discardedSlices[]` with their `discardEvidence`. Discarded intervals are now visible to the customer by design; earlier directory deliveries omitted discard reasons entirely.
`clips/` is absent in this mode, so consumers must branch on the completion manifest `kind` (`part-index` versus `directory`) instead of assuming a directory layout.
## Safety and persistence
- API validates disjoint input/output prefixes (including other workflow listeners), rejects reserved system prefixes and probes write/delete permission before enabling intake. The project directory name is server-owned and reused across subsequent configuration updates.
- New admissions freeze connection, bucket and complete target prefix on the Capture record. Existing snapshots are not changed by settings edits or project renames.
- Older tasks with a frozen no-part target are not silently moved. Publication rejects ancestor/descendant overlap with another task's frozen delivery location; resolve the old binding explicitly before publishing into an overlapping location.
- An old task without a delivery snapshot fails publication explicitly. It is not silently migrated and cannot fall back to the Run directory. No existing output is moved automatically.
- The package operator still prepares and validates files in its internal workspace. The worker publishes the `delivery.directory.index` artifact into the frozen destination. Other intermediate artifacts keep their existing storage policy.
- `.delivery-owner.json` reserves the destination using conditional creation. Every member also uses conditional creation and metadata SHA-256/size verification. Retrying identical publication is idempotent; changed content or a different owner is rejected, not overwritten.
- The package operator still prepares and validates files in its internal workspace. The worker publishes the `delivery.directory.index` artifact, or the `part.json` artifact in part-index mode, into the frozen destination. Other intermediate artifacts keep their existing storage policy.
- `.delivery-owner.json` reserves the destination using conditional creation. Every member also uses conditional creation and metadata SHA-256/size verification. Retrying identical publication is idempotent; changed content or a different owner is rejected, not overwritten. Both delivery modes share one owner marker per destination, so a part cannot be delivered twice by different attempts.
- A part-index publication refuses to adopt a destination that already holds files without an owner marker. Additive migration into an existing legacy directory therefore requires that directory to carry its owner marker; otherwise the destination is rejected rather than claimed.
- Files can be visible during publication, but `manifest.json` is written last and is the authoritative completion marker. The business task cannot complete until publication succeeds. This is not a claim of atomic BOS directory rename.
- `packages-manifest.json` is published after data members. `manifest.json` indexes every published member with size and SHA-256. The output remains a directory, not a ZIP.
- `packages-manifest.json` is published after data members. `manifest.json` indexes every published member with size and SHA-256. The directory form additionally publishes package and evidence manifests; the part-index form publishes the index alone. Neither output is a ZIP.
- Temporary upload objects stay under `system/runs/<run>/delivery-staging/` and successful uploads remove them. Interrupted staging is covered by normal Run cleanup.
- Project and workflow deletion protect configured/frozen final delivery destinations. Deleting source business records is not permission to delete delivered data.
- Capture records show the frozen destination and an “open delivery directory” link after delivery.
## Release boundary
API and worker must be deployed together. The earlier part-layout change additionally needs both updated directory/normalizer operator images. Do not deploy only the frontend or start a new scanner with old operators that ignore its explicit file selection.
API and worker must be deployed together. The earlier part-layout change additionally needs both updated directory/normalizer operator images. Do not deploy only the frontend or start a new scanner with old operators that ignore its explicit file selection. Part-index mode additionally requires the worker that understands the `part.json` routing block: an older worker would upload the index as an ordinary run artifact instead of honoring the frozen destination.

View File

@ -2,6 +2,116 @@
适用范围:`fanyu-deliveries/` 下的最终交付目录。一条目录对应一个采集(Capture)的一个 part。
交付有两种形态,由打包节点的 `PART_INDEX_DELIVERY` 超参数决定:
| 形态 | 开关 | 交付内容 |
| --- | --- | --- |
| **总 JSON 索引**(当前推荐) | `true` | 每个 part 一份 `part.json` + 完成标记 `manifest.json` |
| 逐片目录包(历史形态) | `false`(默认) | 按片切开的 `clips/`、`calibration/` 与多份清单 |
目录名由平台生成:项目名与项目 ID 拼接,同一采集的交付目录快照在配置修改或项目改名后不变;Run ID 不进入路径。
---
## 一、总 JSON 索引形态
```
fanyu-deliveries/
└── 项目名__项目ID/
└── Capture采集目录/
└── part0001/
├── part.json # 该 part 的总索引,本形态下唯一的交付物
└── manifest.json # 最后写入的完成标记与校验清单
```
媒体**不随交付发布**:没有 `clips/`、`left.mp4`、PTS、IMU、姿态或标定文件。索引记录切分与分类,媒体按索引里的引用从已验收的物化版本解析,打包交给后续流程。
### 建议的阅读顺序
1. `manifest.json`:确认交付完整(最后写入,出现即表示本次交付已全部上传),并读取 `kind`。
2. `part.json`:切分与分类的唯一权威文档。
⚠️ 判断交付形态要看 `manifest.json` 的 `kind`:`part-index` 是总 JSON 形态,`directory` 是逐片目录形态。不要假定 `clips/` 一定存在。
### manifest.json
part 目录内最后写入的完成标记,也是校验清单:
- `schemaVersion`:固定 `1`。
- `kind`:`part-index` 或 `directory`。
- `fileCount` / `totalBytes`:成员数量与总字节数,与 `entries` 一致。
- `entries[]`:每个成员一条,`path`(相对 part 目录的路径)、`key`、`sizeBytes`、`sha256`。
总 JSON 形态下 `fileCount` 恒为 `1`,`entries[0].path` 恒为 `part.json`。
下载接口另有一份同源的下载清单(`droboticflow.delivery-download/v1`),额外带 `snapshotId` 与签名 URL,用于浏览器侧批量下载。
### part.json
Schema `droboticflow.delivery.part-index/v1`。顶层字段:
| 字段 | 内容 |
| --- | --- |
| `delivery` | 路由块:`kind: part_index`、`fileName`、`artifactName` |
| `capture` | 采集级元数据(Capture 记录证据) |
| `revision` | 本次交付所绑定的物化修订号 |
| `coverage` | 统一帧范围、连续性证据与重建计数 |
| `slices[]` | **保留**区间:任务级切分 + 分类 + 动作级切分 |
| `discardedSlices[]` | **舍弃**区间:任务级切分 + 分类 + 舍弃原因 |
| `deliveryClassification` | 汇总:保留/舍弃的切片数与时长 |
| `calibration[]` | 标定文件引用(名称、路径、SHA-256、字节数) |
| `poseDelivery` | 头部/手部姿态交付策略 |
| `acceptance` | 验收结论、抽检证据、质检轮次 |
| `lineage` | 来源链路:任务 ID、Run ID、人工条目 ID、物化 artifact ID |
#### 两个层级的切分
一次交互的切分信息分两层,都在这一份文档里:
- **任务级切分**(`taskSplit`)—— 标注任务里划定的切片边界,决定「一段数据」的范围:
`startSecond` / `endSecond`(相对 part 起点)、`startTimestampUs` / `endTimestampUs`(采集硬件时钟绝对时间)、`semantics: half-open`。
- **动作级切分**(`segments[]`)—— 切片内部的动作区间,决定「这段数据在做什么」:
`id`、`description`(动作描述)、`status`、`isInvalid`、绝对与相对起止时间、`labelId`、`phrases`、`source`、`confidence`。
所有区间为半开 `[start, end)`:结束时间戳的那一帧属于下一个区间。跨切片、跨 part 对齐一律用 `*TimestampUs` 绝对时间。
#### 分类字段
每个区间恰好被分类一次,落在 `slices[]` 或 `discardedSlices[]` 之一:
- `classification.deliveryDisposition`:`keep` 或 `discard`。
- `classification.descriptionStatus`(仅保留区间):`described`(有动作描述)、`invalid_only`(该切片全部区间显式标为无效)、`empty_confirmed`(确认无语义描述的留空)。
- `classification.taskLabel` / `taskLabelId`:切片名称与任务标签字典来源。
- `classification.discardEvidence`(仅舍弃区间):`reason`(必填,1–1000 字)、`reasonLabelId`、`reasonName`、`note`、`discardedByUserId`、`discardedAt`。
舍弃区间照常交付并附带原因,这是与历史逐片目录形态的**有意差异**:逐片形态只交付保留切片,舍弃原因不出现在对外文件里。
#### 内部内容边界
索引包含审核与验收结论,但**不含**质检意见原文、验收统计明细和舍弃前的历史描述。这些留在平台内部审核记录里。
#### 可复现性
同一个已验收修订重新打包产出**逐字节相同**的 `part.json`:序列化固定键序、缩进与换行,不写入构建时间。因此同一交付重试不会产生内容差异,也不会覆盖已有内容。
### calibration[] 与 media 引用
标定只记录引用(名称、路径、SHA-256、字节数),不再复制标定文件。每个保留区间的 `synchronization` 里带同一组标定引用与边界证据(`maxStereoAlignmentErrorUs`、`boundaryContract`、`boundaryTimestampUs`)。
姿态结果按 `poseDelivery` 决定是否包含:`include` 时逐区间裁剪并挂在 `slices[].pose.headpose` / `slices[].pose.handpose`,`exclude` 时整个字段不出现。原始数据人工版(raw human)不生成姿态产物。
### 校验
1. 按 `manifest.json` 核对 `entries[]` 的文件数、字节数与 SHA-256。
2. 核对 `coverage` 与 `slices[]`/`discardedSlices[]` 的区间并集是否完整、无重叠。
3. `deliveryClassification` 的计数应与两个数组长度一致。
---
## 二、逐片目录形态(历史)
`PART_INDEX_DELIVERY` 为 `false` 时沿用:
```
fanyu-deliveries/
└── 项目名__项目ID/
@ -18,50 +128,48 @@ fanyu-deliveries/
└── slice-000001/ … slice-000NNN/
```
目录名由平台生成:项目名与项目 ID 拼接,同一采集的交付目录快照在配置修改或项目改名后不变;Run ID 不进入路径。
## 建议的阅读顺序
### 建议的阅读顺序
1. `manifest.json`:确认交付完整(它是最后写入的文件,出现即表示本次交付已全部上传)。
2. `packages-manifest.json`:本次交付的总目录,按它核对文件数量和校验值。
3. `capture-manifest.json`:这个 part 的覆盖范围和来源。
4. `clips/`:逐片数据。
## part 级文件
### part 级文件
### manifest.json
#### manifest.json
所有已交付文件的清单:每个文件的路径、大小、SHA-256。由系统在全部文件上传完成后最后写入,是交付完成的标志。收到数据后先按它校验:文件数、大小、哈希值对得上,交付就是完整的。同一次交付重试不会覆盖已有内容。
### packages-manifest.json
#### packages-manifest.json
交付内容的总目录(`schemaVersion: droboticflow.delivery.manifest/v6`)。包含采集信息、版本号、切片数量、覆盖范围、抽检与验收摘要、姿态交付策略、标定文件引用,以及 `items` 列表。`items` 每个切片一项:切片 ID、目录位置(`clips/slice-xxxxxx`)、起止秒与起止时间戳、该目录下每个文件的名称、SHA-256 和字节数。要按片索引数据,从这张表开始。
### capture-manifest.json
#### capture-manifest.json
采集级元数据(`droboticflow.capture-delivery/v1`):Capture 记录、标注版本、覆盖范围 `coverage`(交付区间的时间戳列表,`mode: selected_accepted_intervals`,只含实际交付的区间)、切片数、来源链路(任务 ID、Run ID、人工条目 ID)。
### quality-review.json
#### quality-review.json
质检结论(`droboticflow.capture-quality-review/v1`):每轮质检的任务 ID、版本、决定、审核人、时间。意见原文不随交付发布。
### acceptance-record.json
#### acceptance-record.json
最终验收结论(`droboticflow.capture-acceptance/v1`):是否确认、是否抽检、验收人、验收时间、处理档位、姿态交付策略。
### sampling-evidence.json
#### sampling-evidence.json
抽检证据(`droboticflow.capture-sampling/v1`):是否被抽中、抽样概率、策略版本、随机值、决定时间。验收按抽检进行时,用它可以复核这次交付为什么被抽中。
### calibration/
#### calibration/
相机与 IMU 的标定文件,当前交付必有(缺失时打包会失败)。每个标定文件的名称、路径、SHA-256、字节数记录在 `packages-manifest.json` 和每片的 `synchronized-slice.json` 里。原始数据人工版(raw human)不生成姿态产物,标定照常交付。
### clips/slice-000001/ ……
#### clips/slice-000001/ ……
一个目录一个切片,编号按时间顺序从 000001 递增。内容见下一节。
## 每个切片目录里的文件
### 每个切片目录里的文件
| 文件 | 内容 |
| --- | --- |
@ -73,7 +181,7 @@ fanyu-deliveries/
| `headpose.json` | 头部姿态逐帧结果(按交付策略包含或不含) |
| `handpose.json` | 手部姿态逐帧结果(按交付策略包含或不含) |
### annotation.json
#### annotation.json
`schemaVersion: droboticflow.timestamped-description/v3`。字段:
@ -83,7 +191,7 @@ fanyu-deliveries/
动作区间按切片起点换算相对时间;要用绝对时间对齐传感器数据时,用 `startTimestampUs` / `endTimestampUs`。
### synchronized-slice.json
#### synchronized-slice.json
`schemaVersion: droboticflow.synchronized-slice/v1`。记录这个切片的同步与边界证据:
@ -95,13 +203,13 @@ fanyu-deliveries/
- `calibration`:标定文件引用(名称、路径、SHA-256)。
- `boundaryTimestampUs` / `boundarySeconds` / `rightBoundarySeconds`:统一帧边界网格在左右时间轴上的位置。
## 时间与区间约定
### 时间与区间约定
- 所有区间为半开 `[start, end)`:结束时间戳的那一帧属于下一个切片。
- `timestamp_ms` / `*TimestampUs` 是采集硬件时钟的绝对时间;`clip_time_ms` / `clipStartMs` 相对本切片起点。跨切片对齐用绝对时间。
- 舍弃的切片不出现在 `clips/` 里,交付范围见 `capture-manifest.json` 的 `coverage`(`mode: selected_accepted_intervals`)。
- 保留切片里明确标为无效的动作区间照常交付,看 `annotation.json` 里该条的 `isInvalid`。
## 校验
### 校验
每个文件在 `packages-manifest.json`(`items[].files`)和 `manifest.json` 里都有 SHA-256 和字节数;切片视频另通过帧数与 PTS 行数一致性校验(交付前由系统执行)。收到数据后做全量或抽样哈希核对即可。

View File

@ -55,7 +55,7 @@ flowchart LR
## 输出端口详解
- `deliveryDirectory` / **deliveryDirectory** / `file` / `oss`:最终交付根目录,包含 clips、calibration 与 packages-manifest.json 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
- `deliveryDirectory` / **deliveryDirectory** / `file` / `oss`:逐片目录交付形态的最终交付根目录(PART_INDEX_DELIVERY=false) 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
- `deliveryManifest` / **deliveryManifest** / `json` / `oss`:最终交付包文件、SHA、lineage 和 Pose 策略清单 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
- `packageDirectory` / **packageDirectory** / `file` / `oss`:交付包目录 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
- `finalVideos` / **finalVideos** / `video` / `oss`:最终 MP4 集合 作为输出端口时,适合视频文件或 clip 引用,通常不会以内联字节返回。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
@ -72,6 +72,7 @@ flowchart LR
- `deliverySummary` / **deliverySummary** / `json` / `bus`:交付摘要 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。
- `headposeFiles` / **headposeFiles** / `file` / `oss`:按最终片段裁切的 HeadPose JSON 集合 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
- `handposeFiles` / **handposeFiles** / `file` / `oss`:按最终片段裁切的 HandPose JSON 集合 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
- `deliveryPartIndex` / **deliveryPartIndex** / `json` / `oss`:最终交付的 part 级总 JSON 索引(切分与分类,含舍弃区间证据) 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
## 关键超参数
@ -83,6 +84,7 @@ flowchart LR
- `GENERATE_SEGMENT_MP4`:类型 `boolean`,默认值 `false`。是否按人工最终边界生成物理 MP4 可直接使用默认值,也可在节点级覆盖。
- `SYNCHRONIZED_SIDECARS`:类型 `boolean`,默认值 `false`。是否按绝对硬件时间窗口同步裁切左右视频、PTS 和 IMU,并排除音频 可直接使用默认值,也可在节点级覆盖。
- `ALLOW_POSE_EXCLUSION`:类型 `boolean`,默认值 `false`。是否允许最终验收决定排除 HeadPose 或 HandPose 而继续交付基础数据 可直接使用默认值,也可在节点级覆盖。
- `PART_INDEX_DELIVERY`:类型 `boolean`,默认值 `false`。交付单个 part 级总 JSON 索引,不再切分视频或输出逐片目录 可直接使用默认值,也可在节点级覆盖。
## 执行前准备
@ -116,7 +118,8 @@ flowchart LR
"REQUIRE_FINAL_ACCEPTANCE": "false",
"GENERATE_SEGMENT_MP4": "false",
"SYNCHRONIZED_SIDECARS": "false",
"ALLOW_POSE_EXCLUSION": "false"
"ALLOW_POSE_EXCLUSION": "false",
"PART_INDEX_DELIVERY": "false"
}
}
```

View File

@ -52,7 +52,8 @@ func artifactLooksLikeCollection(item domain.ExecutionArtifact) bool {
"droboticflow.delivery.manifest/v2",
"droboticflow.delivery.manifest/v3",
"droboticflow.delivery.manifest/v4",
"droboticflow.delivery.manifest/v5":
"droboticflow.delivery.manifest/v5",
"droboticflow.delivery.part-index/v1":
return true
}
if _, ok := item.Data["dataRef"]; ok {

View File

@ -7842,6 +7842,12 @@
"key": "ALLOW_POSE_EXCLUSION",
"type": "boolean",
"value": "false"
},
{
"description": "交付单个 part 级总 JSON 索引,不再切分视频或输出逐片目录",
"key": "PART_INDEX_DELIVERY",
"type": "boolean",
"value": "false"
}
],
"id": "operator-builtin-package",
@ -7975,7 +7981,7 @@
"outputs": [
{
"container": "directory",
"description": "最终交付根目录,包含 clips、calibration 与 packages-manifest.json",
"description": "逐片目录交付形态的最终交付根目录(PART_INDEX_DELIVERY=false)",
"id": "deliveryDirectory",
"name": "deliveryDirectory",
"payloadMode": "ref",
@ -8142,6 +8148,16 @@
"role": "annotation",
"transmission": "oss",
"type": "file"
},
{
"container": "single",
"description": "最终交付的 part 级总 JSON 索引(切分与分类,含舍弃区间证据)",
"id": "deliveryPartIndex",
"name": "deliveryPartIndex",
"payloadMode": "ref",
"role": "summary",
"transmission": "oss",
"type": "json"
}
],
"publicationStatus": "built",

View File

@ -142,10 +142,35 @@ func (s *server) projectDownloadTargets(r *http.Request, id string) ([]deliveryD
continue
}
ref := a.Payload.Ref
if ref.Kind != domain.ArtifactRefKindDirectory || ref.Provider != domain.StorageProviderBOS || ref.ManifestKey != ref.Prefix+"/manifest.json" {
if ref.Provider != domain.StorageProviderBOS || ref.ManifestKey == "" {
continue
}
parts := strings.Split(ref.Prefix, "/")
// Directory deliveries expose their tree root in prefix. A
// single-file part index exposes the object key instead, so its
// part directory is the key's parent and must be used as prefix.
prefix := strings.Trim(ref.Prefix, "/")
key := strings.Trim(ref.Key, "/")
switch ref.Kind {
case domain.ArtifactRefKindDirectory:
// Historical directory refs may omit key; prefix is authoritative.
if key != "" && key != prefix {
continue
}
case domain.ArtifactRefKindObject:
if key == "" {
continue
}
parent := path.Dir(key)
if parent == "." || parent == "/" || parent != prefix {
continue
}
default:
continue
}
if prefix == "" || ref.ManifestKey != prefix+"/manifest.json" {
continue
}
parts := strings.Split(prefix, "/")
relative := ""
for i, p := range parts {
if strings.HasSuffix(p, "__"+id) {
@ -156,10 +181,10 @@ func (s *server) projectDownloadTargets(r *http.Request, id string) ([]deliveryD
if relative == "" {
return nil, fmt.Errorf("历史交付目录没有项目层,请使用交付目录下载模式")
}
key := ref.ConnectionID + "/" + ref.Bucket + "/" + ref.Prefix
if !seen[key] {
targets = append(targets, deliveryDownloadTarget{ref.ConnectionID, ref.Bucket, ref.Prefix, relative})
seen[key] = true
target := ref.ConnectionID + "/" + ref.Bucket + "/" + prefix
if !seen[target] {
targets = append(targets, deliveryDownloadTarget{ref.ConnectionID, ref.Bucket, prefix, relative})
seen[target] = true
}
}
}
@ -257,7 +282,9 @@ func readDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliver
SizeBytes int64
} `json:"entries"`
}
if json.Unmarshal(raw, &m) != nil || m.SchemaVersion != 1 || m.Kind != "directory" || m.FileCount != len(m.Entries) || len(m.Entries) == 0 || len(m.Entries) >= deliveryDownloadMaxFiles {
// A directory delivery indexes many members; a part-index delivery indexes
// its single JSON document. Both share one completion-manifest format.
if json.Unmarshal(raw, &m) != nil || m.SchemaVersion != 1 || (m.Kind != "directory" && m.Kind != "part-index") || m.FileCount != len(m.Entries) || len(m.Entries) == 0 || len(m.Entries) >= deliveryDownloadMaxFiles {
return nil, fmt.Errorf("交付清单格式或数量无效")
}
files := make([]deliveryDownloadFile, 0, len(m.Entries)+1)

View File

@ -0,0 +1,88 @@
package api
import (
"os"
"path/filepath"
"regexp"
"strings"
"testing"
"droboticflow/internal/domain"
)
// The part index crosses a language boundary: the package operator (Python)
// produces it, the worker publishes it, and this package projects it for the
// browser. Nothing type-checks that boundary, so read the producer's own
// schema and classification vocabulary and require it to still match what the
// reader understands. Without this, renaming a field on either side silently
// degrades to "暂不支持预览" instead of failing a test.
func TestPartIndexContractMatchesThePackageOperator(t *testing.T) {
root := filepath.Join("..", "..", "backend", "operators", "builtin", "package", "operator.py")
source, err := os.ReadFile(root)
if err != nil {
t.Fatalf("package operator source is unavailable: %v", err)
}
text := string(source)
quoted := func(pattern string) string {
match := regexp.MustCompile(pattern).FindStringSubmatch(text)
if match == nil {
t.Fatalf("package operator no longer declares %s", pattern)
}
return match[1]
}
schema := quoted(`_PART_INDEX_SCHEMA = "([^"]+)"`)
if schema != "droboticflow.delivery.part-index/v1" {
t.Fatalf("part index schema changed to %q; update the reader and its tests", schema)
}
fileName := quoted(`_PART_INDEX_FILE = "([^"]+)"`)
if fileName != "part.json" {
t.Fatalf("part index file name changed to %q; the worker identifies deliveries by it", fileName)
}
if !strings.Contains(text, `"kind": "part_index"`) {
t.Fatal("part index no longer declares its delivery routing kind")
}
if !strings.Contains(text, `_PART_INDEX_ARTIFACT = "delivery.part-index"`) {
t.Fatal("part index artifact name changed; the worker and port binding rely on it")
}
// The payload the projection reads must keep its producer-side keys.
payload := partIndexPayload()
for _, key := range []string{"schemaVersion", "datasetType", "delivery", "coverage", "deliveryClassification", "slices", "discardedSlices"} {
if _, ok := payload[key]; !ok {
t.Fatalf("part index payload fixture is missing producer key %q", key)
}
}
for _, key := range []string{"sliceId", "sliceName", "taskSplit", "classification", "segments"} {
if !strings.Contains(text, `"`+key+`"`) {
t.Fatalf("package operator no longer emits the task-level key %q", key)
}
}
for _, key := range []string{"deliveryDisposition", "discardEvidence", "descriptionStatus", "taskLabel", "taskLabelId"} {
if !strings.Contains(text, `"`+key+`"`) {
t.Fatalf("package operator no longer emits the classification key %q", key)
}
}
for _, key := range []string{"reason", "reasonLabelId", "reasonName", "note", "discardedByUserId", "discardedAt"} {
if !strings.Contains(text, `"`+key+`"`) {
t.Fatalf("package operator no longer emits the discard evidence key %q", key)
}
}
for _, key := range []string{"id", "description", "status", "isInvalid", "startTimestampUs", "endTimestampUs", "clipStartMs", "clipEndMs", "labelId", "phrases", "source", "confidence"} {
if !strings.Contains(text, `"`+key+`"`) {
t.Fatalf("package operator no longer emits the action-level key %q", key)
}
}
}
// The API must recognise exactly the schema the worker is willing to publish,
// otherwise a delivered part index renders as an unsupported preview.
func TestPartIndexSchemaIsRecognisedByThePresentationWhitelist(t *testing.T) {
artifact := domain.ExecutionArtifact{
Name: "part.json",
Type: domain.PortTypeJSON,
Data: partIndexPayload(),
}
if !artifactLooksLikeCollection(artifact) {
t.Fatal("part index is not offered as a structured collection")
}
}

View File

@ -0,0 +1,200 @@
package api
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func partIndexPayload() map[string]any {
return map[string]any{
"schemaVersion": "droboticflow.delivery.part-index/v1",
"datasetType": "fanyu-accepted-slice-part-index",
"delivery": map[string]any{"kind": "part_index", "fileName": "part.json", "artifactName": "delivery.part-index"},
"coverage": map[string]any{"mode": "full_common_range", "sliceCount": 3, "durationSeconds": 3.0},
"deliveryClassification": map[string]any{
"keptSliceCount": 2, "discardedSliceCount": 1,
"keptDurationSeconds": 2.0, "discardedDurationSeconds": 1.0,
},
"slices": []any{
map[string]any{
"sliceId": "slice-000001", "sliceName": "拿取前",
"taskSplit": map[string]any{"startSecond": 0.0, "endSecond": 1.0, "startTimestampUs": 1000000, "endTimestampUs": 2000000},
"classification": map[string]any{"deliveryDisposition": "keep", "descriptionStatus": "described"},
"segments": []any{map[string]any{"id": "a1", "description": "拿起物品"}, map[string]any{"id": "a2", "description": "放下物品"}},
},
map[string]any{
"sliceId": "slice-000003", "sliceName": "放置",
"taskSplit": map[string]any{"startSecond": 2.0, "endSecond": 3.0, "startTimestampUs": 3000000, "endTimestampUs": 4000000},
"classification": map[string]any{"deliveryDisposition": "keep", "descriptionStatus": "empty_confirmed"},
"segments": []any{},
},
},
"discardedSlices": []any{
map[string]any{
"sliceId": "slice-000002", "sliceName": "遮挡",
"taskSplit": map[string]any{"startSecond": 1.0, "endSecond": 2.0, "startTimestampUs": 2000000, "endTimestampUs": 3000000},
"classification": map[string]any{
"deliveryDisposition": "discard",
"discardEvidence": map[string]any{"reason": "镜头被遮挡", "reasonName": "遮挡"},
},
"segments": []any{},
},
},
}
}
func partIndexRun(t *testing.T, runID string, prefix string) (*store.MemoryStore, domain.ExecutionArtifact) {
t.Helper()
artifact := domain.ExecutionArtifact{
ID: "part-index-artifact", RunID: runID, NodeID: "package", PortID: "deliveryPartIndex", PortName: "deliveryPartIndex",
Name: "part.json", Type: domain.PortTypeJSON,
Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: domain.PortTypeJSON, Ref: &domain.ArtifactReference{
Provider: domain.StorageProviderBOS, ConnectionID: "missing-connection", Bucket: "bucket", Key: prefix + "/part.json",
ManifestKey: prefix + "/manifest.json", Kind: domain.ArtifactRefKindObject, SizeBytes: 2048, SHA256: strings.Repeat("a", 64),
}},
Data: partIndexPayload(),
}
run := domain.RunRecord{ID: runID, OrganizationID: domain.DefaultOrganizationID, ProjectID: "project-part-index", WorkflowID: "workflow-part-index", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{artifact}}
dataStore := store.NewMemoryStore()
if err := dataStore.UpsertRun(context.Background(), run); err != nil {
t.Fatal(err)
}
return dataStore, artifact
}
func TestPartIndexManifestListsClassificationWithoutMediaMembers(t *testing.T) {
runID := "run-part-index"
prefix := "deliveries/项目__project/capture/part0001"
dataStore, artifact := partIndexRun(t, runID, prefix)
handler := runArtifactMembersHandler(t, dataStore, t.TempDir())
principal := runArtifactMembersPrincipal(domain.DefaultOrganizationID)
response := httptest.NewRecorder()
handler.ServeHTTP(response, runArtifactMembersRequest(http.MethodGet, "/api/runs/"+runID+"/artifacts/"+artifact.ID+"/members", principal))
if response.Code != http.StatusOK {
t.Fatalf("members status=%d body=%s", response.Code, response.Body.String())
}
var payload struct {
Recognized bool `json:"recognized"`
Schema string `json:"schemaVersion"`
Items []runArtifactManifestMember `json:"items"`
Summary map[string]any `json:"summary"`
}
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if !payload.Recognized || payload.Schema != "droboticflow.delivery.part-index/v1" {
t.Fatalf("part index not recognised: %#v", payload)
}
// The index is the deliverable: one downloadable JSON, no sliced media.
if len(payload.Items) != 1 || payload.Items[0].Name != "part.json" || !payload.Items[0].ContentAvailable {
t.Fatalf("unexpected member projection %#v", payload.Items)
}
if payload.Summary["keptSliceCount"] != float64(2) || payload.Summary["discardedSliceCount"] != float64(1) {
t.Fatalf("classification summary missing: %#v", payload.Summary)
}
slices, ok := payload.Summary["slices"].([]any)
if !ok || len(slices) != 2 {
t.Fatalf("task-level split missing: %#v", payload.Summary["slices"])
}
first, _ := slices[0].(map[string]any)
if first["sliceId"] != "slice-000001" || first["segmentCount"] != float64(2) || first["descriptionStatus"] != "described" {
t.Fatalf("unexpected slice projection %#v", first)
}
discarded, ok := payload.Summary["discardedSlices"].([]any)
if !ok || len(discarded) != 1 {
t.Fatalf("discarded classification missing: %#v", payload.Summary["discardedSlices"])
}
discardedFirst, _ := discarded[0].(map[string]any)
if discardedFirst["deliveryDisposition"] != "discard" || discardedFirst["discardReason"] != "镜头被遮挡" {
t.Fatalf("discard classification missing: %#v", discardedFirst)
}
body := response.Body.String()
if strings.Contains(body, prefix) || strings.Contains(body, "missing-connection") || strings.Contains(body, `"bucket"`) {
t.Fatalf("storage coordinates leaked: %s", body)
}
}
// partIndexDownloadFixture mirrors downloadFixture but delivers a single-file
// part index, proving the download control plane treats it as a first-class
// final delivery instead of skipping it for lacking a directory ref.
func partIndexDownloadFixture(t *testing.T) (*store.MemoryStore, *downloadTestBrowser, http.Handler, *authenticatedPrincipal, string) {
t.Helper()
ds := store.NewMemoryStore()
ctx := t.Context()
org := "org-part-index-download"
projectID := "project-part-index-download"
prefix := "验收/索引项目__" + projectID + "/采集/part0001"
b := &downloadTestBrowser{objects: map[string][]byte{}, pages: map[string]objectStoragePage{}}
b.objects[prefix+"/.delivery-owner.json"] = []byte(`{"owner":"capture:run"}`)
body := []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"part.json"},"slices":[],"discardedSlices":[]}`)
b.objects[prefix+"/part.json"] = body
manifest := map[string]any{"schemaVersion": 1, "kind": "part-index", "fileCount": 1, "totalBytes": len(body), "entries": []map[string]any{{"path": "part.json", "key": prefix + "/part.json", "sizeBytes": len(body), "sha256": downloadTestHash(string(body))}}}
b.objects[prefix+"/manifest.json"], _ = json.Marshal(manifest)
old := storageConnectionBrowserResolver
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
if c == nil || c.OrganizationID != org {
t.Fatal("foreign connection resolved")
}
return b, nil, nil
}
t.Cleanup(func() { storageConnectionBrowserResolver = old })
for _, e := range []error{
ds.UpsertProject(ctx, domain.ProjectRecord{ID: projectID, OrganizationID: org, Name: "索引项目"}),
ds.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "part-index-storage", OrganizationID: org, Provider: domain.StorageProviderBOS, Bucket: "test-bucket", Endpoint: "https://bj.bcebos.com", AccessKeyID: "never-send-ak", SecretAccessKey: "never-send-sk"}),
ds.UpsertRun(ctx, domain.RunRecord{ID: "part-index-run", OrganizationID: org, ProjectID: projectID, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: "part-index-file", RunID: "part-index-run", Name: "part.json", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindObject, Provider: domain.StorageProviderBOS, ConnectionID: "part-index-storage", Bucket: "test-bucket", Key: prefix + "/part.json", Prefix: prefix, ManifestKey: prefix + "/manifest.json", SizeBytes: int64(len(body))}}}}}),
ds.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{ID: "part-index-package", OrganizationID: org, ProjectID: projectID, ChildRunID: "part-index-run", Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction, FinalExportArtifactIDs: []string{"part-index-file"}}),
} {
if e != nil {
t.Fatal(e)
}
}
p := &authenticatedPrincipal{User: domain.UserRecord{ID: "part-index-admin", Role: domain.UserRoleAdmin, OrganizationID: org}, Organization: domain.OrganizationRecord{ID: org}}
return ds, b, NewHandler(ds, nil), p, prefix
}
func TestPartIndexDeliveryIsDownloadableAsAFinalDelivery(t *testing.T) {
_, b, h, p, prefix := partIndexDownloadFixture(t)
w := downloadRequest(h, p, "/api/delivery-downloads/manifest?projectId=project-part-index-download")
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
var m deliveryDownloadManifest
if e := json.Unmarshal(w.Body.Bytes(), &m); e != nil {
t.Fatal(e)
}
// part.json plus its completion manifest; no sliced media members.
if m.TotalFiles != 2 || len(m.Files) != 2 {
t.Fatalf("part index download manifest=%+v", m)
}
base := "索引项目__project-part-index-download/采集/part0001/"
var indexFile *deliveryDownloadFile
for i := range m.Files {
if m.Files[i].RelativePath == base+"part.json" {
indexFile = &m.Files[i]
}
}
if indexFile == nil {
t.Fatalf("part.json missing from the download manifest: %+v", m.Files)
}
if indexFile.SHA256 != downloadTestHash(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"part.json"},"slices":[],"discardedSlices":[]}`) {
t.Fatalf("part.json checksum mismatch: %+v", indexFile)
}
// A downloadable target was resolved from the object ref, not skipped.
if len(b.signed) == 0 && m.SnapshotID == "" {
t.Fatal("part index produced no resolvable target")
}
w = downloadRequest(h, p, "/api/delivery-downloads/manifest?projectId=project-part-index-download&snapshot="+m.SnapshotID+"&file="+indexFile.RelativePath)
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
if !strings.Contains(strings.Join(b.signed, "\n"), prefix+"/part.json") {
t.Fatalf("part.json was never signed: %v", b.signed)
}
}

View File

@ -195,6 +195,8 @@ func (server *server) runArtifactManifestDocumentWithContext(ctx context.Context
members = server.annotationManifestMembers(run, manifest, schemaVersion, payload)
case "droboticflow.delivery.manifest/v2", "droboticflow.delivery.manifest/v3", "droboticflow.delivery.manifest/v4", "droboticflow.delivery.manifest/v5":
members = server.packageManifestMembers(run, manifest, schemaVersion, payload)
case "droboticflow.delivery.part-index/v1":
return server.partIndexManifestDocument(run, manifest, document, payload)
default:
document.Message = "暂不支持预览,可下载受保护的引用内容。"
return document
@ -723,6 +725,85 @@ func (server *server) annotationManifestMembers(run *domain.RunRecord, manifest
return members
}
// partIndexManifestDocument presents a single-file part delivery. The index is
// the deliverable itself, so the listing exposes the JSON plus the split
// classification instead of per-slice media members.
func (server *server) partIndexManifestDocument(
run *domain.RunRecord,
manifest *domain.ExecutionArtifact,
document runArtifactManifestDocument,
payload map[string]any,
) runArtifactManifestDocument {
document.Recognized = true
document.Members = []runArtifactManifestMember{}
classify := func(entries []any, disposition string) []map[string]any {
classified := make([]map[string]any, 0, len(entries))
for index, entry := range entries {
slice := mapValue(entry)
if slice == nil {
continue
}
split := mapValue(slice["taskSplit"])
classification := mapValue(slice["classification"])
segments := listValue(slice["segments"])
detail := map[string]any{
"sliceId": safeRunArtifactMemberText(firstNonEmptyText(stringValue(slice["sliceId"]), fmt.Sprintf("slice-%06d", index+1)), 300),
"sliceName": safeRunArtifactMemberText(stringValue(slice["sliceName"]), 300),
"deliveryDisposition": firstNonEmptyText(stringValue(classification["deliveryDisposition"]), disposition),
"descriptionStatus": safeRunArtifactMemberText(stringValue(classification["descriptionStatus"]), 80),
"segmentCount": len(segments),
"startTimestampUs": int64Value(split["startTimestampUs"]),
"endTimestampUs": int64Value(split["endTimestampUs"]),
"startSecond": floatValue(split["startSecond"]),
"endSecond": floatValue(split["endSecond"]),
}
if evidence := mapValue(classification["discardEvidence"]); evidence != nil {
detail["discardReason"] = safeRunArtifactMemberText(stringValue(evidence["reason"]), 1000)
}
classified = append(classified, detail)
}
return classified
}
slices := classify(listValue(payload["slices"]), "keep")
discarded := classify(listValue(payload["discardedSlices"]), "discard")
counts := mapValue(payload["deliveryClassification"])
document.Summary["datasetType"] = safeRunArtifactMemberText(stringValue(payload["datasetType"]), 160)
document.Summary["keptSliceCount"] = len(slices)
document.Summary["discardedSliceCount"] = len(discarded)
document.Summary["keptDurationSeconds"] = floatValue(counts["keptDurationSeconds"])
document.Summary["discardedDurationSeconds"] = floatValue(counts["discardedDurationSeconds"])
document.Summary["slices"] = slices
if len(discarded) > 0 {
document.Summary["discardedSlices"] = discarded
}
if coverage := mapValue(payload["coverage"]); coverage != nil {
document.Summary["coverage"] = map[string]any{
"mode": safeRunArtifactMemberText(stringValue(coverage["mode"]), 120),
"sliceCount": int64Value(coverage["sliceCount"]),
"durationSeconds": floatValue(coverage["durationSeconds"]),
}
}
if delivery := mapValue(payload["delivery"]); delivery != nil {
document.Summary["deliveryKind"] = safeRunArtifactMemberText(stringValue(delivery["kind"]), 80)
document.Summary["fileName"] = safeRunArtifactMemberText(stringValue(delivery["fileName"]), 200)
}
// The JSON document itself is the single downloadable member. Its bytes are
// served through the regular artifact content endpoint, never inlined here.
if manifest.Payload != nil && manifest.Payload.Mode == domain.ArtifactTransferModeRef && manifest.Payload.Ref != nil && manifest.Payload.Ref.Kind == domain.ArtifactRefKindObject {
ref := manifest.Payload.Ref
fileName := firstNonEmptyText(filepath.Base(ref.Key), manifest.Name)
member := server.newRunArtifactManifestMember(run, manifest, document.SchemaVersion, "part-index-document", "part-index", fileName, "part-index#"+fileName, "", "application/json", ref.SizeBytes, nil)
member.contentArtifact = cloneRunArtifactMemberContentArtifact(*manifest, fileName, "application/json")
member.ContentAvailable = true
document.Members = []runArtifactManifestMember{member}
}
document.Summary["contentCount"] = len(document.Members)
document.Summary["contentAvailableCount"] = countRunArtifactManifestMembersWithContent(document.Members)
document.Summary["groups"] = runArtifactManifestMemberGroups(document.Members)
document.Message = "该 part 以单份 JSON 索引交付:切分与分类在索引内,媒体由后续流程按索引打包。"
return document
}
func (server *server) packageManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
if schema == "droboticflow.delivery.manifest/v5" {
return server.packageManifestV5Members(run, manifest, schema, payload)

View File

@ -352,6 +352,19 @@ func (runtime *Runtime) normalizeOutputArtifacts(ctx context.Context, workflow *
continue
}
info, statErr := os.Stat(artifact.Path)
// A part-index delivery is one file, not a tree. It runs before the
// generic inline/upload paths so the frozen capture destination and the
// shared conflict rules always win over the run's artifact policy.
if statErr == nil && info.Mode().IsRegular() && artifact.Name == partIndexFileName {
if _, e := partIndexPayloadPeek(artifact.Path); e == nil {
delivered, e := runtime.publishCaptureDeliveryFile(ctx, run, nodeID, artifact)
if e != nil {
return nil, e
}
result[index] = delivered
continue
}
}
if statErr == nil && info.Mode().IsRegular() && artifact.Type == domain.PortTypeJSON && info.Size() <= domain.MaxInlinePayloadBytes {
payloadBytes, readErr := os.ReadFile(artifact.Path)
if readErr != nil {

View File

@ -7,6 +7,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"sort"
@ -76,44 +77,62 @@ func publishDeliveryObject(ctx context.Context, p objectsync.Publisher, c domain
return nil
}
func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
// deliveryTarget is the validated, owner-pinned destination shared by every
// final delivery publication mode.
type deliveryTarget struct {
record *domain.CaptureRecord
target *domain.CaptureDeliveryTarget
connection *domain.StorageConnectionRecord
publisher objectsync.Publisher
lister deliveryPrefixLister
owner string
stageRoot string
marker string
publish func(key string, body []byte) error
}
// prepareDeliveryTarget performs every check that must hold before the first
// byte of a delivery is written: frozen destination, prefix validity, overlap
// with other captures and with enabled intake listeners, delegated scope,
// ownership and the conflict rules of the shared destination prefix.
func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (*deliveryTarget, error) {
record, err := runtime.store.GetCaptureRecordByRunID(ctx, run.ID)
if err != nil {
return artifact, err
return nil, err
}
if record == nil || record.Delivery == nil {
return artifact, fmt.Errorf("该任务未配置交付目录,请明确配置后重新接入;不会回退到 runs 目录")
return nil, fmt.Errorf("该任务未配置交付目录,请明确配置后重新接入;不会回退到 runs 目录")
}
target := record.Delivery
if !domain.ValidDeliveryPrefix(target.Prefix) {
return artifact, fmt.Errorf("无效交付目标目录")
return nil, fmt.Errorf("无效交付目标目录")
}
otherCaptures, lookupErr := runtime.store.ListCaptureRecords(ctx, run.OrganizationID, "", 0)
if lookupErr != nil {
return artifact, lookupErr
return nil, lookupErr
}
for _, other := range otherCaptures {
if other.ID != record.ID && other.Delivery != nil && other.Delivery.Bucket == target.Bucket && domain.StoragePrefixesOverlap(other.Delivery.Prefix, target.Prefix) {
return artifact, fmt.Errorf("交付目录与已有任务目标重叠:%s;请先处理旧目录绑定,不会合并或覆盖", other.Delivery.Prefix)
return nil, fmt.Errorf("交付目录与已有任务目标重叠:%s;请先处理旧目录绑定,不会合并或覆盖", other.Delivery.Prefix)
}
}
listeners, e := runtime.store.ListEnabledCaptureAutomations(ctx, run.OrganizationID)
if e != nil {
return artifact, e
return nil, e
}
for _, workflow := range listeners {
if c := workflow.CaptureAutomation; c != nil && c.Bucket == target.Bucket && domain.StoragePrefixesOverlap(c.Prefix, target.Prefix) {
return artifact, fmt.Errorf("交付位置与已启用监听目录重叠,已停止发布")
return nil, fmt.Errorf("交付位置与已启用监听目录重叠,已停止发布")
}
}
connection, err := runtime.captureDeliveryConnection(ctx, run, record, artifact)
if err != nil {
return artifact, err
return nil, err
}
publisher := runtime.outputArtifactPublisher()
lister, ok := publisher.(deliveryPrefixLister)
if !ok {
return artifact, fmt.Errorf("交付存储不支持目录冲突检查")
return nil, fmt.Errorf("交付存储不支持目录冲突检查")
}
ownerRunID := record.RunID
// Capture reconciliation may point RunID at its latest release child. The
@ -125,7 +144,17 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
// PutInternal is conditional-create. A failed invocation may leave staging
// objects behind, so each retry needs its own private staging namespace.
stageRoot := "system/runs/" + run.ID + "/delivery-staging/" + nodeID + "/" + domain.NewID("publish")
publishBytes := func(key string, body []byte) error {
prepared := &deliveryTarget{
record: record,
target: target,
connection: connection,
publisher: publisher,
lister: lister,
owner: owner,
stageRoot: stageRoot,
marker: target.Prefix + "/.delivery-owner.json",
}
prepared.publish = func(key string, body []byte) error {
sum := sha256.Sum256(body)
sha := hex.EncodeToString(sum[:])
stage := stageRoot + "/" + sha
@ -137,24 +166,37 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
}
return publisher.Delete(ctx, *connection, target.Bucket, stage)
}
marker := target.Prefix + "/.delivery-owner.json"
known, err := publisher.Head(ctx, *connection, target.Bucket, marker)
known, err := publisher.Head(ctx, *connection, target.Bucket, prepared.marker)
if err != nil {
return artifact, err
return nil, err
}
if !known.Exists {
keys, e := lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
keys, e := prepared.lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
if e != nil {
return artifact, e
return nil, e
}
if len(keys) > 0 {
return artifact, fmt.Errorf("目标目录已有文件,拒绝覆盖 %s", target.Prefix)
// Refuse to adopt an unowned destination. Part-index migration
// therefore requires the destination to carry its owner marker
// instead of quietly claiming an existing customer directory.
return nil, fmt.Errorf("目标目录已有文件,拒绝覆盖 %s", target.Prefix)
}
}
markerBody, _ := json.Marshal(map[string]string{"owner": owner})
if err = publishBytes(marker, markerBody); err != nil {
if err = prepared.publish(prepared.marker, markerBody); err != nil {
return nil, err
}
return prepared, nil
}
func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
prepared, err := runtime.prepareDeliveryTarget(ctx, run, nodeID, artifact)
if err != nil {
return artifact, err
}
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
publishBytes := prepared.publish
marker := prepared.marker
paths := []string{}
err = filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, e error) error {
if e != nil {
@ -188,7 +230,7 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
if strings.HasPrefix(relative, "../") || relative == "manifest.json" || relative == ".delivery-owner.json" {
return fmt.Errorf("无效交付成员 %s", relative)
}
stage := stageRoot + "/files/" + relative
stage := prepared.stageRoot + "/files/" + relative
meta, sha, e := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, path)
if e != nil {
return e
@ -238,7 +280,7 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
for _, entry := range entries {
expected[entry.Key] = true
}
keys, e := lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
keys, e := prepared.lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
if e != nil {
return artifact, e
}
@ -257,3 +299,95 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Prefix: target.Prefix, ManifestKey: manifestKey, Kind: domain.ArtifactRefKindDirectory}}
return artifact, nil
}
// partIndexSchemaVersion and partIndexArtifactName are the routing contract
// shared with the package operator. Both sides must move together.
const (
partIndexSchemaVersion = "droboticflow.delivery.part-index/v1"
partIndexArtifactName = "delivery.part-index"
partIndexFileName = "part.json"
)
// partIndexDelivery is the routing block a single-file delivery carries inside
// its own payload, so the worker never has to guess from a file name.
type partIndexDelivery struct {
Kind string `json:"kind"`
FileName string `json:"fileName"`
}
// partIndexPayloadPeek reads only the routing block. Payload sizes are bounded
// by the operator contract; the full document is verified by sha256 on upload.
func partIndexPayloadPeek(path string) (partIndexDelivery, error) {
handle, err := os.Open(path)
if err != nil {
return partIndexDelivery{}, err
}
defer handle.Close()
var probe struct {
SchemaVersion string `json:"schemaVersion"`
Delivery *partIndexDelivery `json:"delivery"`
}
if err = json.NewDecoder(io.LimitReader(handle, 1<<16)).Decode(&probe); err != nil {
return partIndexDelivery{}, fmt.Errorf("part index payload is not readable JSON: %w", err)
}
if probe.Delivery == nil || probe.Delivery.Kind != "part_index" {
return partIndexDelivery{}, fmt.Errorf("part index payload is missing its delivery routing block")
}
if probe.SchemaVersion != partIndexSchemaVersion {
return partIndexDelivery{}, fmt.Errorf("unsupported part index schema %q", probe.SchemaVersion)
}
name := filepath.Base(strings.TrimSpace(probe.Delivery.FileName))
if name == "" || name == "." || name == ".." || name != probe.Delivery.FileName {
return partIndexDelivery{}, fmt.Errorf("part index payload declares an unsafe file name")
}
return *probe.Delivery, nil
}
// publishCaptureDeliveryFile places a single file into the frozen part
// destination. It shares every protection of the directory publication — one
// owner marker per destination, conditional create, sha256/size verification,
// idempotent retry and "never overwrite another delivery" — but writes one
// member plus the shared completion manifest instead of a tree.
func (runtime *Runtime) publishCaptureDeliveryFile(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
info, err := os.Lstat(artifact.Path)
if err != nil {
return artifact, err
}
if !info.Mode().IsRegular() {
return artifact, fmt.Errorf("part index delivery requires a regular file")
}
routing, err := partIndexPayloadPeek(artifact.Path)
if err != nil {
return artifact, err
}
prepared, err := runtime.prepareDeliveryTarget(ctx, run, nodeID, artifact)
if err != nil {
return artifact, err
}
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
publishBytes := prepared.publish
fileName := filepath.Base(routing.FileName)
relative := fileName
stage := prepared.stageRoot + "/files/" + relative
meta, sha, err := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, artifact.Path)
if err != nil {
return artifact, err
}
key := target.Prefix + "/" + relative
if err = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); err != nil {
return artifact, err
}
if err = publisher.Delete(ctx, *connection, target.Bucket, stage); err != nil {
return artifact, err
}
entries := []directoryManifestEntry{{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}}
manifestKey := target.Prefix + "/manifest.json"
body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "part-index", FileCount: 1, TotalBytes: meta.SizeBytes, Entries: entries})
if err = publishBytes(manifestKey, body); err != nil {
return artifact, err
}
artifact.Path = "bos://" + target.Bucket + "/" + key
artifact.SizeBytes = meta.SizeBytes
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Key: key, ManifestKey: manifestKey, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject}}
return artifact, nil
}

View File

@ -0,0 +1,246 @@
package worker
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"regexp"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func partIndexFixture(t *testing.T, fileName string, body []byte) string {
t.Helper()
root := t.TempDir()
path := filepath.Join(root, fileName)
if err := os.WriteFile(path, body, 0600); err != nil {
t.Fatal(err)
}
return path
}
func partIndexBody(t *testing.T, fileName string) []byte {
t.Helper()
payload := map[string]any{
"schemaVersion": "droboticflow.delivery.part-index/v1",
"datasetType": "fanyu-accepted-slice-part-index",
"delivery": map[string]any{
"kind": "part_index",
"fileName": fileName,
"artifactName": "delivery.part-index",
},
"slices": []any{},
"discardedSlices": []any{},
"deliveryClassification": map[string]any{"keptSliceCount": 0},
}
body, err := json.Marshal(payload)
if err != nil {
t.Fatal(err)
}
return body
}
func newPartIndexRuntime(t *testing.T, prefix string) (*Runtime, *deliveryPublisher, *domain.CaptureRecord, domain.RunRecord) {
t.Helper()
ctx := context.Background()
db := store.NewMemoryStore()
p := &deliveryPublisher{}
runtime := &Runtime{store: db, artifactPublisher: p}
connection := domain.StorageConnectionRecord{ID: "c", OrganizationID: "org", Provider: domain.StorageProviderBOS, Bucket: "b"}
if err := db.UpsertStorageConnection(ctx, connection); err != nil {
t.Fatal(err)
}
record := &domain.CaptureRecord{ID: "capture", OrganizationID: "org", RunID: "run-1", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "c", Bucket: "b", Prefix: prefix}}
if err := db.UpsertCaptureRecord(ctx, *record); err != nil {
t.Fatal(err)
}
return runtime, p, record, domain.RunRecord{ID: "run-1", OrganizationID: "org"}
}
func TestPartIndexDeliveryPublishesSingleFileAndRetries(t *testing.T) {
ctx := context.Background()
prefix := "deliveries/测试__project/capture/part0001"
runtime, p, _, run := newPartIndexRuntime(t, prefix)
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
got, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact)
if err != nil {
t.Fatal(err)
}
if got.Path != "bos://b/"+prefix+"/part.json" {
t.Fatalf("unexpected published path %s", got.Path)
}
if got.Payload == nil || got.Payload.Ref == nil || got.Payload.Ref.Kind != domain.ArtifactRefKindObject {
t.Fatalf("part index must publish as an object ref: %+v", got.Payload)
}
// No sliced media tree is created by this mode.
if _, exists := p.objects["b/"+prefix+"/clips/slice-000001/left.mp4"]; exists {
t.Fatal("part index delivery wrote sliced media")
}
if _, exists := p.objects["b/"+prefix+"/part.json"]; !exists {
t.Fatal("part.json missing")
}
// The completion manifest records exactly one member and stays last.
manifestBytes, exists := p.objects["b/"+prefix+"/manifest.json"]
if !exists {
t.Fatal("manifest.json missing")
}
var manifest directoryManifest
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
t.Fatal(err)
}
if manifest.Kind != "part-index" || manifest.FileCount != 1 || len(manifest.Entries) != 1 {
t.Fatalf("unexpected manifest %+v", manifest)
}
if manifest.Entries[0].Path != "part.json" || manifest.Entries[0].Key != prefix+"/part.json" {
t.Fatalf("unexpected manifest entry %+v", manifest.Entries[0])
}
sum := sha256.Sum256(partIndexBody(t, "part.json"))
if manifest.Entries[0].SHA256 != hex.EncodeToString(sum[:]) {
t.Fatal("manifest checksum does not match the delivered bytes")
}
if last := p.published[len(p.published)-1]; last != prefix+"/manifest.json" {
t.Fatalf("completion manifest was not written last: %s", last)
}
// An identical retry is idempotent and writes nothing new.
count := len(p.published)
if _, err = runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err != nil {
t.Fatal(err)
}
if len(p.published) != count {
t.Fatal("retry rewrote a completed part index")
}
}
func TestPartIndexDeliveryIsIdempotentAcrossAChangedObjectBody(t *testing.T) {
ctx := context.Background()
prefix := "deliveries/p__1/capture/part0001"
runtime, _, _, run := newPartIndexRuntime(t, prefix)
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err != nil {
t.Fatal(err)
}
// Same owner, different bytes: never silently replace delivered content.
changed := map[string]any{
"schemaVersion": "droboticflow.delivery.part-index/v1",
"delivery": map[string]any{"kind": "part_index", "fileName": "part.json"},
"slices": []any{map[string]any{"sliceId": "slice-000001"}},
}
body, err := json.Marshal(changed)
if err != nil {
t.Fatal(err)
}
if err = os.WriteFile(path, body, 0600); err != nil {
t.Fatal(err)
}
if _, err = runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err == nil {
t.Fatal("changed content overwrote an existing delivery")
}
}
func TestPartIndexDeliveryRejectsANewOwner(t *testing.T) {
ctx := context.Background()
prefix := "deliveries/p__1/capture/part0001"
runtime, p, record, run := newPartIndexRuntime(t, prefix)
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err != nil {
t.Fatal(err)
}
count := len(p.published)
record.RunID = "run-2"
if err := runtime.store.UpsertCaptureRecord(ctx, *record); err != nil {
t.Fatal(err)
}
if _, err := runtime.publishCaptureDeliveryFile(ctx, &domain.RunRecord{ID: "run-2", OrganizationID: "org"}, "package", artifact); err == nil {
t.Fatal("a new attempt replaced a prior part index")
}
if len(p.published) != count {
t.Fatal("conflict mutated output")
}
}
func TestPartIndexDeliveryRefusesToAdoptAnUnownedDirectory(t *testing.T) {
ctx := context.Background()
prefix := "deliveries/p__1/capture/part0001"
runtime, p, _, run := newPartIndexRuntime(t, prefix)
// A legacy directory already exists without an owner marker. Additive
// migration must not quietly claim the customer's directory.
p.objects = map[string][]byte{"b/" + prefix + "/clips/slice-000001/left.mp4": []byte("legacy")}
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err == nil {
t.Fatal("unowned legacy destination was adopted")
}
if _, exists := p.objects["b/"+prefix+"/part.json"]; exists {
t.Fatal("rejected publication still wrote a member")
}
}
func TestPartIndexDeliveryRefusesAnUnrecognisedPayload(t *testing.T) {
path := partIndexFixture(t, "part.json", []byte(`{"schemaVersion":"something/else"}`))
if _, err := partIndexPayloadPeek(path); err == nil {
t.Fatal("unrelated JSON was accepted as a part index")
}
path = partIndexFixture(t, "part.json", []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"../escape.json"}}`))
if _, err := partIndexPayloadPeek(path); err == nil {
t.Fatal("unsafe declared file name was accepted")
}
}
func TestPartIndexDeliveryRequiresAFileArtifact(t *testing.T) {
ctx := context.Background()
prefix := "deliveries/p__1/capture/part0001"
runtime, _, _, run := newPartIndexRuntime(t, prefix)
artifact := domain.ExecutionArtifact{Name: "part.json", Path: t.TempDir(), Type: domain.PortTypeJSON}
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err == nil {
t.Fatal("a directory was accepted as a part-index file delivery")
}
}
// The routing contract is written twice, once in Python and once here. Read the
// producer's own literals so a one-sided rename fails a test instead of leaving
// delivered part indexes unpublished.
func TestPartIndexRoutingContractMatchesThePackageOperator(t *testing.T) {
source, err := os.ReadFile(filepath.Join("..", "..", "backend", "operators", "builtin", "package", "operator.py"))
if err != nil {
t.Fatalf("package operator source is unavailable: %v", err)
}
literal := func(pattern string) string {
match := regexp.MustCompile(pattern).FindStringSubmatch(string(source))
if match == nil {
t.Fatalf("package operator no longer declares %s", pattern)
}
return match[1]
}
if got := literal(`_PART_INDEX_SCHEMA = "([^"]+)"`); got != partIndexSchemaVersion {
t.Fatalf("schema drift: operator=%q worker=%q", got, partIndexSchemaVersion)
}
if got := literal(`_PART_INDEX_FILE = "([^"]+)"`); got != partIndexFileName {
t.Fatalf("file name drift: operator=%q worker=%q", got, partIndexFileName)
}
if got := literal(`_PART_INDEX_ARTIFACT = "([^"]+)"`); got != partIndexArtifactName {
t.Fatalf("artifact name drift: operator=%q worker=%q", got, partIndexArtifactName)
}
}
// A payload declaring a different schema must never reach the frozen delivery
// destination, even when its file name matches the convention.
func TestPartIndexPayloadRejectsAForeignSchema(t *testing.T) {
path := partIndexFixture(t, "part.json", []byte(`{"schemaVersion":"droboticflow.delivery.manifest/v6","delivery":{"kind":"part_index","fileName":"part.json"}}`))
if _, err := partIndexPayloadPeek(path); err == nil {
t.Fatal("a directory manifest was accepted as a part index")
}
path = partIndexFixture(t, "part.json", []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"directory","fileName":"part.json"}}`))
if _, err := partIndexPayloadPeek(path); err == nil {
t.Fatal("a non part-index routing kind was accepted")
}
}

View File

@ -661,6 +661,392 @@ def _collect_unique_package_artifacts(
return artifacts
_PART_INDEX_SCHEMA = "droboticflow.delivery.part-index/v1"
_PART_INDEX_ARTIFACT = "delivery.part-index"
_PART_INDEX_FILE = "part.json"
# Only these review fields leave the platform. Reviewer identity, free-text
# quality comments and unfiltered acceptance statistics stay internal.
_QUALITY_REVIEW_EXPORT_FIELDS = ("taskId", "revision", "decision", "reviewerUserId", "createdAt")
_ACCEPTANCE_EXPORT_FIELDS = (
"confirmed",
"sampled",
"samplingDecision",
"samplingProbability",
"samplingPolicyRevision",
"samplingRandomValue",
"samplingDecidedAt",
"acceptedAt",
"confirmedByUserId",
"processingProfile",
"poseDelivery",
)
def _optional_float(value: Any) -> float | None:
if value is None or isinstance(value, bool):
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def _part_index_capture_record(summary_payload: dict[str, Any], revision_manifest: dict[str, Any]) -> dict[str, Any]:
if isinstance(summary_payload.get("capture"), dict):
return summary_payload["capture"]
if isinstance(revision_manifest.get("capture"), dict):
return revision_manifest["capture"]
return {}
def _part_index_discarded_slice(
row: dict[str, Any],
source_root: pathlib.Path,
) -> dict[str, Any]:
"""Classify a discarded interval without shipping its media or old text.
History of prior descriptions stays internal: the acceptance contract keeps
it as review evidence, not as part of the external deliverable.
"""
slice_id = str(row.get("sliceId") or "")
declared = str(row.get("deliveryDisposition") or "")
evidence = row.get("discardEvidence") if isinstance(row.get("discardEvidence"), dict) else {}
if not str(evidence.get("reason") or "").strip():
raise RuntimeError(f"discarded slice {slice_id} has no reason evidence")
relative = pathlib.PurePosixPath(str(row.get("directory") or ""))
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
raise RuntimeError("unsafe accepted slice directory")
source_slice = source_root / pathlib.Path(*relative.parts)
if not source_slice.is_dir():
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
synchronization_path = source_slice / "synchronized-slice.json"
if not synchronization_path.is_file():
raise RuntimeError(f"accepted slice {slice_id} has no synchronized-slice.json")
try:
synchronization = json.loads(synchronization_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError) as error:
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not valid JSON") from error
if not isinstance(synchronization, dict):
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not an object")
recorded = str(synchronization.get("deliveryDisposition") or declared)
if recorded != declared:
raise RuntimeError(f"accepted slice {slice_id} disposition disagrees with its synchronization evidence")
start = _optional_float(row.get("startSecond")) or 0.0
end = _optional_float(row.get("endSecond")) or 0.0
return {
"sliceId": slice_id,
"sliceName": str(row.get("sliceName") or ""),
"taskSplit": {
"startSecond": start,
"endSecond": end,
"startTimestampUs": int(row["startTimestampUs"]) if row.get("startTimestampUs") is not None else None,
"endTimestampUs": int(row["endTimestampUs"]) if row.get("endTimestampUs") is not None else None,
"semantics": "half-open",
},
"classification": {
"deliveryDisposition": declared,
"discardEvidence": {
"reason": str(evidence.get("reason") or ""),
"reasonLabelId": str(evidence.get("reasonLabelId") or ""),
"reasonName": str(evidence.get("reasonName") or ""),
"note": str(evidence.get("note") or ""),
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
"discardedAt": str(evidence.get("discardedAt") or ""),
},
},
"segments": [],
}
def _part_index_calibration_refs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Reference calibration honestly instead of copying it into a delivery tree.
The part index records where calibration lives and its checksum; the bytes
stay in the accepted materialization revision that this index pins.
"""
refs: list[dict[str, Any]] = []
seen: set[str] = set()
for artifact in _sidecar_artifacts(inputs, {"calibrations"}):
name = str(artifact.get("name") or "").strip()
if not name or name in seen:
continue
seen.add(name)
path = str(artifact.get("path") or "").strip()
digest = str(artifact.get("sha256") or (artifact.get("metadata") or {}).get("sha256") or "").strip().lower()
if path and not _SHA.match(digest):
try:
digest = _sha256_file(pathlib.Path(path))
except OSError as error:
raise RuntimeError(f"calibration file is unavailable: {name}") from error
if not _SHA.match(digest):
raise RuntimeError(f"calibration file has no verifiable checksum: {name}")
size_bytes = int(artifact.get("sizeBytes") or 0)
if path and not size_bytes:
try:
size_bytes = pathlib.Path(path).stat().st_size
except OSError:
size_bytes = 0
refs.append({"name": name, "path": path, "sha256": digest, "bytes": size_bytes})
if not refs:
raise RuntimeError("part index delivery requires camera/IMU calibration files")
return refs
def _emit_part_index(
run_id: str,
node_id: str,
output: pathlib.Path,
node_data: dict[str, Any],
helpers: HelperMap,
payload_bytes: bytes,
) -> list[dict[str, Any]]:
"""Emit the part index as a single file artifact.
The artifact path is the deliverable itself: the worker publishes exactly
this file into the frozen capture delivery destination. The directory is
only a private staging area so a concurrently written node output cannot be
captured by accident. Byte content never carries a build timestamp, so a
retry of the same accepted revision republishes identical bytes.
"""
staging = output / "part-index"
staging.mkdir(parents=True, exist_ok=True)
path = staging / _PART_INDEX_FILE
path.write_bytes(payload_bytes)
return [
helpers["build_artifact"](
run_id,
node_id,
path,
"json",
metadata={"mimeType": "application/json", "sha256": _sha256_file(path)},
output=_output(node_data, helpers, "deliveryPartIndex"),
)
]
def _description_status(actions: list[dict[str, Any]]) -> str:
"""Describe a kept slice without reintroducing the internal annotation state.
``empty_confirmed`` keeps its platform meaning: no description was written
and the submitter confirmed that as a deliberate semantic blank.
"""
if not actions:
return "empty_confirmed"
if all(bool(action.get("isInvalid")) for action in actions):
return "invalid_only"
return "described"
def _pose_section(payload: dict[str, Any] | None, kind: str, start: float, end: float) -> dict[str, Any] | None:
if payload is None:
return None
cropped = _crop_pose(payload, start, end, kind)
if cropped.get("frameCount", 0) <= 0:
return None
return cropped
def _part_index_slice(
row: dict[str, Any],
index: int,
source_slice: pathlib.Path,
calibration: list[dict[str, Any]],
pose_policy: dict[str, Any],
headpose: dict[str, Any] | None,
handpose: dict[str, Any] | None,
) -> dict[str, Any]:
declared = str(row.get("deliveryDisposition") or "")
slice_id = str(row.get("sliceId") or f"slice-{index + 1:06d}")
start = _optional_float(row.get("startSecond")) or 0.0
end = _optional_float(row.get("endSecond")) or 0.0
annotation_path = source_slice / "annotation.json"
if not annotation_path.is_file():
raise RuntimeError(f"accepted slice {slice_id} has no annotation.json")
try:
annotation = json.loads(annotation_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError) as error:
raise RuntimeError(f"accepted slice {slice_id} annotation.json is not valid JSON") from error
if not isinstance(annotation, dict):
raise RuntimeError(f"accepted slice {slice_id} annotation.json is not an object")
raw_actions = annotation.get("actions")
if raw_actions is None:
raw_actions = annotation.get("descriptions")
if raw_actions is None:
raw_actions = []
if not isinstance(raw_actions, list):
raise RuntimeError(f"accepted slice {slice_id} annotation actions are not a list")
actions: list[dict[str, Any]] = []
for action in raw_actions:
if not isinstance(action, dict):
raise RuntimeError(f"accepted slice {slice_id} has a malformed action segment")
start_us = action.get("startTimestampUs")
end_us = action.get("endTimestampUs")
actions.append({
"id": str(action.get("id") or ""),
"description": str(action.get("description") or ""),
"status": str(action.get("status") or ""),
"isInvalid": bool(action.get("isInvalid", False)),
"startTimestampUs": int(start_us) if start_us is not None else None,
"endTimestampUs": int(end_us) if end_us is not None else None,
"startSecond": _optional_float(action.get("clipStartMs")) / 1000.0 if _optional_float(action.get("clipStartMs")) is not None else None,
"endSecond": _optional_float(action.get("clipEndMs")) / 1000.0 if _optional_float(action.get("clipEndMs")) is not None else None,
"clipStartMs": _optional_float(action.get("clipStartMs")),
"clipEndMs": _optional_float(action.get("clipEndMs")),
"labelId": str(action.get("descriptionTemplateId") or ""),
"phrases": [str(item) for item in action.get("phrases", [])] if isinstance(action.get("phrases"), list) else [],
"source": str(action.get("source") or ""),
"confidence": _optional_float(action.get("confidence")),
})
synchronization_path = source_slice / "synchronized-slice.json"
if not synchronization_path.is_file():
raise RuntimeError(f"accepted slice {slice_id} has no synchronized-slice.json")
try:
synchronization = json.loads(synchronization_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError) as error:
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not valid JSON") from error
if not isinstance(synchronization, dict):
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not an object")
# A disposition that disagrees with the pinned review revision must fail
# closed instead of silently exporting the wrong classification.
recorded = str(synchronization.get("deliveryDisposition") or declared)
if recorded != declared:
raise RuntimeError(f"accepted slice {slice_id} disposition disagrees with its synchronization evidence")
entry: dict[str, Any] = {
"sliceId": slice_id,
"sliceName": str(row.get("sliceName") or ""),
"taskSplit": {
"startSecond": start,
"endSecond": end,
"startTimestampUs": int(row["startTimestampUs"]) if row.get("startTimestampUs") is not None else None,
"endTimestampUs": int(row["endTimestampUs"]) if row.get("endTimestampUs") is not None else None,
"semantics": "half-open",
},
"classification": {
"deliveryDisposition": declared,
"taskLabel": str(synchronization.get("taskLabel") or ""),
"taskLabelId": str(synchronization.get("taskLabelId") or ""),
"descriptionStatus": _description_status(actions),
},
"segments": actions,
"synchronization": {
"maxStereoAlignmentErrorUs": synchronization.get("maxStereoAlignmentErrorUs"),
"boundaryContract": str(synchronization.get("boundaryContract") or ""),
"boundaryTimestampUs": [int(value) for value in synchronization.get("boundaryTimestampUs", [])] if isinstance(synchronization.get("boundaryTimestampUs"), list) else [],
"calibration": [
{"name": item["name"], "path": item["path"], "sha256": item["sha256"]}
for item in calibration
],
},
}
if declared == "discard":
evidence = row.get("discardEvidence") if isinstance(row.get("discardEvidence"), dict) else {}
if not str(evidence.get("reason") or "").strip():
raise RuntimeError(f"discarded slice {slice_id} has no reason evidence")
# Discard classification is exported deliberately: downstream packaging
# must know why an interval is excluded.
entry["classification"]["discardEvidence"] = {
"reason": str(evidence.get("reason") or ""),
"reasonLabelId": str(evidence.get("reasonLabelId") or ""),
"reasonName": str(evidence.get("reasonName") or ""),
"note": str(evidence.get("note") or ""),
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
"discardedAt": str(evidence.get("discardedAt") or ""),
}
if pose_policy.get("headpose") == "include":
cropped = _pose_section(headpose, "headpose", start, end)
if cropped is None:
raise RuntimeError(f"HeadPose has no frames for accepted slice {slice_id}")
entry.setdefault("pose", {})["headpose"] = cropped
if pose_policy.get("handpose") == "include":
cropped = _pose_section(handpose, "handpose", start, end)
if cropped is None:
raise RuntimeError(f"HandPose has no frames for accepted slice {slice_id}")
entry.setdefault("pose", {})["handpose"] = cropped
return entry
def _build_part_index(
revision_manifest: dict[str, Any],
coverage: dict[str, Any],
kept_slices: list[dict[str, Any]],
source_root: pathlib.Path,
calibration: list[dict[str, Any]],
pose_policy: dict[str, Any],
headpose: dict[str, Any] | None,
handpose: dict[str, Any] | None,
summary_payload: dict[str, Any],
final_acceptance: dict[str, Any],
capture_record: dict[str, Any],
lineage: dict[str, Any],
discarded_slices: list[dict[str, Any]],
) -> tuple[dict[str, Any], bytes]:
slices: list[dict[str, Any]] = []
for index, row in enumerate(kept_slices):
if not isinstance(row, dict):
raise RuntimeError("invalid accepted slice entry")
relative = pathlib.PurePosixPath(str(row.get("directory") or ""))
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
raise RuntimeError("unsafe accepted slice directory")
source_slice = source_root / pathlib.Path(*relative.parts)
if not source_slice.is_dir():
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
slices.append(_part_index_slice(row, index, source_slice, calibration, pose_policy, headpose, handpose))
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
exported_reviews = [
{key: review[key] for key in _QUALITY_REVIEW_EXPORT_FIELDS if key in review}
for review in quality_reviews
if isinstance(review, dict)
]
payload: dict[str, Any] = {
"schemaVersion": _PART_INDEX_SCHEMA,
"datasetType": "fanyu-accepted-slice-part-index",
# The worker publishes this file unchanged into the part's frozen
# delivery destination and needs stable routing facts, not a filename
# convention it has to re-derive.
"delivery": {
"kind": "part_index",
"fileName": _PART_INDEX_FILE,
"artifactName": _PART_INDEX_ARTIFACT,
},
"capture": capture_record,
"revision": revision_manifest.get("revision"),
"coverage": coverage,
"slices": slices,
"discardedSlices": [
_part_index_discarded_slice(row, source_root)
for row in discarded_slices
if isinstance(row, dict)
],
"deliveryClassification": {
"keptSliceCount": len(slices),
"discardedSliceCount": len(discarded_slices),
"keptDurationSeconds": sum(float(item["taskSplit"]["endSecond"]) - float(item["taskSplit"]["startSecond"]) for item in slices),
"discardedDurationSeconds": sum(
(_optional_float(row.get("endSecond")) or 0.0) - (_optional_float(row.get("startSecond")) or 0.0)
for row in discarded_slices
if isinstance(row, dict)
),
},
"calibration": calibration,
"poseDelivery": pose_policy,
"acceptance": {
"sampling": {
"sampled": bool(final_acceptance.get("sampled", True)),
"decision": str(final_acceptance.get("samplingDecision") or "selected"),
"probability": final_acceptance.get("samplingProbability"),
"policyRevision": final_acceptance.get("samplingPolicyRevision"),
"randomValue": final_acceptance.get("samplingRandomValue"),
"decidedAt": final_acceptance.get("samplingDecidedAt"),
},
"finalAcceptance": {key: final_acceptance[key] for key in _ACCEPTANCE_EXPORT_FIELDS if key in final_acceptance},
"qualityReviews": exported_reviews,
},
"lineage": lineage,
}
body = (json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8")
return payload, body
def _copy_calibrations(inputs: list[dict[str, Any]], packages_root: pathlib.Path, output: pathlib.Path, helpers: HelperMap) -> list[dict[str, Any]]:
destination = packages_root / "calibration"
result: list[dict[str, Any]] = []
@ -1107,6 +1493,7 @@ def _run_accepted_slice_directory(
summary_artifact = next((item for item in inputs if _port(item) == "acceptanceSummary"), None)
summary_payload = summary_artifact.get("data") if isinstance((summary_artifact or {}).get("data"), dict) else {}
final_acceptance = summary_payload.get("finalAcceptance") if isinstance(summary_payload.get("finalAcceptance"), dict) else {}
capture_record = _part_index_capture_record(summary_payload, revision_manifest)
all_slices = revision_manifest.get("items", [])
if not isinstance(all_slices, list) or not all_slices:
raise RuntimeError("accepted slice manifest has no review evidence")
@ -1151,6 +1538,12 @@ def _run_accepted_slice_directory(
payloads = _payload_tree(_payloads_with_full_pose_manifests(inputs, output, helpers))
headpose = _first_payload(payloads, "droboticflow.headpose/v1")
handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1")
params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {}
if _bool(params, "PART_INDEX_DELIVERY", False):
calibration_refs = _part_index_calibration_refs(inputs)
lineage = {"acceptedSliceDirectoryArtifactId": directory_artifact.get("id"), "humanItemId": revision_manifest.get("humanItemId"), "taskId": revision_manifest.get("taskId"), "sourceRunId": revision_manifest.get("runId")}
_, body = _build_part_index(revision_manifest, coverage, kept_slices, source_root, calibration_refs, pose_policy, headpose, handpose, summary_payload, final_acceptance, capture_record, lineage, discarded_slices)
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
delivery_root = pathlib.Path(tempfile.mkdtemp(prefix="delivery-", dir=output)) if disposition_contract else output / "delivery"
clips_root = delivery_root / "clips"
clips_root.mkdir(parents=True, exist_ok=True)
@ -1241,7 +1634,6 @@ def _run_accepted_slice_directory(
"randomValue": final_acceptance.get("samplingRandomValue"),
"decidedAt": final_acceptance.get("samplingDecidedAt"),
}
capture_record = summary_payload.get("capture") if isinstance(summary_payload.get("capture"), dict) else revision_manifest.get("capture") if isinstance(revision_manifest.get("capture"), dict) else {}
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
if disposition_contract:
quality_reviews = [{k:r[k] for k in ("taskId","revision","decision","reviewerUserId","createdAt") if k in r} for r in quality_reviews if isinstance(r,dict)]
@ -1380,6 +1772,12 @@ def _run_multiport(
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
if _output(node_data, helpers, "packageDirectory") is not None or _output(node_data, helpers, "deliveryDirectory") is not None:
# A part-index delivery intentionally has no directory port; it must still
# reach the accepted-slice path instead of the legacy inline contract.
if (
_output(node_data, helpers, "packageDirectory") is not None
or _output(node_data, helpers, "deliveryDirectory") is not None
or _output(node_data, helpers, "deliveryPartIndex") is not None
):
return _run_multiport(run_id, node_id, output_dir, node_data, inputs, helpers)
return _run_existing_contract(run_id, node_id, output_dir, node_data, inputs, helpers)

View File

@ -0,0 +1,178 @@
"""Part-index delivery contract: one JSON per part, no sliced media tree."""
import json
from pathlib import Path
import pytest
from backend.operators.builtin.package import operator as package
from tests.backend.test_slice_discard_package import NODE as BASE_NODE
from tests.backend.test_slice_discard_package import fixture
PART_NODE = {
"outputs": [{"id": name, "name": name} for name in ("deliveryPartIndex", "deliverySummary")],
"hyperparams": [{"key": "PART_INDEX_DELIVERY", "value": "true"}],
}
def helpers():
def write(run, node, directory, name, payload, output=None):
path = Path(directory) / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload), encoding="utf8")
return {"id": run + name, "name": name, "path": str(path), "data": payload, "portId": output["id"]}
def build(run, node, path, artifact_type, data=None, metadata=None, output=None):
return {
"id": run + Path(path).stem,
"name": Path(path).name,
"path": str(path),
"type": artifact_type,
"metadata": dict(metadata or {}),
"portId": output["id"] if output else "",
}
return {
"write_json_artifact": write,
"build_artifact": build,
"build_virtual_artifact": lambda run, node, name, path, kind, size, data=None, output=None, **kw: {
"id": run + name,
"name": name,
"path": path,
"data": data,
"portId": output["id"] if output else "",
},
"payloads_from_inputs": lambda items: [i.get("data", {}) for i in items],
"hyperparams": lambda node: {str(p.get("key")): str(p.get("value")) for p in node.get("hyperparams", []) if isinstance(p, dict)},
"source_path_from_item": lambda item, *args: Path(item.get("path") or item["source"]["path"]),
"source_ref_from_artifact": lambda item: {"path": item.get("path")},
}
def run_part_index(tmp_path, dispositions, node=None, root=None):
inputs = fixture(tmp_path, dispositions)
if node is None:
# The shared fixture is a raw-human revision; give it the pose decision
# the accepted-slice contract requires so the part-index branch runs.
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "exclude", "handpose": "exclude"}
artifacts = package.run("r", "p", root or tmp_path / "output", node or PART_NODE, inputs, helpers())
return inputs, artifacts
def read_index(artifact):
return json.loads(Path(artifact["path"]).read_text(encoding="utf-8"))
def test_part_index_is_a_single_json_delivery_with_both_split_levels(tmp_path):
_, artifacts = run_part_index(tmp_path, ["keep", "discard", "keep"])
assert len(artifacts) == 1
artifact = artifacts[0]
assert artifact["portId"] == "deliveryPartIndex"
assert artifact["name"] == "part.json"
# The artifact path is the deliverable itself; no sliced media tree exists.
assert Path(artifact["path"]).is_file()
assert not list((tmp_path / "output").rglob("*.mp4"))
index = read_index(artifact)
assert index["schemaVersion"] == "droboticflow.delivery.part-index/v1"
assert index["delivery"] == {
"kind": "part_index",
"fileName": "part.json",
"artifactName": "delivery.part-index",
}
# Task-level split carries every delivered interval with its classification.
assert [entry["sliceId"] for entry in index["slices"]] == ["slice-000001", "slice-000003"]
assert [entry["classification"]["deliveryDisposition"] for entry in index["slices"]] == ["keep", "keep"]
assert all(entry["taskSplit"]["semantics"] == "half-open" for entry in index["slices"])
assert index["deliveryClassification"]["keptSliceCount"] == 2
assert index["deliveryClassification"]["discardedSliceCount"] == 1
# The pinned review revision still covers every reviewed interval.
assert index["coverage"]["sliceCount"] == 3
# Action-level split is part of the same document.
first = index["slices"][0]
assert first["segments"][0]["description"] == "无效片段"
assert first["segments"][0]["isInvalid"] is True
assert first["classification"]["descriptionStatus"] == "invalid_only"
def test_discarded_slices_are_delivered_as_their_own_classification(tmp_path):
_, artifacts = run_part_index(tmp_path, ["keep", "discard"])
index = read_index(artifacts[0])
assert [entry["sliceId"] for entry in index["slices"]] == ["slice-000001"]
assert [entry["sliceId"] for entry in index["discardedSlices"]] == ["slice-000002"]
discarded = index["discardedSlices"][0]
assert discarded["classification"]["deliveryDisposition"] == "discard"
assert discarded["classification"]["discardEvidence"]["reason"] == "PRIVATE_DISCARD_REASON"
# No media members: the index only classifies, it does not ship bytes.
assert "files" not in discarded and "mediaKey" not in discarded
def test_calibration_is_referenced_with_checksums(tmp_path):
_, artifacts = run_part_index(tmp_path, ["keep"])
index = read_index(artifacts[0])
assert {entry["name"] for entry in index["calibration"]} == {"calibration_camera.yaml", "calibration_imu.yaml"}
for entry in index["calibration"]:
assert len(entry["sha256"]) == 64
assert entry["bytes"] > 0
for entry in index["slices"]:
assert len(entry["synchronization"]["calibration"]) == 2
def test_review_comments_and_acceptance_statistics_stay_internal(tmp_path):
_, artifacts = run_part_index(tmp_path, ["keep", "discard"])
body = Path(artifacts[0]["path"]).read_text(encoding="utf-8")
# The quality review comment shares its text with the discard reason here,
# so assert on the review projection itself rather than raw text.
index = json.loads(body)
assert "statistics" not in index["acceptance"]["finalAcceptance"]
assert all("comment" not in review for review in index["acceptance"]["qualityReviews"])
def test_part_index_bytes_are_reproducible_for_the_same_revision(tmp_path):
inputs, _ = run_part_index(tmp_path, ["keep", "discard"], root=tmp_path / "a")
first = package.run("r", "p", tmp_path / "a", PART_NODE, inputs, helpers())
second = package.run("r", "p", tmp_path / "b", PART_NODE, inputs, helpers())
assert Path(first[0]["path"]).read_bytes() == Path(second[0]["path"]).read_bytes()
def test_disposition_disagreement_fails_closed(tmp_path):
inputs = fixture(tmp_path, ["keep"])
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "exclude", "handpose": "exclude"}
synchronization = Path(inputs[0]["path"]) / "slices/slice-000001/synchronized-slice.json"
synchronization.write_text(json.dumps({"schemaVersion": "droboticflow.synchronized-slice/v1", "deliveryDisposition": "discard"}))
with pytest.raises(RuntimeError, match="disposition disagrees"):
package.run("r", "p", tmp_path / "output", PART_NODE, inputs, helpers())
def test_part_index_keeps_requiring_an_approved_revision(tmp_path):
inputs = fixture(tmp_path, ["keep", "discard", "keep"])
inputs[1]["data"]["finalAcceptance"]["confirmed"] = False
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "exclude", "handpose": "exclude"}
with pytest.raises(RuntimeError):
package.run("r", "p", tmp_path / "output", PART_NODE, inputs, helpers())
def test_all_discard_still_emits_no_deliverable(tmp_path):
_, artifacts = run_part_index(tmp_path, ["discard", "discard"])
assert len(artifacts) == 1 and artifacts[0]["portId"] == "deliverySummary"
assert artifacts[0]["data"]["outcome"] == "no_deliverables"
assert not list((tmp_path / "output").glob("part-index"))
def test_missing_calibration_fails_closed(tmp_path):
inputs = fixture(tmp_path, ["keep"])
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "exclude", "handpose": "exclude"}
trimmed = [item for item in inputs if item.get("portId") != "calibrations"]
with pytest.raises(RuntimeError, match="calibration"):
package.run("r", "p", tmp_path / "output", PART_NODE, trimmed, helpers())
def test_directory_contract_is_unchanged_when_the_switch_is_off(tmp_path, monkeypatch):
inputs = fixture(tmp_path, ["keep", "discard", "keep"])
monkeypatch.setattr(package, "_ffprobe_frame_count", lambda path: 2)
artifacts = package.run("r", "p", tmp_path / "output", BASE_NODE, inputs, helpers())
directory = Path(next(a["path"] for a in artifacts if a["portId"] == "deliveryDirectory"))
assert {entry.name for entry in (directory / "clips").iterdir()} == {"slice-000001", "slice-000003"}