forked from longtao.wu/droboticflow
🧾 Deliver one part index JSON instead of sliced directories
Add a PART_INDEX_DELIVERY mode to the package operator (default off): a part delivers a single part.json carrying both split levels, the task-level slice boundaries and the action-level segments, plus a classification for every reviewed interval. Kept intervals land in slices[], discarded intervals in discardedSlices[] with their discard evidence, which deliberately makes the discard reason part of the customer deliverable. Media, PTS, IMU, pose and calibration bytes are no longer published; downstream packaging resolves them from the pinned materialization revision. The worker publishes the index through the same frozen-destination path as the directory form. Extract prepareDeliveryTarget so single-file and directory publication share one owner marker, overlap guards, delegated-scope check, conditional create, sha256 verification, idempotent retry and the refusal to overwrite another delivery. A part index also refuses to adopt a destination that holds files without an owner marker, so additive migration cannot claim a legacy directory by accident. The API and download control plane recognise the new schema, project the split classification for the browser, and treat a part index as a first-class final delivery. Cross-language contract tests read the operator's own schema and field literals so a one-sided rename fails a test. Verification: 10 operator tests, 8 worker tests and 3 API tests for the new behaviour pass. Failure sets match the HEAD baseline name-for-name for internal/api (75), internal/worker (4), tests/backend and the frontend suite; no regression. The end-to-end local replica stack was not exercised: it needs image builds and the opt-in Go materialization fixture, so the Go to operator handoff is covered by the contract tests instead.
This commit is contained in:
parent
dfe61d99bb
commit
7b5604f9e0
@ -661,6 +661,392 @@ def _collect_unique_package_artifacts(
|
||||
return artifacts
|
||||
|
||||
|
||||
_PART_INDEX_SCHEMA = "droboticflow.delivery.part-index/v1"
|
||||
_PART_INDEX_ARTIFACT = "delivery.part-index"
|
||||
_PART_INDEX_FILE = "part.json"
|
||||
# Only these review fields leave the platform. Reviewer identity, free-text
|
||||
# quality comments and unfiltered acceptance statistics stay internal.
|
||||
_QUALITY_REVIEW_EXPORT_FIELDS = ("taskId", "revision", "decision", "reviewerUserId", "createdAt")
|
||||
_ACCEPTANCE_EXPORT_FIELDS = (
|
||||
"confirmed",
|
||||
"sampled",
|
||||
"samplingDecision",
|
||||
"samplingProbability",
|
||||
"samplingPolicyRevision",
|
||||
"samplingRandomValue",
|
||||
"samplingDecidedAt",
|
||||
"acceptedAt",
|
||||
"confirmedByUserId",
|
||||
"processingProfile",
|
||||
"poseDelivery",
|
||||
)
|
||||
|
||||
|
||||
def _optional_float(value: Any) -> float | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _part_index_capture_record(summary_payload: dict[str, Any], revision_manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
if isinstance(summary_payload.get("capture"), dict):
|
||||
return summary_payload["capture"]
|
||||
if isinstance(revision_manifest.get("capture"), dict):
|
||||
return revision_manifest["capture"]
|
||||
return {}
|
||||
|
||||
|
||||
def _part_index_discarded_slice(
|
||||
row: dict[str, Any],
|
||||
source_root: pathlib.Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Classify a discarded interval without shipping its media or old text.
|
||||
|
||||
History of prior descriptions stays internal: the acceptance contract keeps
|
||||
it as review evidence, not as part of the external deliverable.
|
||||
"""
|
||||
slice_id = str(row.get("sliceId") or "")
|
||||
declared = str(row.get("deliveryDisposition") or "")
|
||||
evidence = row.get("discardEvidence") if isinstance(row.get("discardEvidence"), dict) else {}
|
||||
if not str(evidence.get("reason") or "").strip():
|
||||
raise RuntimeError(f"discarded slice {slice_id} has no reason evidence")
|
||||
relative = pathlib.PurePosixPath(str(row.get("directory") or ""))
|
||||
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
||||
raise RuntimeError("unsafe accepted slice directory")
|
||||
source_slice = source_root / pathlib.Path(*relative.parts)
|
||||
if not source_slice.is_dir():
|
||||
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
|
||||
synchronization_path = source_slice / "synchronized-slice.json"
|
||||
if not synchronization_path.is_file():
|
||||
raise RuntimeError(f"accepted slice {slice_id} has no synchronized-slice.json")
|
||||
try:
|
||||
synchronization = json.loads(synchronization_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
||||
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not valid JSON") from error
|
||||
if not isinstance(synchronization, dict):
|
||||
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not an object")
|
||||
recorded = str(synchronization.get("deliveryDisposition") or declared)
|
||||
if recorded != declared:
|
||||
raise RuntimeError(f"accepted slice {slice_id} disposition disagrees with its synchronization evidence")
|
||||
start = _optional_float(row.get("startSecond")) or 0.0
|
||||
end = _optional_float(row.get("endSecond")) or 0.0
|
||||
return {
|
||||
"sliceId": slice_id,
|
||||
"sliceName": str(row.get("sliceName") or ""),
|
||||
"taskSplit": {
|
||||
"startSecond": start,
|
||||
"endSecond": end,
|
||||
"startTimestampUs": int(row["startTimestampUs"]) if row.get("startTimestampUs") is not None else None,
|
||||
"endTimestampUs": int(row["endTimestampUs"]) if row.get("endTimestampUs") is not None else None,
|
||||
"semantics": "half-open",
|
||||
},
|
||||
"classification": {
|
||||
"deliveryDisposition": declared,
|
||||
"discardEvidence": {
|
||||
"reason": str(evidence.get("reason") or ""),
|
||||
"reasonLabelId": str(evidence.get("reasonLabelId") or ""),
|
||||
"reasonName": str(evidence.get("reasonName") or ""),
|
||||
"note": str(evidence.get("note") or ""),
|
||||
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
|
||||
"discardedAt": str(evidence.get("discardedAt") or ""),
|
||||
},
|
||||
},
|
||||
"segments": [],
|
||||
}
|
||||
|
||||
|
||||
def _part_index_calibration_refs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Reference calibration honestly instead of copying it into a delivery tree.
|
||||
|
||||
The part index records where calibration lives and its checksum; the bytes
|
||||
stay in the accepted materialization revision that this index pins.
|
||||
"""
|
||||
refs: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for artifact in _sidecar_artifacts(inputs, {"calibrations"}):
|
||||
name = str(artifact.get("name") or "").strip()
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
path = str(artifact.get("path") or "").strip()
|
||||
digest = str(artifact.get("sha256") or (artifact.get("metadata") or {}).get("sha256") or "").strip().lower()
|
||||
if path and not _SHA.match(digest):
|
||||
try:
|
||||
digest = _sha256_file(pathlib.Path(path))
|
||||
except OSError as error:
|
||||
raise RuntimeError(f"calibration file is unavailable: {name}") from error
|
||||
if not _SHA.match(digest):
|
||||
raise RuntimeError(f"calibration file has no verifiable checksum: {name}")
|
||||
size_bytes = int(artifact.get("sizeBytes") or 0)
|
||||
if path and not size_bytes:
|
||||
try:
|
||||
size_bytes = pathlib.Path(path).stat().st_size
|
||||
except OSError:
|
||||
size_bytes = 0
|
||||
refs.append({"name": name, "path": path, "sha256": digest, "bytes": size_bytes})
|
||||
if not refs:
|
||||
raise RuntimeError("part index delivery requires camera/IMU calibration files")
|
||||
return refs
|
||||
|
||||
|
||||
def _emit_part_index(
|
||||
run_id: str,
|
||||
node_id: str,
|
||||
output: pathlib.Path,
|
||||
node_data: dict[str, Any],
|
||||
helpers: HelperMap,
|
||||
payload_bytes: bytes,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Emit the part index as a single file artifact.
|
||||
|
||||
The artifact path is the deliverable itself: the worker publishes exactly
|
||||
this file into the frozen capture delivery destination. The directory is
|
||||
only a private staging area so a concurrently written node output cannot be
|
||||
captured by accident. Byte content never carries a build timestamp, so a
|
||||
retry of the same accepted revision republishes identical bytes.
|
||||
"""
|
||||
staging = output / "part-index"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
path = staging / _PART_INDEX_FILE
|
||||
path.write_bytes(payload_bytes)
|
||||
return [
|
||||
helpers["build_artifact"](
|
||||
run_id,
|
||||
node_id,
|
||||
path,
|
||||
"json",
|
||||
metadata={"mimeType": "application/json", "sha256": _sha256_file(path)},
|
||||
output=_output(node_data, helpers, "deliveryPartIndex"),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _description_status(actions: list[dict[str, Any]]) -> str:
|
||||
"""Describe a kept slice without reintroducing the internal annotation state.
|
||||
|
||||
``empty_confirmed`` keeps its platform meaning: no description was written
|
||||
and the submitter confirmed that as a deliberate semantic blank.
|
||||
"""
|
||||
if not actions:
|
||||
return "empty_confirmed"
|
||||
if all(bool(action.get("isInvalid")) for action in actions):
|
||||
return "invalid_only"
|
||||
return "described"
|
||||
|
||||
|
||||
def _pose_section(payload: dict[str, Any] | None, kind: str, start: float, end: float) -> dict[str, Any] | None:
|
||||
if payload is None:
|
||||
return None
|
||||
cropped = _crop_pose(payload, start, end, kind)
|
||||
if cropped.get("frameCount", 0) <= 0:
|
||||
return None
|
||||
return cropped
|
||||
|
||||
|
||||
def _part_index_slice(
|
||||
row: dict[str, Any],
|
||||
index: int,
|
||||
source_slice: pathlib.Path,
|
||||
calibration: list[dict[str, Any]],
|
||||
pose_policy: dict[str, Any],
|
||||
headpose: dict[str, Any] | None,
|
||||
handpose: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
declared = str(row.get("deliveryDisposition") or "")
|
||||
slice_id = str(row.get("sliceId") or f"slice-{index + 1:06d}")
|
||||
start = _optional_float(row.get("startSecond")) or 0.0
|
||||
end = _optional_float(row.get("endSecond")) or 0.0
|
||||
annotation_path = source_slice / "annotation.json"
|
||||
if not annotation_path.is_file():
|
||||
raise RuntimeError(f"accepted slice {slice_id} has no annotation.json")
|
||||
try:
|
||||
annotation = json.loads(annotation_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
||||
raise RuntimeError(f"accepted slice {slice_id} annotation.json is not valid JSON") from error
|
||||
if not isinstance(annotation, dict):
|
||||
raise RuntimeError(f"accepted slice {slice_id} annotation.json is not an object")
|
||||
raw_actions = annotation.get("actions")
|
||||
if raw_actions is None:
|
||||
raw_actions = annotation.get("descriptions")
|
||||
if raw_actions is None:
|
||||
raw_actions = []
|
||||
if not isinstance(raw_actions, list):
|
||||
raise RuntimeError(f"accepted slice {slice_id} annotation actions are not a list")
|
||||
actions: list[dict[str, Any]] = []
|
||||
for action in raw_actions:
|
||||
if not isinstance(action, dict):
|
||||
raise RuntimeError(f"accepted slice {slice_id} has a malformed action segment")
|
||||
start_us = action.get("startTimestampUs")
|
||||
end_us = action.get("endTimestampUs")
|
||||
actions.append({
|
||||
"id": str(action.get("id") or ""),
|
||||
"description": str(action.get("description") or ""),
|
||||
"status": str(action.get("status") or ""),
|
||||
"isInvalid": bool(action.get("isInvalid", False)),
|
||||
"startTimestampUs": int(start_us) if start_us is not None else None,
|
||||
"endTimestampUs": int(end_us) if end_us is not None else None,
|
||||
"startSecond": _optional_float(action.get("clipStartMs")) / 1000.0 if _optional_float(action.get("clipStartMs")) is not None else None,
|
||||
"endSecond": _optional_float(action.get("clipEndMs")) / 1000.0 if _optional_float(action.get("clipEndMs")) is not None else None,
|
||||
"clipStartMs": _optional_float(action.get("clipStartMs")),
|
||||
"clipEndMs": _optional_float(action.get("clipEndMs")),
|
||||
"labelId": str(action.get("descriptionTemplateId") or ""),
|
||||
"phrases": [str(item) for item in action.get("phrases", [])] if isinstance(action.get("phrases"), list) else [],
|
||||
"source": str(action.get("source") or ""),
|
||||
"confidence": _optional_float(action.get("confidence")),
|
||||
})
|
||||
synchronization_path = source_slice / "synchronized-slice.json"
|
||||
if not synchronization_path.is_file():
|
||||
raise RuntimeError(f"accepted slice {slice_id} has no synchronized-slice.json")
|
||||
try:
|
||||
synchronization = json.loads(synchronization_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
||||
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not valid JSON") from error
|
||||
if not isinstance(synchronization, dict):
|
||||
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not an object")
|
||||
# A disposition that disagrees with the pinned review revision must fail
|
||||
# closed instead of silently exporting the wrong classification.
|
||||
recorded = str(synchronization.get("deliveryDisposition") or declared)
|
||||
if recorded != declared:
|
||||
raise RuntimeError(f"accepted slice {slice_id} disposition disagrees with its synchronization evidence")
|
||||
entry: dict[str, Any] = {
|
||||
"sliceId": slice_id,
|
||||
"sliceName": str(row.get("sliceName") or ""),
|
||||
"taskSplit": {
|
||||
"startSecond": start,
|
||||
"endSecond": end,
|
||||
"startTimestampUs": int(row["startTimestampUs"]) if row.get("startTimestampUs") is not None else None,
|
||||
"endTimestampUs": int(row["endTimestampUs"]) if row.get("endTimestampUs") is not None else None,
|
||||
"semantics": "half-open",
|
||||
},
|
||||
"classification": {
|
||||
"deliveryDisposition": declared,
|
||||
"taskLabel": str(synchronization.get("taskLabel") or ""),
|
||||
"taskLabelId": str(synchronization.get("taskLabelId") or ""),
|
||||
"descriptionStatus": _description_status(actions),
|
||||
},
|
||||
"segments": actions,
|
||||
"synchronization": {
|
||||
"maxStereoAlignmentErrorUs": synchronization.get("maxStereoAlignmentErrorUs"),
|
||||
"boundaryContract": str(synchronization.get("boundaryContract") or ""),
|
||||
"boundaryTimestampUs": [int(value) for value in synchronization.get("boundaryTimestampUs", [])] if isinstance(synchronization.get("boundaryTimestampUs"), list) else [],
|
||||
"calibration": [
|
||||
{"name": item["name"], "path": item["path"], "sha256": item["sha256"]}
|
||||
for item in calibration
|
||||
],
|
||||
},
|
||||
}
|
||||
if declared == "discard":
|
||||
evidence = row.get("discardEvidence") if isinstance(row.get("discardEvidence"), dict) else {}
|
||||
if not str(evidence.get("reason") or "").strip():
|
||||
raise RuntimeError(f"discarded slice {slice_id} has no reason evidence")
|
||||
# Discard classification is exported deliberately: downstream packaging
|
||||
# must know why an interval is excluded.
|
||||
entry["classification"]["discardEvidence"] = {
|
||||
"reason": str(evidence.get("reason") or ""),
|
||||
"reasonLabelId": str(evidence.get("reasonLabelId") or ""),
|
||||
"reasonName": str(evidence.get("reasonName") or ""),
|
||||
"note": str(evidence.get("note") or ""),
|
||||
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
|
||||
"discardedAt": str(evidence.get("discardedAt") or ""),
|
||||
}
|
||||
if pose_policy.get("headpose") == "include":
|
||||
cropped = _pose_section(headpose, "headpose", start, end)
|
||||
if cropped is None:
|
||||
raise RuntimeError(f"HeadPose has no frames for accepted slice {slice_id}")
|
||||
entry.setdefault("pose", {})["headpose"] = cropped
|
||||
if pose_policy.get("handpose") == "include":
|
||||
cropped = _pose_section(handpose, "handpose", start, end)
|
||||
if cropped is None:
|
||||
raise RuntimeError(f"HandPose has no frames for accepted slice {slice_id}")
|
||||
entry.setdefault("pose", {})["handpose"] = cropped
|
||||
return entry
|
||||
|
||||
|
||||
def _build_part_index(
|
||||
revision_manifest: dict[str, Any],
|
||||
coverage: dict[str, Any],
|
||||
kept_slices: list[dict[str, Any]],
|
||||
source_root: pathlib.Path,
|
||||
calibration: list[dict[str, Any]],
|
||||
pose_policy: dict[str, Any],
|
||||
headpose: dict[str, Any] | None,
|
||||
handpose: dict[str, Any] | None,
|
||||
summary_payload: dict[str, Any],
|
||||
final_acceptance: dict[str, Any],
|
||||
capture_record: dict[str, Any],
|
||||
lineage: dict[str, Any],
|
||||
discarded_slices: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, Any], bytes]:
|
||||
slices: list[dict[str, Any]] = []
|
||||
for index, row in enumerate(kept_slices):
|
||||
if not isinstance(row, dict):
|
||||
raise RuntimeError("invalid accepted slice entry")
|
||||
relative = pathlib.PurePosixPath(str(row.get("directory") or ""))
|
||||
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
||||
raise RuntimeError("unsafe accepted slice directory")
|
||||
source_slice = source_root / pathlib.Path(*relative.parts)
|
||||
if not source_slice.is_dir():
|
||||
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
|
||||
slices.append(_part_index_slice(row, index, source_slice, calibration, pose_policy, headpose, handpose))
|
||||
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
|
||||
exported_reviews = [
|
||||
{key: review[key] for key in _QUALITY_REVIEW_EXPORT_FIELDS if key in review}
|
||||
for review in quality_reviews
|
||||
if isinstance(review, dict)
|
||||
]
|
||||
payload: dict[str, Any] = {
|
||||
"schemaVersion": _PART_INDEX_SCHEMA,
|
||||
"datasetType": "fanyu-accepted-slice-part-index",
|
||||
# The worker publishes this file unchanged into the part's frozen
|
||||
# delivery destination and needs stable routing facts, not a filename
|
||||
# convention it has to re-derive.
|
||||
"delivery": {
|
||||
"kind": "part_index",
|
||||
"fileName": _PART_INDEX_FILE,
|
||||
"artifactName": _PART_INDEX_ARTIFACT,
|
||||
},
|
||||
"capture": capture_record,
|
||||
"revision": revision_manifest.get("revision"),
|
||||
"coverage": coverage,
|
||||
"slices": slices,
|
||||
"discardedSlices": [
|
||||
_part_index_discarded_slice(row, source_root)
|
||||
for row in discarded_slices
|
||||
if isinstance(row, dict)
|
||||
],
|
||||
"deliveryClassification": {
|
||||
"keptSliceCount": len(slices),
|
||||
"discardedSliceCount": len(discarded_slices),
|
||||
"keptDurationSeconds": sum(float(item["taskSplit"]["endSecond"]) - float(item["taskSplit"]["startSecond"]) for item in slices),
|
||||
"discardedDurationSeconds": sum(
|
||||
(_optional_float(row.get("endSecond")) or 0.0) - (_optional_float(row.get("startSecond")) or 0.0)
|
||||
for row in discarded_slices
|
||||
if isinstance(row, dict)
|
||||
),
|
||||
},
|
||||
"calibration": calibration,
|
||||
"poseDelivery": pose_policy,
|
||||
"acceptance": {
|
||||
"sampling": {
|
||||
"sampled": bool(final_acceptance.get("sampled", True)),
|
||||
"decision": str(final_acceptance.get("samplingDecision") or "selected"),
|
||||
"probability": final_acceptance.get("samplingProbability"),
|
||||
"policyRevision": final_acceptance.get("samplingPolicyRevision"),
|
||||
"randomValue": final_acceptance.get("samplingRandomValue"),
|
||||
"decidedAt": final_acceptance.get("samplingDecidedAt"),
|
||||
},
|
||||
"finalAcceptance": {key: final_acceptance[key] for key in _ACCEPTANCE_EXPORT_FIELDS if key in final_acceptance},
|
||||
"qualityReviews": exported_reviews,
|
||||
},
|
||||
"lineage": lineage,
|
||||
}
|
||||
body = (json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
||||
return payload, body
|
||||
|
||||
|
||||
def _copy_calibrations(inputs: list[dict[str, Any]], packages_root: pathlib.Path, output: pathlib.Path, helpers: HelperMap) -> list[dict[str, Any]]:
|
||||
destination = packages_root / "calibration"
|
||||
result: list[dict[str, Any]] = []
|
||||
@ -1107,6 +1493,7 @@ def _run_accepted_slice_directory(
|
||||
summary_artifact = next((item for item in inputs if _port(item) == "acceptanceSummary"), None)
|
||||
summary_payload = summary_artifact.get("data") if isinstance((summary_artifact or {}).get("data"), dict) else {}
|
||||
final_acceptance = summary_payload.get("finalAcceptance") if isinstance(summary_payload.get("finalAcceptance"), dict) else {}
|
||||
capture_record = _part_index_capture_record(summary_payload, revision_manifest)
|
||||
all_slices = revision_manifest.get("items", [])
|
||||
if not isinstance(all_slices, list) or not all_slices:
|
||||
raise RuntimeError("accepted slice manifest has no review evidence")
|
||||
@ -1151,6 +1538,12 @@ def _run_accepted_slice_directory(
|
||||
payloads = _payload_tree(_payloads_with_full_pose_manifests(inputs, output, helpers))
|
||||
headpose = _first_payload(payloads, "droboticflow.headpose/v1")
|
||||
handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1")
|
||||
params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {}
|
||||
if _bool(params, "PART_INDEX_DELIVERY", False):
|
||||
calibration_refs = _part_index_calibration_refs(inputs)
|
||||
lineage = {"acceptedSliceDirectoryArtifactId": directory_artifact.get("id"), "humanItemId": revision_manifest.get("humanItemId"), "taskId": revision_manifest.get("taskId"), "sourceRunId": revision_manifest.get("runId")}
|
||||
_, body = _build_part_index(revision_manifest, coverage, kept_slices, source_root, calibration_refs, pose_policy, headpose, handpose, summary_payload, final_acceptance, capture_record, lineage, discarded_slices)
|
||||
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
|
||||
delivery_root = pathlib.Path(tempfile.mkdtemp(prefix="delivery-", dir=output)) if disposition_contract else output / "delivery"
|
||||
clips_root = delivery_root / "clips"
|
||||
clips_root.mkdir(parents=True, exist_ok=True)
|
||||
@ -1241,7 +1634,6 @@ def _run_accepted_slice_directory(
|
||||
"randomValue": final_acceptance.get("samplingRandomValue"),
|
||||
"decidedAt": final_acceptance.get("samplingDecidedAt"),
|
||||
}
|
||||
capture_record = summary_payload.get("capture") if isinstance(summary_payload.get("capture"), dict) else revision_manifest.get("capture") if isinstance(revision_manifest.get("capture"), dict) else {}
|
||||
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
|
||||
if disposition_contract:
|
||||
quality_reviews = [{k:r[k] for k in ("taskId","revision","decision","reviewerUserId","createdAt") if k in r} for r in quality_reviews if isinstance(r,dict)]
|
||||
@ -1380,6 +1772,12 @@ def _run_multiport(
|
||||
|
||||
|
||||
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
|
||||
if _output(node_data, helpers, "packageDirectory") is not None or _output(node_data, helpers, "deliveryDirectory") is not None:
|
||||
# A part-index delivery intentionally has no directory port; it must still
|
||||
# reach the accepted-slice path instead of the legacy inline contract.
|
||||
if (
|
||||
_output(node_data, helpers, "packageDirectory") is not None
|
||||
or _output(node_data, helpers, "deliveryDirectory") is not None
|
||||
or _output(node_data, helpers, "deliveryPartIndex") is not None
|
||||
):
|
||||
return _run_multiport(run_id, node_id, output_dir, node_data, inputs, helpers)
|
||||
return _run_existing_contract(run_id, node_id, output_dir, node_data, inputs, helpers)
|
||||
|
||||
29
docs/adr/0005-part-index-delivery-shape.md
Normal file
29
docs/adr/0005-part-index-delivery-shape.md
Normal file
@ -0,0 +1,29 @@
|
||||
---
|
||||
status: accepted
|
||||
---
|
||||
|
||||
# part 交付以单份总 JSON 索引为交付物,媒体交由下游流程按索引打包
|
||||
|
||||
来源数据在人工流程里已经按任务级切分(切片边界)与动作级切分(动作区间)组织完毕,平台也已经在物化修订里持有全部证据。逐片目录形态把这份信息再物化成一份目录树:为每个保留切片重新编码左右视频、切分 PTS 与 IMU、逐片裁剪姿态、复制标定文件,并为每个文件重算 SHA-256。交付的价值在于**索引与分类**,不在于再搬运一遍字节,因此交付物改为每个 part 一份 `part.json`(`droboticflow.delivery.part-index/v1`),媒体由下游流程按索引解析。打包节点用 `PART_INDEX_DELIVERY`(默认 `false`)在两种形态之间切换,历史形态与其契约保持不变。
|
||||
|
||||
已确认的产品决定:
|
||||
|
||||
- 舍弃切片是**一等分类**,随索引交付并携带 `discardEvidence`。此前的对外交付不包含舍弃原因;这一条有意变更,因为下游必须知道某个区间为什么被排除。舍弃前的历史描述仍属于内部审核证据,不进入索引。
|
||||
- 交付范围由已验收的物化修订固定。索引像逐片形态一样校验逐片 `deliveryDisposition` 与 `synchronized-slice.json` 一致,不一致即 fail-closed。
|
||||
- 已交付数据的迁移是**增量**的:只新增索引,不删除既有 `clips/` 与清单。因此两种形态会在同一交付根下共存,消费方按 `manifest.json` 的 `kind`(`part-index` / `directory`)分流。
|
||||
|
||||
## 保持不变的安全约束
|
||||
|
||||
单文件交付不降低交付安全性,`prepareDeliveryTarget` 是两种形态共用的前置校验:冻结目标校验、与其他 Capture 目标及启用监听目录的重叠检测、委派组织范围校验、`system/runs/` 暂存、`.delivery-owner.json` 条件创建与归属校验、SHA-256/大小元数据校验、幂等重试,以及「绝不覆盖他人交付」的冲突拒绝。两种形态共用同一份归属标记,因此一个 part 不会被两次不同交付写入。
|
||||
|
||||
索引内容不含构建时间,同一已验收修订重新打包产出逐字节相同的文件,使重试天然幂等。
|
||||
|
||||
## 被拒绝的替代方案
|
||||
|
||||
- **按区间列表声明交付范围、视频仍为整段 part 视频**:客户可见的交付视频里会含未交付区间,与「舍弃区间不出现在交付物里」的既有语义冲突。
|
||||
- **索引与逐片媒体并存于同一 part 目录**:等于付出两次物化成本而收益不变;且逐片形态已能表达同一份信息。
|
||||
- **让算子直接写冻结交付目标**:算子不持有交付连接与凭据,会绕过 Worker 的条件创建与归属校验。
|
||||
|
||||
## 审核层不受影响
|
||||
|
||||
物化修订里的逐片视频仍是质检与验收工作台的输入(`MaterializedSliceWorkbench`、`FinalAcceptanceWorkspace` 依赖 `slices/slice-*/left.mp4`)。本决定只改变**交付层**的输出形态,不改变内部审核层。
|
||||
@ -9,7 +9,7 @@
|
||||
- 质检和验收均保留全部切片、视频、PTS/IMU和舍弃原因;按原流程打回和重新审核。不改变抽检比例或既有角色权限。最终验收继续保持匿名,操作人身份保存在内部审计记录中。
|
||||
- “处理覆盖”包含保留和舍弃区间,仍必须完整、无重叠地覆盖统一帧范围。“交付范围”仅包含保留区间,不把正常舍弃造成的交付间隔误报成漏标。
|
||||
- 当前描述型工作台(模板 v6 起)允许保留切片部分或全部语义留空。提交任务即确认空白,不要求逐片标记无效;没有描述的保留切片以 `empty_confirmed` 保存(仅表示确认没有语义描述),不改变交付用途。已有描述仍须有文本、位于所属切片、无重叠且对齐统一帧边界;未保存的编辑仍须保存或取消。旧版任务标签流程的完成规则不变。
|
||||
- 留空仍经过原有质检、验收,保留全部视频及同步文件;`annotation.json` 只记录实际填写的描述,全片留空时 `actions` / `descriptions` 为 `[]`。不会自动补无效片段,不会把未标注区间从交付视频中裁掉。
|
||||
- 留空仍经过原有质检、验收,保留全部视频及同步文件;`annotation.json` 只记录实际填写的描述,全片留空时 `actions` / `descriptions` 为 `[]`。不会自动补无效片段,不会把未标注区间从交付视频中裁掉。总 JSON 交付形态不发布视频,“不裁掉”体现为未交付区间的证据仍留在已验收的物化版本里。
|
||||
- 新增描述从当前播放头开始,不自动回填前面的空白。已有动作描述(包括最后一条)可删除并保留空白,不移动相邻描述;这与“删除切片并合并范围”是两种不同操作。只有真正首尾相接的动作才显示共用分界拖动手柄。
|
||||
- 全部舍弃仍需完成原有审核。完成后状态为“无可交付片段”,不创建客户交付目录或空包,交付包记录没有可导出文件;父/子 Run 正常结束。
|
||||
- 有舍弃/恢复历史的任务保留此前内部审核版本和生成证据,增加相应存储占用;不新增历史版本选择器,不改变项目删除权限或既有项目删除范围。
|
||||
@ -22,12 +22,14 @@
|
||||
|
||||
## 打包与兼容
|
||||
|
||||
- 含舍弃的内部清单使用 `droboticflow.slice-materialization/v3`,仍包含所有审核切片;交付使用 `droboticflow.delivery.manifest/v6`,只列保留项。
|
||||
- Worker 将已审核的物化版本和清单 SHA-256传给打包算子。打包必须匹配该版本,不接受更改后的清单或未知用途。
|
||||
- 每次新契约打包在独立临时目录构建,仅复制保留切片的白名单文件;左右视频、PTS、IMU、描述和姿态文件一致过滤。舍弃原因和历史描述不进入对外交付文件。
|
||||
- 发布仍使用原有指定交付根目录和项目/采集目录结构;已有交付目标冲突不会自动覆盖或删除历史文件。
|
||||
- 含舍弃的内部清单使用 `droboticflow.slice-materialization/v3`,仍包含所有审核切片;逐片目录交付使用 `droboticflow.delivery.manifest/v6`,只列保留项;总 JSON 交付使用 `droboticflow.delivery.part-index/v1`,保留与舍弃区间分别落在 `slices[]` 与 `discardedSlices[]`。
|
||||
- Worker 将已审核的物化版本和清单 SHA-256传给打包算子。打包必须匹配该版本,不接受更改后的清单或未知用途。两种交付形态都校验逐片 `deliveryDisposition` 与 `synchronized-slice.json` 一致,不一致即 fail-closed。
|
||||
- 逐片目录形态:每次新契约打包在独立临时目录构建,仅复制保留切片的白名单文件;左右视频、PTS、IMU、描述和姿态文件一致过滤。舍弃原因和历史描述不进入对外交付文件。
|
||||
- 总 JSON 形态(`PART_INDEX_DELIVERY=true`):交付物只有一份 `part.json`,媒体不随交付发布。舍弃区间作为一等分类随索引交付,**携带 `discardEvidence`(原因、原因字典名、备注、操作人、时间)**;舍弃前的历史描述仍留在平台内部审核记录,不进入索引。
|
||||
- 发布仍使用原有指定交付根目录和项目/采集目录结构;已有交付目标冲突不会自动覆盖或删除历史文件。两种形态共用同一份 `.delivery-owner.json` 归属标记,因此同一 part 不会被两次不同交付写入。
|
||||
- 总 JSON 形态拒绝接管「有文件但没有归属标记」的目标目录:向已有历史目录追加索引前,该目录必须已带归属标记,否则拒绝发布而不是静默认领。
|
||||
- 全保留的旧清单继续走原有 v1/v2兼容逻辑。原始人工和带前置处理工作流都保留原能力。
|
||||
- Fanyu Worker 的部署配置 `FANYU_SLICE_DISCARD_PACKAGE_IMAGE` 指定兼容新旧契约的 digest-pinned 打包镜像,仅覆盖 builtin package 节点;实际镜像摘要写入执行快照,不改历史 Run 或其他前置算子镜像。
|
||||
- Fanyu Worker 的部署配置 `FANYU_SLICE_DISCARD_PACKAGE_IMAGE` 指定兼容新旧契约的 digest-pinned 打包镜像,仅覆盖 builtin package 节点;实际镜像摘要写入执行快照,不改历史 Run 或其他前置算子镜像。总 JSON 形态还要求 Worker 识别 `part.json` 的路由块:旧 Worker 会把它当作普通运行 artifact 上传,而不写入冻结交付目标。
|
||||
|
||||
## 验证
|
||||
|
||||
|
||||
@ -6,20 +6,32 @@ The continuous-processing settings now include a required delivery root in the s
|
||||
|
||||
`clips/`, `calibration/` and package/evidence manifests are direct children of each part directory. Every part creates an independent Run using the same workflow instance, with its own annotation, quality review and acceptance sampling. A single part also retains its directory. Flat legacy sources omit the part layer. Run IDs are not added to the delivery path.
|
||||
|
||||
## Part-index delivery mode
|
||||
|
||||
The package node selects the delivery shape with the `PART_INDEX_DELIVERY` hyperparameter (default `false`).
|
||||
|
||||
- `false` — the directory form above: sliced media per kept slice, calibration copies, package/evidence manifests.
|
||||
- `true` — one JSON document per part. The part directory holds `part.json` plus the shared completion `manifest.json`. No sliced media, PTS, IMU, pose or calibration bytes are delivered; downstream packaging reads the index and resolves media from the pinned materialization revision.
|
||||
|
||||
The index carries both split levels in one document: the task-level split (`slices[].taskSplit`) and the action-level split (`slices[].segments`). Every reviewed interval is classified exactly once — kept intervals in `slices[]`, discarded intervals in `discardedSlices[]` with their `discardEvidence`. Discarded intervals are now visible to the customer by design; earlier directory deliveries omitted discard reasons entirely.
|
||||
|
||||
`clips/` is absent in this mode, so consumers must branch on the completion manifest `kind` (`part-index` versus `directory`) instead of assuming a directory layout.
|
||||
|
||||
## Safety and persistence
|
||||
|
||||
- API validates disjoint input/output prefixes (including other workflow listeners), rejects reserved system prefixes and probes write/delete permission before enabling intake. The project directory name is server-owned and reused across subsequent configuration updates.
|
||||
- New admissions freeze connection, bucket and complete target prefix on the Capture record. Existing snapshots are not changed by settings edits or project renames.
|
||||
- Older tasks with a frozen no-part target are not silently moved. Publication rejects ancestor/descendant overlap with another task's frozen delivery location; resolve the old binding explicitly before publishing into an overlapping location.
|
||||
- An old task without a delivery snapshot fails publication explicitly. It is not silently migrated and cannot fall back to the Run directory. No existing output is moved automatically.
|
||||
- The package operator still prepares and validates files in its internal workspace. The worker publishes the `delivery.directory.index` artifact into the frozen destination. Other intermediate artifacts keep their existing storage policy.
|
||||
- `.delivery-owner.json` reserves the destination using conditional creation. Every member also uses conditional creation and metadata SHA-256/size verification. Retrying identical publication is idempotent; changed content or a different owner is rejected, not overwritten.
|
||||
- The package operator still prepares and validates files in its internal workspace. The worker publishes the `delivery.directory.index` artifact, or the `part.json` artifact in part-index mode, into the frozen destination. Other intermediate artifacts keep their existing storage policy.
|
||||
- `.delivery-owner.json` reserves the destination using conditional creation. Every member also uses conditional creation and metadata SHA-256/size verification. Retrying identical publication is idempotent; changed content or a different owner is rejected, not overwritten. Both delivery modes share one owner marker per destination, so a part cannot be delivered twice by different attempts.
|
||||
- A part-index publication refuses to adopt a destination that already holds files without an owner marker. Additive migration into an existing legacy directory therefore requires that directory to carry its owner marker; otherwise the destination is rejected rather than claimed.
|
||||
- Files can be visible during publication, but `manifest.json` is written last and is the authoritative completion marker. The business task cannot complete until publication succeeds. This is not a claim of atomic BOS directory rename.
|
||||
- `packages-manifest.json` is published after data members. `manifest.json` indexes every published member with size and SHA-256. The output remains a directory, not a ZIP.
|
||||
- `packages-manifest.json` is published after data members. `manifest.json` indexes every published member with size and SHA-256. The directory form additionally publishes package and evidence manifests; the part-index form publishes the index alone. Neither output is a ZIP.
|
||||
- Temporary upload objects stay under `system/runs/<run>/delivery-staging/` and successful uploads remove them. Interrupted staging is covered by normal Run cleanup.
|
||||
- Project and workflow deletion protect configured/frozen final delivery destinations. Deleting source business records is not permission to delete delivered data.
|
||||
- Capture records show the frozen destination and an “open delivery directory” link after delivery.
|
||||
|
||||
## Release boundary
|
||||
|
||||
API and worker must be deployed together. The earlier part-layout change additionally needs both updated directory/normalizer operator images. Do not deploy only the frontend or start a new scanner with old operators that ignore its explicit file selection.
|
||||
API and worker must be deployed together. The earlier part-layout change additionally needs both updated directory/normalizer operator images. Do not deploy only the frontend or start a new scanner with old operators that ignore its explicit file selection. Part-index mode additionally requires the worker that understands the `part.json` routing block: an older worker would upload the index as an ordinary run artifact instead of honoring the frozen destination.
|
||||
|
||||
@ -2,6 +2,116 @@
|
||||
|
||||
适用范围:`fanyu-deliveries/` 下的最终交付目录。一条目录对应一个采集(Capture)的一个 part。
|
||||
|
||||
交付有两种形态,由打包节点的 `PART_INDEX_DELIVERY` 超参数决定:
|
||||
|
||||
| 形态 | 开关 | 交付内容 |
|
||||
| --- | --- | --- |
|
||||
| **总 JSON 索引**(当前推荐) | `true` | 每个 part 一份 `part.json` + 完成标记 `manifest.json` |
|
||||
| 逐片目录包(历史形态) | `false`(默认) | 按片切开的 `clips/`、`calibration/` 与多份清单 |
|
||||
|
||||
目录名由平台生成:项目名与项目 ID 拼接,同一采集的交付目录快照在配置修改或项目改名后不变;Run ID 不进入路径。
|
||||
|
||||
---
|
||||
|
||||
## 一、总 JSON 索引形态
|
||||
|
||||
```
|
||||
fanyu-deliveries/
|
||||
└── 项目名__项目ID/
|
||||
└── Capture采集目录/
|
||||
└── part0001/
|
||||
├── part.json # 该 part 的总索引,本形态下唯一的交付物
|
||||
└── manifest.json # 最后写入的完成标记与校验清单
|
||||
```
|
||||
|
||||
媒体**不随交付发布**:没有 `clips/`、`left.mp4`、PTS、IMU、姿态或标定文件。索引记录切分与分类,媒体按索引里的引用从已验收的物化版本解析,打包交给后续流程。
|
||||
|
||||
### 建议的阅读顺序
|
||||
|
||||
1. `manifest.json`:确认交付完整(最后写入,出现即表示本次交付已全部上传),并读取 `kind`。
|
||||
2. `part.json`:切分与分类的唯一权威文档。
|
||||
|
||||
⚠️ 判断交付形态要看 `manifest.json` 的 `kind`:`part-index` 是总 JSON 形态,`directory` 是逐片目录形态。不要假定 `clips/` 一定存在。
|
||||
|
||||
### manifest.json
|
||||
|
||||
part 目录内最后写入的完成标记,也是校验清单:
|
||||
|
||||
- `schemaVersion`:固定 `1`。
|
||||
- `kind`:`part-index` 或 `directory`。
|
||||
- `fileCount` / `totalBytes`:成员数量与总字节数,与 `entries` 一致。
|
||||
- `entries[]`:每个成员一条,`path`(相对 part 目录的路径)、`key`、`sizeBytes`、`sha256`。
|
||||
|
||||
总 JSON 形态下 `fileCount` 恒为 `1`,`entries[0].path` 恒为 `part.json`。
|
||||
|
||||
下载接口另有一份同源的下载清单(`droboticflow.delivery-download/v1`),额外带 `snapshotId` 与签名 URL,用于浏览器侧批量下载。
|
||||
|
||||
### part.json
|
||||
|
||||
Schema `droboticflow.delivery.part-index/v1`。顶层字段:
|
||||
|
||||
| 字段 | 内容 |
|
||||
| --- | --- |
|
||||
| `delivery` | 路由块:`kind: part_index`、`fileName`、`artifactName` |
|
||||
| `capture` | 采集级元数据(Capture 记录证据) |
|
||||
| `revision` | 本次交付所绑定的物化修订号 |
|
||||
| `coverage` | 统一帧范围、连续性证据与重建计数 |
|
||||
| `slices[]` | **保留**区间:任务级切分 + 分类 + 动作级切分 |
|
||||
| `discardedSlices[]` | **舍弃**区间:任务级切分 + 分类 + 舍弃原因 |
|
||||
| `deliveryClassification` | 汇总:保留/舍弃的切片数与时长 |
|
||||
| `calibration[]` | 标定文件引用(名称、路径、SHA-256、字节数) |
|
||||
| `poseDelivery` | 头部/手部姿态交付策略 |
|
||||
| `acceptance` | 验收结论、抽检证据、质检轮次 |
|
||||
| `lineage` | 来源链路:任务 ID、Run ID、人工条目 ID、物化 artifact ID |
|
||||
|
||||
#### 两个层级的切分
|
||||
|
||||
一次交互的切分信息分两层,都在这一份文档里:
|
||||
|
||||
- **任务级切分**(`taskSplit`)—— 标注任务里划定的切片边界,决定「一段数据」的范围:
|
||||
`startSecond` / `endSecond`(相对 part 起点)、`startTimestampUs` / `endTimestampUs`(采集硬件时钟绝对时间)、`semantics: half-open`。
|
||||
- **动作级切分**(`segments[]`)—— 切片内部的动作区间,决定「这段数据在做什么」:
|
||||
`id`、`description`(动作描述)、`status`、`isInvalid`、绝对与相对起止时间、`labelId`、`phrases`、`source`、`confidence`。
|
||||
|
||||
所有区间为半开 `[start, end)`:结束时间戳的那一帧属于下一个区间。跨切片、跨 part 对齐一律用 `*TimestampUs` 绝对时间。
|
||||
|
||||
#### 分类字段
|
||||
|
||||
每个区间恰好被分类一次,落在 `slices[]` 或 `discardedSlices[]` 之一:
|
||||
|
||||
- `classification.deliveryDisposition`:`keep` 或 `discard`。
|
||||
- `classification.descriptionStatus`(仅保留区间):`described`(有动作描述)、`invalid_only`(该切片全部区间显式标为无效)、`empty_confirmed`(确认无语义描述的留空)。
|
||||
- `classification.taskLabel` / `taskLabelId`:切片名称与任务标签字典来源。
|
||||
- `classification.discardEvidence`(仅舍弃区间):`reason`(必填,1–1000 字)、`reasonLabelId`、`reasonName`、`note`、`discardedByUserId`、`discardedAt`。
|
||||
|
||||
舍弃区间照常交付并附带原因,这是与历史逐片目录形态的**有意差异**:逐片形态只交付保留切片,舍弃原因不出现在对外文件里。
|
||||
|
||||
#### 内部内容边界
|
||||
|
||||
索引包含审核与验收结论,但**不含**质检意见原文、验收统计明细和舍弃前的历史描述。这些留在平台内部审核记录里。
|
||||
|
||||
#### 可复现性
|
||||
|
||||
同一个已验收修订重新打包产出**逐字节相同**的 `part.json`:序列化固定键序、缩进与换行,不写入构建时间。因此同一交付重试不会产生内容差异,也不会覆盖已有内容。
|
||||
|
||||
### calibration[] 与 media 引用
|
||||
|
||||
标定只记录引用(名称、路径、SHA-256、字节数),不再复制标定文件。每个保留区间的 `synchronization` 里带同一组标定引用与边界证据(`maxStereoAlignmentErrorUs`、`boundaryContract`、`boundaryTimestampUs`)。
|
||||
|
||||
姿态结果按 `poseDelivery` 决定是否包含:`include` 时逐区间裁剪并挂在 `slices[].pose.headpose` / `slices[].pose.handpose`,`exclude` 时整个字段不出现。原始数据人工版(raw human)不生成姿态产物。
|
||||
|
||||
### 校验
|
||||
|
||||
1. 按 `manifest.json` 核对 `entries[]` 的文件数、字节数与 SHA-256。
|
||||
2. 核对 `coverage` 与 `slices[]`/`discardedSlices[]` 的区间并集是否完整、无重叠。
|
||||
3. `deliveryClassification` 的计数应与两个数组长度一致。
|
||||
|
||||
---
|
||||
|
||||
## 二、逐片目录形态(历史)
|
||||
|
||||
`PART_INDEX_DELIVERY` 为 `false` 时沿用:
|
||||
|
||||
```
|
||||
fanyu-deliveries/
|
||||
└── 项目名__项目ID/
|
||||
@ -18,50 +128,48 @@ fanyu-deliveries/
|
||||
└── slice-000001/ … slice-000NNN/
|
||||
```
|
||||
|
||||
目录名由平台生成:项目名与项目 ID 拼接,同一采集的交付目录快照在配置修改或项目改名后不变;Run ID 不进入路径。
|
||||
|
||||
## 建议的阅读顺序
|
||||
### 建议的阅读顺序
|
||||
|
||||
1. `manifest.json`:确认交付完整(它是最后写入的文件,出现即表示本次交付已全部上传)。
|
||||
2. `packages-manifest.json`:本次交付的总目录,按它核对文件数量和校验值。
|
||||
3. `capture-manifest.json`:这个 part 的覆盖范围和来源。
|
||||
4. `clips/`:逐片数据。
|
||||
|
||||
## part 级文件
|
||||
### part 级文件
|
||||
|
||||
### manifest.json
|
||||
#### manifest.json
|
||||
|
||||
所有已交付文件的清单:每个文件的路径、大小、SHA-256。由系统在全部文件上传完成后最后写入,是交付完成的标志。收到数据后先按它校验:文件数、大小、哈希值对得上,交付就是完整的。同一次交付重试不会覆盖已有内容。
|
||||
|
||||
### packages-manifest.json
|
||||
#### packages-manifest.json
|
||||
|
||||
交付内容的总目录(`schemaVersion: droboticflow.delivery.manifest/v6`)。包含采集信息、版本号、切片数量、覆盖范围、抽检与验收摘要、姿态交付策略、标定文件引用,以及 `items` 列表。`items` 每个切片一项:切片 ID、目录位置(`clips/slice-xxxxxx`)、起止秒与起止时间戳、该目录下每个文件的名称、SHA-256 和字节数。要按片索引数据,从这张表开始。
|
||||
|
||||
### capture-manifest.json
|
||||
#### capture-manifest.json
|
||||
|
||||
采集级元数据(`droboticflow.capture-delivery/v1`):Capture 记录、标注版本、覆盖范围 `coverage`(交付区间的时间戳列表,`mode: selected_accepted_intervals`,只含实际交付的区间)、切片数、来源链路(任务 ID、Run ID、人工条目 ID)。
|
||||
|
||||
### quality-review.json
|
||||
#### quality-review.json
|
||||
|
||||
质检结论(`droboticflow.capture-quality-review/v1`):每轮质检的任务 ID、版本、决定、审核人、时间。意见原文不随交付发布。
|
||||
|
||||
### acceptance-record.json
|
||||
#### acceptance-record.json
|
||||
|
||||
最终验收结论(`droboticflow.capture-acceptance/v1`):是否确认、是否抽检、验收人、验收时间、处理档位、姿态交付策略。
|
||||
|
||||
### sampling-evidence.json
|
||||
#### sampling-evidence.json
|
||||
|
||||
抽检证据(`droboticflow.capture-sampling/v1`):是否被抽中、抽样概率、策略版本、随机值、决定时间。验收按抽检进行时,用它可以复核这次交付为什么被抽中。
|
||||
|
||||
### calibration/
|
||||
#### calibration/
|
||||
|
||||
相机与 IMU 的标定文件,当前交付必有(缺失时打包会失败)。每个标定文件的名称、路径、SHA-256、字节数记录在 `packages-manifest.json` 和每片的 `synchronized-slice.json` 里。原始数据人工版(raw human)不生成姿态产物,标定照常交付。
|
||||
|
||||
### clips/slice-000001/ ……
|
||||
#### clips/slice-000001/ ……
|
||||
|
||||
一个目录一个切片,编号按时间顺序从 000001 递增。内容见下一节。
|
||||
|
||||
## 每个切片目录里的文件
|
||||
### 每个切片目录里的文件
|
||||
|
||||
| 文件 | 内容 |
|
||||
| --- | --- |
|
||||
@ -73,7 +181,7 @@ fanyu-deliveries/
|
||||
| `headpose.json` | 头部姿态逐帧结果(按交付策略包含或不含) |
|
||||
| `handpose.json` | 手部姿态逐帧结果(按交付策略包含或不含) |
|
||||
|
||||
### annotation.json
|
||||
#### annotation.json
|
||||
|
||||
`schemaVersion: droboticflow.timestamped-description/v3`。字段:
|
||||
|
||||
@ -83,7 +191,7 @@ fanyu-deliveries/
|
||||
|
||||
动作区间按切片起点换算相对时间;要用绝对时间对齐传感器数据时,用 `startTimestampUs` / `endTimestampUs`。
|
||||
|
||||
### synchronized-slice.json
|
||||
#### synchronized-slice.json
|
||||
|
||||
`schemaVersion: droboticflow.synchronized-slice/v1`。记录这个切片的同步与边界证据:
|
||||
|
||||
@ -95,13 +203,13 @@ fanyu-deliveries/
|
||||
- `calibration`:标定文件引用(名称、路径、SHA-256)。
|
||||
- `boundaryTimestampUs` / `boundarySeconds` / `rightBoundarySeconds`:统一帧边界网格在左右时间轴上的位置。
|
||||
|
||||
## 时间与区间约定
|
||||
### 时间与区间约定
|
||||
|
||||
- 所有区间为半开 `[start, end)`:结束时间戳的那一帧属于下一个切片。
|
||||
- `timestamp_ms` / `*TimestampUs` 是采集硬件时钟的绝对时间;`clip_time_ms` / `clipStartMs` 相对本切片起点。跨切片对齐用绝对时间。
|
||||
- 舍弃的切片不出现在 `clips/` 里,交付范围见 `capture-manifest.json` 的 `coverage`(`mode: selected_accepted_intervals`)。
|
||||
- 保留切片里明确标为无效的动作区间照常交付,看 `annotation.json` 里该条的 `isInvalid`。
|
||||
|
||||
## 校验
|
||||
### 校验
|
||||
|
||||
每个文件在 `packages-manifest.json`(`items[].files`)和 `manifest.json` 里都有 SHA-256 和字节数;切片视频另通过帧数与 PTS 行数一致性校验(交付前由系统执行)。收到数据后做全量或抽样哈希核对即可。
|
||||
|
||||
@ -55,7 +55,7 @@ flowchart LR
|
||||
|
||||
## 输出端口详解
|
||||
|
||||
- `deliveryDirectory` / **deliveryDirectory** / `file` / `oss`:最终交付根目录,包含 clips、calibration 与 packages-manifest.json 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
|
||||
- `deliveryDirectory` / **deliveryDirectory** / `file` / `oss`:逐片目录交付形态的最终交付根目录(PART_INDEX_DELIVERY=false) 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
|
||||
- `deliveryManifest` / **deliveryManifest** / `json` / `oss`:最终交付包文件、SHA、lineage 和 Pose 策略清单 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
|
||||
- `packageDirectory` / **packageDirectory** / `file` / `oss`:交付包目录 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
|
||||
- `finalVideos` / **finalVideos** / `video` / `oss`:最终 MP4 集合 作为输出端口时,适合视频文件或 clip 引用,通常不会以内联字节返回。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
|
||||
@ -72,6 +72,7 @@ flowchart LR
|
||||
- `deliverySummary` / **deliverySummary** / `json` / `bus`:交付摘要 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。
|
||||
- `headposeFiles` / **headposeFiles** / `file` / `oss`:按最终片段裁切的 HeadPose JSON 集合 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
|
||||
- `handposeFiles` / **handposeFiles** / `file` / `oss`:按最终片段裁切的 HandPose JSON 集合 作为输出端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
|
||||
- `deliveryPartIndex` / **deliveryPartIndex** / `json` / `oss`:最终交付的 part 级总 JSON 索引(切分与分类,含舍弃区间证据) 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。
|
||||
|
||||
## 关键超参数
|
||||
|
||||
@ -83,6 +84,7 @@ flowchart LR
|
||||
- `GENERATE_SEGMENT_MP4`:类型 `boolean`,默认值 `false`。是否按人工最终边界生成物理 MP4 可直接使用默认值,也可在节点级覆盖。
|
||||
- `SYNCHRONIZED_SIDECARS`:类型 `boolean`,默认值 `false`。是否按绝对硬件时间窗口同步裁切左右视频、PTS 和 IMU,并排除音频 可直接使用默认值,也可在节点级覆盖。
|
||||
- `ALLOW_POSE_EXCLUSION`:类型 `boolean`,默认值 `false`。是否允许最终验收决定排除 HeadPose 或 HandPose 而继续交付基础数据 可直接使用默认值,也可在节点级覆盖。
|
||||
- `PART_INDEX_DELIVERY`:类型 `boolean`,默认值 `false`。交付单个 part 级总 JSON 索引,不再切分视频或输出逐片目录 可直接使用默认值,也可在节点级覆盖。
|
||||
|
||||
## 执行前准备
|
||||
|
||||
@ -116,7 +118,8 @@ flowchart LR
|
||||
"REQUIRE_FINAL_ACCEPTANCE": "false",
|
||||
"GENERATE_SEGMENT_MP4": "false",
|
||||
"SYNCHRONIZED_SIDECARS": "false",
|
||||
"ALLOW_POSE_EXCLUSION": "false"
|
||||
"ALLOW_POSE_EXCLUSION": "false",
|
||||
"PART_INDEX_DELIVERY": "false"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@ -52,7 +52,8 @@ func artifactLooksLikeCollection(item domain.ExecutionArtifact) bool {
|
||||
"droboticflow.delivery.manifest/v2",
|
||||
"droboticflow.delivery.manifest/v3",
|
||||
"droboticflow.delivery.manifest/v4",
|
||||
"droboticflow.delivery.manifest/v5":
|
||||
"droboticflow.delivery.manifest/v5",
|
||||
"droboticflow.delivery.part-index/v1":
|
||||
return true
|
||||
}
|
||||
if _, ok := item.Data["dataRef"]; ok {
|
||||
|
||||
@ -7842,6 +7842,12 @@
|
||||
"key": "ALLOW_POSE_EXCLUSION",
|
||||
"type": "boolean",
|
||||
"value": "false"
|
||||
},
|
||||
{
|
||||
"description": "交付单个 part 级总 JSON 索引,不再切分视频或输出逐片目录",
|
||||
"key": "PART_INDEX_DELIVERY",
|
||||
"type": "boolean",
|
||||
"value": "false"
|
||||
}
|
||||
],
|
||||
"id": "operator-builtin-package",
|
||||
@ -7975,7 +7981,7 @@
|
||||
"outputs": [
|
||||
{
|
||||
"container": "directory",
|
||||
"description": "最终交付根目录,包含 clips、calibration 与 packages-manifest.json",
|
||||
"description": "逐片目录交付形态的最终交付根目录(PART_INDEX_DELIVERY=false)",
|
||||
"id": "deliveryDirectory",
|
||||
"name": "deliveryDirectory",
|
||||
"payloadMode": "ref",
|
||||
@ -8142,6 +8148,16 @@
|
||||
"role": "annotation",
|
||||
"transmission": "oss",
|
||||
"type": "file"
|
||||
},
|
||||
{
|
||||
"container": "single",
|
||||
"description": "最终交付的 part 级总 JSON 索引(切分与分类,含舍弃区间证据)",
|
||||
"id": "deliveryPartIndex",
|
||||
"name": "deliveryPartIndex",
|
||||
"payloadMode": "ref",
|
||||
"role": "summary",
|
||||
"transmission": "oss",
|
||||
"type": "json"
|
||||
}
|
||||
],
|
||||
"publicationStatus": "built",
|
||||
|
||||
@ -142,10 +142,35 @@ func (s *server) projectDownloadTargets(r *http.Request, id string) ([]deliveryD
|
||||
continue
|
||||
}
|
||||
ref := a.Payload.Ref
|
||||
if ref.Kind != domain.ArtifactRefKindDirectory || ref.Provider != domain.StorageProviderBOS || ref.ManifestKey != ref.Prefix+"/manifest.json" {
|
||||
if ref.Provider != domain.StorageProviderBOS || ref.ManifestKey == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(ref.Prefix, "/")
|
||||
// Directory deliveries expose their tree root in prefix. A
|
||||
// single-file part index exposes the object key instead, so its
|
||||
// part directory is the key's parent and must be used as prefix.
|
||||
prefix := strings.Trim(ref.Prefix, "/")
|
||||
key := strings.Trim(ref.Key, "/")
|
||||
switch ref.Kind {
|
||||
case domain.ArtifactRefKindDirectory:
|
||||
// Historical directory refs may omit key; prefix is authoritative.
|
||||
if key != "" && key != prefix {
|
||||
continue
|
||||
}
|
||||
case domain.ArtifactRefKindObject:
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
parent := path.Dir(key)
|
||||
if parent == "." || parent == "/" || parent != prefix {
|
||||
continue
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if prefix == "" || ref.ManifestKey != prefix+"/manifest.json" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(prefix, "/")
|
||||
relative := ""
|
||||
for i, p := range parts {
|
||||
if strings.HasSuffix(p, "__"+id) {
|
||||
@ -156,10 +181,10 @@ func (s *server) projectDownloadTargets(r *http.Request, id string) ([]deliveryD
|
||||
if relative == "" {
|
||||
return nil, fmt.Errorf("历史交付目录没有项目层,请使用交付目录下载模式")
|
||||
}
|
||||
key := ref.ConnectionID + "/" + ref.Bucket + "/" + ref.Prefix
|
||||
if !seen[key] {
|
||||
targets = append(targets, deliveryDownloadTarget{ref.ConnectionID, ref.Bucket, ref.Prefix, relative})
|
||||
seen[key] = true
|
||||
target := ref.ConnectionID + "/" + ref.Bucket + "/" + prefix
|
||||
if !seen[target] {
|
||||
targets = append(targets, deliveryDownloadTarget{ref.ConnectionID, ref.Bucket, prefix, relative})
|
||||
seen[target] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -257,7 +282,9 @@ func readDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliver
|
||||
SizeBytes int64
|
||||
} `json:"entries"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.SchemaVersion != 1 || m.Kind != "directory" || m.FileCount != len(m.Entries) || len(m.Entries) == 0 || len(m.Entries) >= deliveryDownloadMaxFiles {
|
||||
// A directory delivery indexes many members; a part-index delivery indexes
|
||||
// its single JSON document. Both share one completion-manifest format.
|
||||
if json.Unmarshal(raw, &m) != nil || m.SchemaVersion != 1 || (m.Kind != "directory" && m.Kind != "part-index") || m.FileCount != len(m.Entries) || len(m.Entries) == 0 || len(m.Entries) >= deliveryDownloadMaxFiles {
|
||||
return nil, fmt.Errorf("交付清单格式或数量无效")
|
||||
}
|
||||
files := make([]deliveryDownloadFile, 0, len(m.Entries)+1)
|
||||
|
||||
88
internal/api/part_index_contract_test.go
Normal file
88
internal/api/part_index_contract_test.go
Normal file
@ -0,0 +1,88 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
// The part index crosses a language boundary: the package operator (Python)
|
||||
// produces it, the worker publishes it, and this package projects it for the
|
||||
// browser. Nothing type-checks that boundary, so read the producer's own
|
||||
// schema and classification vocabulary and require it to still match what the
|
||||
// reader understands. Without this, renaming a field on either side silently
|
||||
// degrades to "暂不支持预览" instead of failing a test.
|
||||
func TestPartIndexContractMatchesThePackageOperator(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "backend", "operators", "builtin", "package", "operator.py")
|
||||
source, err := os.ReadFile(root)
|
||||
if err != nil {
|
||||
t.Fatalf("package operator source is unavailable: %v", err)
|
||||
}
|
||||
text := string(source)
|
||||
quoted := func(pattern string) string {
|
||||
match := regexp.MustCompile(pattern).FindStringSubmatch(text)
|
||||
if match == nil {
|
||||
t.Fatalf("package operator no longer declares %s", pattern)
|
||||
}
|
||||
return match[1]
|
||||
}
|
||||
schema := quoted(`_PART_INDEX_SCHEMA = "([^"]+)"`)
|
||||
if schema != "droboticflow.delivery.part-index/v1" {
|
||||
t.Fatalf("part index schema changed to %q; update the reader and its tests", schema)
|
||||
}
|
||||
fileName := quoted(`_PART_INDEX_FILE = "([^"]+)"`)
|
||||
if fileName != "part.json" {
|
||||
t.Fatalf("part index file name changed to %q; the worker identifies deliveries by it", fileName)
|
||||
}
|
||||
if !strings.Contains(text, `"kind": "part_index"`) {
|
||||
t.Fatal("part index no longer declares its delivery routing kind")
|
||||
}
|
||||
if !strings.Contains(text, `_PART_INDEX_ARTIFACT = "delivery.part-index"`) {
|
||||
t.Fatal("part index artifact name changed; the worker and port binding rely on it")
|
||||
}
|
||||
|
||||
// The payload the projection reads must keep its producer-side keys.
|
||||
payload := partIndexPayload()
|
||||
for _, key := range []string{"schemaVersion", "datasetType", "delivery", "coverage", "deliveryClassification", "slices", "discardedSlices"} {
|
||||
if _, ok := payload[key]; !ok {
|
||||
t.Fatalf("part index payload fixture is missing producer key %q", key)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"sliceId", "sliceName", "taskSplit", "classification", "segments"} {
|
||||
if !strings.Contains(text, `"`+key+`"`) {
|
||||
t.Fatalf("package operator no longer emits the task-level key %q", key)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"deliveryDisposition", "discardEvidence", "descriptionStatus", "taskLabel", "taskLabelId"} {
|
||||
if !strings.Contains(text, `"`+key+`"`) {
|
||||
t.Fatalf("package operator no longer emits the classification key %q", key)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"reason", "reasonLabelId", "reasonName", "note", "discardedByUserId", "discardedAt"} {
|
||||
if !strings.Contains(text, `"`+key+`"`) {
|
||||
t.Fatalf("package operator no longer emits the discard evidence key %q", key)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"id", "description", "status", "isInvalid", "startTimestampUs", "endTimestampUs", "clipStartMs", "clipEndMs", "labelId", "phrases", "source", "confidence"} {
|
||||
if !strings.Contains(text, `"`+key+`"`) {
|
||||
t.Fatalf("package operator no longer emits the action-level key %q", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The API must recognise exactly the schema the worker is willing to publish,
|
||||
// otherwise a delivered part index renders as an unsupported preview.
|
||||
func TestPartIndexSchemaIsRecognisedByThePresentationWhitelist(t *testing.T) {
|
||||
artifact := domain.ExecutionArtifact{
|
||||
Name: "part.json",
|
||||
Type: domain.PortTypeJSON,
|
||||
Data: partIndexPayload(),
|
||||
}
|
||||
if !artifactLooksLikeCollection(artifact) {
|
||||
t.Fatal("part index is not offered as a structured collection")
|
||||
}
|
||||
}
|
||||
200
internal/api/part_index_members_test.go
Normal file
200
internal/api/part_index_members_test.go
Normal file
@ -0,0 +1,200 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func partIndexPayload() map[string]any {
|
||||
return map[string]any{
|
||||
"schemaVersion": "droboticflow.delivery.part-index/v1",
|
||||
"datasetType": "fanyu-accepted-slice-part-index",
|
||||
"delivery": map[string]any{"kind": "part_index", "fileName": "part.json", "artifactName": "delivery.part-index"},
|
||||
"coverage": map[string]any{"mode": "full_common_range", "sliceCount": 3, "durationSeconds": 3.0},
|
||||
"deliveryClassification": map[string]any{
|
||||
"keptSliceCount": 2, "discardedSliceCount": 1,
|
||||
"keptDurationSeconds": 2.0, "discardedDurationSeconds": 1.0,
|
||||
},
|
||||
"slices": []any{
|
||||
map[string]any{
|
||||
"sliceId": "slice-000001", "sliceName": "拿取前",
|
||||
"taskSplit": map[string]any{"startSecond": 0.0, "endSecond": 1.0, "startTimestampUs": 1000000, "endTimestampUs": 2000000},
|
||||
"classification": map[string]any{"deliveryDisposition": "keep", "descriptionStatus": "described"},
|
||||
"segments": []any{map[string]any{"id": "a1", "description": "拿起物品"}, map[string]any{"id": "a2", "description": "放下物品"}},
|
||||
},
|
||||
map[string]any{
|
||||
"sliceId": "slice-000003", "sliceName": "放置",
|
||||
"taskSplit": map[string]any{"startSecond": 2.0, "endSecond": 3.0, "startTimestampUs": 3000000, "endTimestampUs": 4000000},
|
||||
"classification": map[string]any{"deliveryDisposition": "keep", "descriptionStatus": "empty_confirmed"},
|
||||
"segments": []any{},
|
||||
},
|
||||
},
|
||||
"discardedSlices": []any{
|
||||
map[string]any{
|
||||
"sliceId": "slice-000002", "sliceName": "遮挡",
|
||||
"taskSplit": map[string]any{"startSecond": 1.0, "endSecond": 2.0, "startTimestampUs": 2000000, "endTimestampUs": 3000000},
|
||||
"classification": map[string]any{
|
||||
"deliveryDisposition": "discard",
|
||||
"discardEvidence": map[string]any{"reason": "镜头被遮挡", "reasonName": "遮挡"},
|
||||
},
|
||||
"segments": []any{},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func partIndexRun(t *testing.T, runID string, prefix string) (*store.MemoryStore, domain.ExecutionArtifact) {
|
||||
t.Helper()
|
||||
artifact := domain.ExecutionArtifact{
|
||||
ID: "part-index-artifact", RunID: runID, NodeID: "package", PortID: "deliveryPartIndex", PortName: "deliveryPartIndex",
|
||||
Name: "part.json", Type: domain.PortTypeJSON,
|
||||
Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: domain.PortTypeJSON, Ref: &domain.ArtifactReference{
|
||||
Provider: domain.StorageProviderBOS, ConnectionID: "missing-connection", Bucket: "bucket", Key: prefix + "/part.json",
|
||||
ManifestKey: prefix + "/manifest.json", Kind: domain.ArtifactRefKindObject, SizeBytes: 2048, SHA256: strings.Repeat("a", 64),
|
||||
}},
|
||||
Data: partIndexPayload(),
|
||||
}
|
||||
run := domain.RunRecord{ID: runID, OrganizationID: domain.DefaultOrganizationID, ProjectID: "project-part-index", WorkflowID: "workflow-part-index", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{artifact}}
|
||||
dataStore := store.NewMemoryStore()
|
||||
if err := dataStore.UpsertRun(context.Background(), run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dataStore, artifact
|
||||
}
|
||||
|
||||
func TestPartIndexManifestListsClassificationWithoutMediaMembers(t *testing.T) {
|
||||
runID := "run-part-index"
|
||||
prefix := "deliveries/项目__project/capture/part0001"
|
||||
dataStore, artifact := partIndexRun(t, runID, prefix)
|
||||
handler := runArtifactMembersHandler(t, dataStore, t.TempDir())
|
||||
principal := runArtifactMembersPrincipal(domain.DefaultOrganizationID)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, runArtifactMembersRequest(http.MethodGet, "/api/runs/"+runID+"/artifacts/"+artifact.ID+"/members", principal))
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("members status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
var payload struct {
|
||||
Recognized bool `json:"recognized"`
|
||||
Schema string `json:"schemaVersion"`
|
||||
Items []runArtifactManifestMember `json:"items"`
|
||||
Summary map[string]any `json:"summary"`
|
||||
}
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !payload.Recognized || payload.Schema != "droboticflow.delivery.part-index/v1" {
|
||||
t.Fatalf("part index not recognised: %#v", payload)
|
||||
}
|
||||
// The index is the deliverable: one downloadable JSON, no sliced media.
|
||||
if len(payload.Items) != 1 || payload.Items[0].Name != "part.json" || !payload.Items[0].ContentAvailable {
|
||||
t.Fatalf("unexpected member projection %#v", payload.Items)
|
||||
}
|
||||
if payload.Summary["keptSliceCount"] != float64(2) || payload.Summary["discardedSliceCount"] != float64(1) {
|
||||
t.Fatalf("classification summary missing: %#v", payload.Summary)
|
||||
}
|
||||
slices, ok := payload.Summary["slices"].([]any)
|
||||
if !ok || len(slices) != 2 {
|
||||
t.Fatalf("task-level split missing: %#v", payload.Summary["slices"])
|
||||
}
|
||||
first, _ := slices[0].(map[string]any)
|
||||
if first["sliceId"] != "slice-000001" || first["segmentCount"] != float64(2) || first["descriptionStatus"] != "described" {
|
||||
t.Fatalf("unexpected slice projection %#v", first)
|
||||
}
|
||||
discarded, ok := payload.Summary["discardedSlices"].([]any)
|
||||
if !ok || len(discarded) != 1 {
|
||||
t.Fatalf("discarded classification missing: %#v", payload.Summary["discardedSlices"])
|
||||
}
|
||||
discardedFirst, _ := discarded[0].(map[string]any)
|
||||
if discardedFirst["deliveryDisposition"] != "discard" || discardedFirst["discardReason"] != "镜头被遮挡" {
|
||||
t.Fatalf("discard classification missing: %#v", discardedFirst)
|
||||
}
|
||||
body := response.Body.String()
|
||||
if strings.Contains(body, prefix) || strings.Contains(body, "missing-connection") || strings.Contains(body, `"bucket"`) {
|
||||
t.Fatalf("storage coordinates leaked: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// partIndexDownloadFixture mirrors downloadFixture but delivers a single-file
|
||||
// part index, proving the download control plane treats it as a first-class
|
||||
// final delivery instead of skipping it for lacking a directory ref.
|
||||
func partIndexDownloadFixture(t *testing.T) (*store.MemoryStore, *downloadTestBrowser, http.Handler, *authenticatedPrincipal, string) {
|
||||
t.Helper()
|
||||
ds := store.NewMemoryStore()
|
||||
ctx := t.Context()
|
||||
org := "org-part-index-download"
|
||||
projectID := "project-part-index-download"
|
||||
prefix := "验收/索引项目__" + projectID + "/采集/part0001"
|
||||
b := &downloadTestBrowser{objects: map[string][]byte{}, pages: map[string]objectStoragePage{}}
|
||||
b.objects[prefix+"/.delivery-owner.json"] = []byte(`{"owner":"capture:run"}`)
|
||||
body := []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"part.json"},"slices":[],"discardedSlices":[]}`)
|
||||
b.objects[prefix+"/part.json"] = body
|
||||
manifest := map[string]any{"schemaVersion": 1, "kind": "part-index", "fileCount": 1, "totalBytes": len(body), "entries": []map[string]any{{"path": "part.json", "key": prefix + "/part.json", "sizeBytes": len(body), "sha256": downloadTestHash(string(body))}}}
|
||||
b.objects[prefix+"/manifest.json"], _ = json.Marshal(manifest)
|
||||
old := storageConnectionBrowserResolver
|
||||
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
||||
if c == nil || c.OrganizationID != org {
|
||||
t.Fatal("foreign connection resolved")
|
||||
}
|
||||
return b, nil, nil
|
||||
}
|
||||
t.Cleanup(func() { storageConnectionBrowserResolver = old })
|
||||
for _, e := range []error{
|
||||
ds.UpsertProject(ctx, domain.ProjectRecord{ID: projectID, OrganizationID: org, Name: "索引项目"}),
|
||||
ds.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "part-index-storage", OrganizationID: org, Provider: domain.StorageProviderBOS, Bucket: "test-bucket", Endpoint: "https://bj.bcebos.com", AccessKeyID: "never-send-ak", SecretAccessKey: "never-send-sk"}),
|
||||
ds.UpsertRun(ctx, domain.RunRecord{ID: "part-index-run", OrganizationID: org, ProjectID: projectID, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: "part-index-file", RunID: "part-index-run", Name: "part.json", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindObject, Provider: domain.StorageProviderBOS, ConnectionID: "part-index-storage", Bucket: "test-bucket", Key: prefix + "/part.json", Prefix: prefix, ManifestKey: prefix + "/manifest.json", SizeBytes: int64(len(body))}}}}}),
|
||||
ds.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{ID: "part-index-package", OrganizationID: org, ProjectID: projectID, ChildRunID: "part-index-run", Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction, FinalExportArtifactIDs: []string{"part-index-file"}}),
|
||||
} {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
p := &authenticatedPrincipal{User: domain.UserRecord{ID: "part-index-admin", Role: domain.UserRoleAdmin, OrganizationID: org}, Organization: domain.OrganizationRecord{ID: org}}
|
||||
return ds, b, NewHandler(ds, nil), p, prefix
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryIsDownloadableAsAFinalDelivery(t *testing.T) {
|
||||
_, b, h, p, prefix := partIndexDownloadFixture(t)
|
||||
w := downloadRequest(h, p, "/api/delivery-downloads/manifest?projectId=project-part-index-download")
|
||||
if w.Code != 200 {
|
||||
t.Fatal(w.Code, w.Body.String())
|
||||
}
|
||||
var m deliveryDownloadManifest
|
||||
if e := json.Unmarshal(w.Body.Bytes(), &m); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// part.json plus its completion manifest; no sliced media members.
|
||||
if m.TotalFiles != 2 || len(m.Files) != 2 {
|
||||
t.Fatalf("part index download manifest=%+v", m)
|
||||
}
|
||||
base := "索引项目__project-part-index-download/采集/part0001/"
|
||||
var indexFile *deliveryDownloadFile
|
||||
for i := range m.Files {
|
||||
if m.Files[i].RelativePath == base+"part.json" {
|
||||
indexFile = &m.Files[i]
|
||||
}
|
||||
}
|
||||
if indexFile == nil {
|
||||
t.Fatalf("part.json missing from the download manifest: %+v", m.Files)
|
||||
}
|
||||
if indexFile.SHA256 != downloadTestHash(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"part.json"},"slices":[],"discardedSlices":[]}`) {
|
||||
t.Fatalf("part.json checksum mismatch: %+v", indexFile)
|
||||
}
|
||||
// A downloadable target was resolved from the object ref, not skipped.
|
||||
if len(b.signed) == 0 && m.SnapshotID == "" {
|
||||
t.Fatal("part index produced no resolvable target")
|
||||
}
|
||||
w = downloadRequest(h, p, "/api/delivery-downloads/manifest?projectId=project-part-index-download&snapshot="+m.SnapshotID+"&file="+indexFile.RelativePath)
|
||||
if w.Code != 200 {
|
||||
t.Fatal(w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(strings.Join(b.signed, "\n"), prefix+"/part.json") {
|
||||
t.Fatalf("part.json was never signed: %v", b.signed)
|
||||
}
|
||||
}
|
||||
@ -195,6 +195,8 @@ func (server *server) runArtifactManifestDocumentWithContext(ctx context.Context
|
||||
members = server.annotationManifestMembers(run, manifest, schemaVersion, payload)
|
||||
case "droboticflow.delivery.manifest/v2", "droboticflow.delivery.manifest/v3", "droboticflow.delivery.manifest/v4", "droboticflow.delivery.manifest/v5":
|
||||
members = server.packageManifestMembers(run, manifest, schemaVersion, payload)
|
||||
case "droboticflow.delivery.part-index/v1":
|
||||
return server.partIndexManifestDocument(run, manifest, document, payload)
|
||||
default:
|
||||
document.Message = "暂不支持预览,可下载受保护的引用内容。"
|
||||
return document
|
||||
@ -723,6 +725,85 @@ func (server *server) annotationManifestMembers(run *domain.RunRecord, manifest
|
||||
return members
|
||||
}
|
||||
|
||||
// partIndexManifestDocument presents a single-file part delivery. The index is
|
||||
// the deliverable itself, so the listing exposes the JSON plus the split
|
||||
// classification instead of per-slice media members.
|
||||
func (server *server) partIndexManifestDocument(
|
||||
run *domain.RunRecord,
|
||||
manifest *domain.ExecutionArtifact,
|
||||
document runArtifactManifestDocument,
|
||||
payload map[string]any,
|
||||
) runArtifactManifestDocument {
|
||||
document.Recognized = true
|
||||
document.Members = []runArtifactManifestMember{}
|
||||
classify := func(entries []any, disposition string) []map[string]any {
|
||||
classified := make([]map[string]any, 0, len(entries))
|
||||
for index, entry := range entries {
|
||||
slice := mapValue(entry)
|
||||
if slice == nil {
|
||||
continue
|
||||
}
|
||||
split := mapValue(slice["taskSplit"])
|
||||
classification := mapValue(slice["classification"])
|
||||
segments := listValue(slice["segments"])
|
||||
detail := map[string]any{
|
||||
"sliceId": safeRunArtifactMemberText(firstNonEmptyText(stringValue(slice["sliceId"]), fmt.Sprintf("slice-%06d", index+1)), 300),
|
||||
"sliceName": safeRunArtifactMemberText(stringValue(slice["sliceName"]), 300),
|
||||
"deliveryDisposition": firstNonEmptyText(stringValue(classification["deliveryDisposition"]), disposition),
|
||||
"descriptionStatus": safeRunArtifactMemberText(stringValue(classification["descriptionStatus"]), 80),
|
||||
"segmentCount": len(segments),
|
||||
"startTimestampUs": int64Value(split["startTimestampUs"]),
|
||||
"endTimestampUs": int64Value(split["endTimestampUs"]),
|
||||
"startSecond": floatValue(split["startSecond"]),
|
||||
"endSecond": floatValue(split["endSecond"]),
|
||||
}
|
||||
if evidence := mapValue(classification["discardEvidence"]); evidence != nil {
|
||||
detail["discardReason"] = safeRunArtifactMemberText(stringValue(evidence["reason"]), 1000)
|
||||
}
|
||||
classified = append(classified, detail)
|
||||
}
|
||||
return classified
|
||||
}
|
||||
slices := classify(listValue(payload["slices"]), "keep")
|
||||
discarded := classify(listValue(payload["discardedSlices"]), "discard")
|
||||
counts := mapValue(payload["deliveryClassification"])
|
||||
document.Summary["datasetType"] = safeRunArtifactMemberText(stringValue(payload["datasetType"]), 160)
|
||||
document.Summary["keptSliceCount"] = len(slices)
|
||||
document.Summary["discardedSliceCount"] = len(discarded)
|
||||
document.Summary["keptDurationSeconds"] = floatValue(counts["keptDurationSeconds"])
|
||||
document.Summary["discardedDurationSeconds"] = floatValue(counts["discardedDurationSeconds"])
|
||||
document.Summary["slices"] = slices
|
||||
if len(discarded) > 0 {
|
||||
document.Summary["discardedSlices"] = discarded
|
||||
}
|
||||
if coverage := mapValue(payload["coverage"]); coverage != nil {
|
||||
document.Summary["coverage"] = map[string]any{
|
||||
"mode": safeRunArtifactMemberText(stringValue(coverage["mode"]), 120),
|
||||
"sliceCount": int64Value(coverage["sliceCount"]),
|
||||
"durationSeconds": floatValue(coverage["durationSeconds"]),
|
||||
}
|
||||
}
|
||||
if delivery := mapValue(payload["delivery"]); delivery != nil {
|
||||
document.Summary["deliveryKind"] = safeRunArtifactMemberText(stringValue(delivery["kind"]), 80)
|
||||
document.Summary["fileName"] = safeRunArtifactMemberText(stringValue(delivery["fileName"]), 200)
|
||||
}
|
||||
// The JSON document itself is the single downloadable member. Its bytes are
|
||||
// served through the regular artifact content endpoint, never inlined here.
|
||||
if manifest.Payload != nil && manifest.Payload.Mode == domain.ArtifactTransferModeRef && manifest.Payload.Ref != nil && manifest.Payload.Ref.Kind == domain.ArtifactRefKindObject {
|
||||
ref := manifest.Payload.Ref
|
||||
fileName := firstNonEmptyText(filepath.Base(ref.Key), manifest.Name)
|
||||
member := server.newRunArtifactManifestMember(run, manifest, document.SchemaVersion, "part-index-document", "part-index", fileName, "part-index#"+fileName, "", "application/json", ref.SizeBytes, nil)
|
||||
member.contentArtifact = cloneRunArtifactMemberContentArtifact(*manifest, fileName, "application/json")
|
||||
member.ContentAvailable = true
|
||||
document.Members = []runArtifactManifestMember{member}
|
||||
}
|
||||
document.Summary["contentCount"] = len(document.Members)
|
||||
document.Summary["contentAvailableCount"] = countRunArtifactManifestMembersWithContent(document.Members)
|
||||
document.Summary["groups"] = runArtifactManifestMemberGroups(document.Members)
|
||||
document.Message = "该 part 以单份 JSON 索引交付:切分与分类在索引内,媒体由后续流程按索引打包。"
|
||||
return document
|
||||
}
|
||||
|
||||
func (server *server) packageManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
|
||||
if schema == "droboticflow.delivery.manifest/v5" {
|
||||
return server.packageManifestV5Members(run, manifest, schema, payload)
|
||||
|
||||
@ -352,6 +352,19 @@ func (runtime *Runtime) normalizeOutputArtifacts(ctx context.Context, workflow *
|
||||
continue
|
||||
}
|
||||
info, statErr := os.Stat(artifact.Path)
|
||||
// A part-index delivery is one file, not a tree. It runs before the
|
||||
// generic inline/upload paths so the frozen capture destination and the
|
||||
// shared conflict rules always win over the run's artifact policy.
|
||||
if statErr == nil && info.Mode().IsRegular() && artifact.Name == partIndexFileName {
|
||||
if _, e := partIndexPayloadPeek(artifact.Path); e == nil {
|
||||
delivered, e := runtime.publishCaptureDeliveryFile(ctx, run, nodeID, artifact)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
result[index] = delivered
|
||||
continue
|
||||
}
|
||||
}
|
||||
if statErr == nil && info.Mode().IsRegular() && artifact.Type == domain.PortTypeJSON && info.Size() <= domain.MaxInlinePayloadBytes {
|
||||
payloadBytes, readErr := os.ReadFile(artifact.Path)
|
||||
if readErr != nil {
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@ -76,44 +77,62 @@ func publishDeliveryObject(ctx context.Context, p objectsync.Publisher, c domain
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
|
||||
// deliveryTarget is the validated, owner-pinned destination shared by every
|
||||
// final delivery publication mode.
|
||||
type deliveryTarget struct {
|
||||
record *domain.CaptureRecord
|
||||
target *domain.CaptureDeliveryTarget
|
||||
connection *domain.StorageConnectionRecord
|
||||
publisher objectsync.Publisher
|
||||
lister deliveryPrefixLister
|
||||
owner string
|
||||
stageRoot string
|
||||
marker string
|
||||
publish func(key string, body []byte) error
|
||||
}
|
||||
|
||||
// prepareDeliveryTarget performs every check that must hold before the first
|
||||
// byte of a delivery is written: frozen destination, prefix validity, overlap
|
||||
// with other captures and with enabled intake listeners, delegated scope,
|
||||
// ownership and the conflict rules of the shared destination prefix.
|
||||
func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (*deliveryTarget, error) {
|
||||
record, err := runtime.store.GetCaptureRecordByRunID(ctx, run.ID)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
return nil, err
|
||||
}
|
||||
if record == nil || record.Delivery == nil {
|
||||
return artifact, fmt.Errorf("该任务未配置交付目录,请明确配置后重新接入;不会回退到 runs 目录")
|
||||
return nil, fmt.Errorf("该任务未配置交付目录,请明确配置后重新接入;不会回退到 runs 目录")
|
||||
}
|
||||
target := record.Delivery
|
||||
if !domain.ValidDeliveryPrefix(target.Prefix) {
|
||||
return artifact, fmt.Errorf("无效交付目标目录")
|
||||
return nil, fmt.Errorf("无效交付目标目录")
|
||||
}
|
||||
otherCaptures, lookupErr := runtime.store.ListCaptureRecords(ctx, run.OrganizationID, "", 0)
|
||||
if lookupErr != nil {
|
||||
return artifact, lookupErr
|
||||
return nil, lookupErr
|
||||
}
|
||||
for _, other := range otherCaptures {
|
||||
if other.ID != record.ID && other.Delivery != nil && other.Delivery.Bucket == target.Bucket && domain.StoragePrefixesOverlap(other.Delivery.Prefix, target.Prefix) {
|
||||
return artifact, fmt.Errorf("交付目录与已有任务目标重叠:%s;请先处理旧目录绑定,不会合并或覆盖", other.Delivery.Prefix)
|
||||
return nil, fmt.Errorf("交付目录与已有任务目标重叠:%s;请先处理旧目录绑定,不会合并或覆盖", other.Delivery.Prefix)
|
||||
}
|
||||
}
|
||||
listeners, e := runtime.store.ListEnabledCaptureAutomations(ctx, run.OrganizationID)
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
return nil, e
|
||||
}
|
||||
for _, workflow := range listeners {
|
||||
if c := workflow.CaptureAutomation; c != nil && c.Bucket == target.Bucket && domain.StoragePrefixesOverlap(c.Prefix, target.Prefix) {
|
||||
return artifact, fmt.Errorf("交付位置与已启用监听目录重叠,已停止发布")
|
||||
return nil, fmt.Errorf("交付位置与已启用监听目录重叠,已停止发布")
|
||||
}
|
||||
}
|
||||
connection, err := runtime.captureDeliveryConnection(ctx, run, record, artifact)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
return nil, err
|
||||
}
|
||||
publisher := runtime.outputArtifactPublisher()
|
||||
lister, ok := publisher.(deliveryPrefixLister)
|
||||
if !ok {
|
||||
return artifact, fmt.Errorf("交付存储不支持目录冲突检查")
|
||||
return nil, fmt.Errorf("交付存储不支持目录冲突检查")
|
||||
}
|
||||
ownerRunID := record.RunID
|
||||
// Capture reconciliation may point RunID at its latest release child. The
|
||||
@ -125,7 +144,17 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
|
||||
// PutInternal is conditional-create. A failed invocation may leave staging
|
||||
// objects behind, so each retry needs its own private staging namespace.
|
||||
stageRoot := "system/runs/" + run.ID + "/delivery-staging/" + nodeID + "/" + domain.NewID("publish")
|
||||
publishBytes := func(key string, body []byte) error {
|
||||
prepared := &deliveryTarget{
|
||||
record: record,
|
||||
target: target,
|
||||
connection: connection,
|
||||
publisher: publisher,
|
||||
lister: lister,
|
||||
owner: owner,
|
||||
stageRoot: stageRoot,
|
||||
marker: target.Prefix + "/.delivery-owner.json",
|
||||
}
|
||||
prepared.publish = func(key string, body []byte) error {
|
||||
sum := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(sum[:])
|
||||
stage := stageRoot + "/" + sha
|
||||
@ -137,24 +166,37 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
|
||||
}
|
||||
return publisher.Delete(ctx, *connection, target.Bucket, stage)
|
||||
}
|
||||
marker := target.Prefix + "/.delivery-owner.json"
|
||||
known, err := publisher.Head(ctx, *connection, target.Bucket, marker)
|
||||
known, err := publisher.Head(ctx, *connection, target.Bucket, prepared.marker)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
return nil, err
|
||||
}
|
||||
if !known.Exists {
|
||||
keys, e := lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
|
||||
keys, e := prepared.lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
return nil, e
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
return artifact, fmt.Errorf("目标目录已有文件,拒绝覆盖 %s", target.Prefix)
|
||||
// Refuse to adopt an unowned destination. Part-index migration
|
||||
// therefore requires the destination to carry its owner marker
|
||||
// instead of quietly claiming an existing customer directory.
|
||||
return nil, fmt.Errorf("目标目录已有文件,拒绝覆盖 %s", target.Prefix)
|
||||
}
|
||||
}
|
||||
markerBody, _ := json.Marshal(map[string]string{"owner": owner})
|
||||
if err = publishBytes(marker, markerBody); err != nil {
|
||||
if err = prepared.publish(prepared.marker, markerBody); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
|
||||
prepared, err := runtime.prepareDeliveryTarget(ctx, run, nodeID, artifact)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
|
||||
publishBytes := prepared.publish
|
||||
marker := prepared.marker
|
||||
paths := []string{}
|
||||
err = filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, e error) error {
|
||||
if e != nil {
|
||||
@ -188,7 +230,7 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
|
||||
if strings.HasPrefix(relative, "../") || relative == "manifest.json" || relative == ".delivery-owner.json" {
|
||||
return fmt.Errorf("无效交付成员 %s", relative)
|
||||
}
|
||||
stage := stageRoot + "/files/" + relative
|
||||
stage := prepared.stageRoot + "/files/" + relative
|
||||
meta, sha, e := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, path)
|
||||
if e != nil {
|
||||
return e
|
||||
@ -238,7 +280,7 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
|
||||
for _, entry := range entries {
|
||||
expected[entry.Key] = true
|
||||
}
|
||||
keys, e := lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
|
||||
keys, e := prepared.lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
@ -257,3 +299,95 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
|
||||
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Prefix: target.Prefix, ManifestKey: manifestKey, Kind: domain.ArtifactRefKindDirectory}}
|
||||
return artifact, nil
|
||||
}
|
||||
|
||||
// partIndexSchemaVersion and partIndexArtifactName are the routing contract
|
||||
// shared with the package operator. Both sides must move together.
|
||||
const (
|
||||
partIndexSchemaVersion = "droboticflow.delivery.part-index/v1"
|
||||
partIndexArtifactName = "delivery.part-index"
|
||||
partIndexFileName = "part.json"
|
||||
)
|
||||
|
||||
// partIndexDelivery is the routing block a single-file delivery carries inside
|
||||
// its own payload, so the worker never has to guess from a file name.
|
||||
type partIndexDelivery struct {
|
||||
Kind string `json:"kind"`
|
||||
FileName string `json:"fileName"`
|
||||
}
|
||||
|
||||
// partIndexPayloadPeek reads only the routing block. Payload sizes are bounded
|
||||
// by the operator contract; the full document is verified by sha256 on upload.
|
||||
func partIndexPayloadPeek(path string) (partIndexDelivery, error) {
|
||||
handle, err := os.Open(path)
|
||||
if err != nil {
|
||||
return partIndexDelivery{}, err
|
||||
}
|
||||
defer handle.Close()
|
||||
var probe struct {
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
Delivery *partIndexDelivery `json:"delivery"`
|
||||
}
|
||||
if err = json.NewDecoder(io.LimitReader(handle, 1<<16)).Decode(&probe); err != nil {
|
||||
return partIndexDelivery{}, fmt.Errorf("part index payload is not readable JSON: %w", err)
|
||||
}
|
||||
if probe.Delivery == nil || probe.Delivery.Kind != "part_index" {
|
||||
return partIndexDelivery{}, fmt.Errorf("part index payload is missing its delivery routing block")
|
||||
}
|
||||
if probe.SchemaVersion != partIndexSchemaVersion {
|
||||
return partIndexDelivery{}, fmt.Errorf("unsupported part index schema %q", probe.SchemaVersion)
|
||||
}
|
||||
name := filepath.Base(strings.TrimSpace(probe.Delivery.FileName))
|
||||
if name == "" || name == "." || name == ".." || name != probe.Delivery.FileName {
|
||||
return partIndexDelivery{}, fmt.Errorf("part index payload declares an unsafe file name")
|
||||
}
|
||||
return *probe.Delivery, nil
|
||||
}
|
||||
|
||||
// publishCaptureDeliveryFile places a single file into the frozen part
|
||||
// destination. It shares every protection of the directory publication — one
|
||||
// owner marker per destination, conditional create, sha256/size verification,
|
||||
// idempotent retry and "never overwrite another delivery" — but writes one
|
||||
// member plus the shared completion manifest instead of a tree.
|
||||
func (runtime *Runtime) publishCaptureDeliveryFile(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
|
||||
info, err := os.Lstat(artifact.Path)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return artifact, fmt.Errorf("part index delivery requires a regular file")
|
||||
}
|
||||
routing, err := partIndexPayloadPeek(artifact.Path)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
prepared, err := runtime.prepareDeliveryTarget(ctx, run, nodeID, artifact)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner
|
||||
publishBytes := prepared.publish
|
||||
fileName := filepath.Base(routing.FileName)
|
||||
relative := fileName
|
||||
stage := prepared.stageRoot + "/files/" + relative
|
||||
meta, sha, err := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, artifact.Path)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
key := target.Prefix + "/" + relative
|
||||
if err = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
if err = publisher.Delete(ctx, *connection, target.Bucket, stage); err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
entries := []directoryManifestEntry{{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}}
|
||||
manifestKey := target.Prefix + "/manifest.json"
|
||||
body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "part-index", FileCount: 1, TotalBytes: meta.SizeBytes, Entries: entries})
|
||||
if err = publishBytes(manifestKey, body); err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
artifact.Path = "bos://" + target.Bucket + "/" + key
|
||||
artifact.SizeBytes = meta.SizeBytes
|
||||
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Key: key, ManifestKey: manifestKey, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject}}
|
||||
return artifact, nil
|
||||
}
|
||||
|
||||
246
internal/worker/capture_part_index_test.go
Normal file
246
internal/worker/capture_part_index_test.go
Normal file
@ -0,0 +1,246 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func partIndexFixture(t *testing.T, fileName string, body []byte) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, fileName)
|
||||
if err := os.WriteFile(path, body, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func partIndexBody(t *testing.T, fileName string) []byte {
|
||||
t.Helper()
|
||||
payload := map[string]any{
|
||||
"schemaVersion": "droboticflow.delivery.part-index/v1",
|
||||
"datasetType": "fanyu-accepted-slice-part-index",
|
||||
"delivery": map[string]any{
|
||||
"kind": "part_index",
|
||||
"fileName": fileName,
|
||||
"artifactName": "delivery.part-index",
|
||||
},
|
||||
"slices": []any{},
|
||||
"discardedSlices": []any{},
|
||||
"deliveryClassification": map[string]any{"keptSliceCount": 0},
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func newPartIndexRuntime(t *testing.T, prefix string) (*Runtime, *deliveryPublisher, *domain.CaptureRecord, domain.RunRecord) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
db := store.NewMemoryStore()
|
||||
p := &deliveryPublisher{}
|
||||
runtime := &Runtime{store: db, artifactPublisher: p}
|
||||
connection := domain.StorageConnectionRecord{ID: "c", OrganizationID: "org", Provider: domain.StorageProviderBOS, Bucket: "b"}
|
||||
if err := db.UpsertStorageConnection(ctx, connection); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record := &domain.CaptureRecord{ID: "capture", OrganizationID: "org", RunID: "run-1", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "c", Bucket: "b", Prefix: prefix}}
|
||||
if err := db.UpsertCaptureRecord(ctx, *record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return runtime, p, record, domain.RunRecord{ID: "run-1", OrganizationID: "org"}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryPublishesSingleFileAndRetries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
prefix := "deliveries/测试__project/capture/part0001"
|
||||
runtime, p, _, run := newPartIndexRuntime(t, prefix)
|
||||
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
|
||||
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
|
||||
|
||||
got, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Path != "bos://b/"+prefix+"/part.json" {
|
||||
t.Fatalf("unexpected published path %s", got.Path)
|
||||
}
|
||||
if got.Payload == nil || got.Payload.Ref == nil || got.Payload.Ref.Kind != domain.ArtifactRefKindObject {
|
||||
t.Fatalf("part index must publish as an object ref: %+v", got.Payload)
|
||||
}
|
||||
// No sliced media tree is created by this mode.
|
||||
if _, exists := p.objects["b/"+prefix+"/clips/slice-000001/left.mp4"]; exists {
|
||||
t.Fatal("part index delivery wrote sliced media")
|
||||
}
|
||||
if _, exists := p.objects["b/"+prefix+"/part.json"]; !exists {
|
||||
t.Fatal("part.json missing")
|
||||
}
|
||||
// The completion manifest records exactly one member and stays last.
|
||||
manifestBytes, exists := p.objects["b/"+prefix+"/manifest.json"]
|
||||
if !exists {
|
||||
t.Fatal("manifest.json missing")
|
||||
}
|
||||
var manifest directoryManifest
|
||||
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manifest.Kind != "part-index" || manifest.FileCount != 1 || len(manifest.Entries) != 1 {
|
||||
t.Fatalf("unexpected manifest %+v", manifest)
|
||||
}
|
||||
if manifest.Entries[0].Path != "part.json" || manifest.Entries[0].Key != prefix+"/part.json" {
|
||||
t.Fatalf("unexpected manifest entry %+v", manifest.Entries[0])
|
||||
}
|
||||
sum := sha256.Sum256(partIndexBody(t, "part.json"))
|
||||
if manifest.Entries[0].SHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Fatal("manifest checksum does not match the delivered bytes")
|
||||
}
|
||||
if last := p.published[len(p.published)-1]; last != prefix+"/manifest.json" {
|
||||
t.Fatalf("completion manifest was not written last: %s", last)
|
||||
}
|
||||
|
||||
// An identical retry is idempotent and writes nothing new.
|
||||
count := len(p.published)
|
||||
if _, err = runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(p.published) != count {
|
||||
t.Fatal("retry rewrote a completed part index")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryIsIdempotentAcrossAChangedObjectBody(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
prefix := "deliveries/p__1/capture/part0001"
|
||||
runtime, _, _, run := newPartIndexRuntime(t, prefix)
|
||||
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
|
||||
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
|
||||
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Same owner, different bytes: never silently replace delivered content.
|
||||
changed := map[string]any{
|
||||
"schemaVersion": "droboticflow.delivery.part-index/v1",
|
||||
"delivery": map[string]any{"kind": "part_index", "fileName": "part.json"},
|
||||
"slices": []any{map[string]any{"sliceId": "slice-000001"}},
|
||||
}
|
||||
body, err := json.Marshal(changed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(path, body, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err == nil {
|
||||
t.Fatal("changed content overwrote an existing delivery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryRejectsANewOwner(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
prefix := "deliveries/p__1/capture/part0001"
|
||||
runtime, p, record, run := newPartIndexRuntime(t, prefix)
|
||||
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
|
||||
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
|
||||
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count := len(p.published)
|
||||
record.RunID = "run-2"
|
||||
if err := runtime.store.UpsertCaptureRecord(ctx, *record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.publishCaptureDeliveryFile(ctx, &domain.RunRecord{ID: "run-2", OrganizationID: "org"}, "package", artifact); err == nil {
|
||||
t.Fatal("a new attempt replaced a prior part index")
|
||||
}
|
||||
if len(p.published) != count {
|
||||
t.Fatal("conflict mutated output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryRefusesToAdoptAnUnownedDirectory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
prefix := "deliveries/p__1/capture/part0001"
|
||||
runtime, p, _, run := newPartIndexRuntime(t, prefix)
|
||||
// A legacy directory already exists without an owner marker. Additive
|
||||
// migration must not quietly claim the customer's directory.
|
||||
p.objects = map[string][]byte{"b/" + prefix + "/clips/slice-000001/left.mp4": []byte("legacy")}
|
||||
path := partIndexFixture(t, "part.json", partIndexBody(t, "part.json"))
|
||||
artifact := domain.ExecutionArtifact{Name: "part.json", Path: path, Type: domain.PortTypeJSON}
|
||||
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err == nil {
|
||||
t.Fatal("unowned legacy destination was adopted")
|
||||
}
|
||||
if _, exists := p.objects["b/"+prefix+"/part.json"]; exists {
|
||||
t.Fatal("rejected publication still wrote a member")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryRefusesAnUnrecognisedPayload(t *testing.T) {
|
||||
path := partIndexFixture(t, "part.json", []byte(`{"schemaVersion":"something/else"}`))
|
||||
if _, err := partIndexPayloadPeek(path); err == nil {
|
||||
t.Fatal("unrelated JSON was accepted as a part index")
|
||||
}
|
||||
path = partIndexFixture(t, "part.json", []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"../escape.json"}}`))
|
||||
if _, err := partIndexPayloadPeek(path); err == nil {
|
||||
t.Fatal("unsafe declared file name was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartIndexDeliveryRequiresAFileArtifact(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
prefix := "deliveries/p__1/capture/part0001"
|
||||
runtime, _, _, run := newPartIndexRuntime(t, prefix)
|
||||
artifact := domain.ExecutionArtifact{Name: "part.json", Path: t.TempDir(), Type: domain.PortTypeJSON}
|
||||
if _, err := runtime.publishCaptureDeliveryFile(ctx, &run, "package", artifact); err == nil {
|
||||
t.Fatal("a directory was accepted as a part-index file delivery")
|
||||
}
|
||||
}
|
||||
|
||||
// The routing contract is written twice, once in Python and once here. Read the
|
||||
// producer's own literals so a one-sided rename fails a test instead of leaving
|
||||
// delivered part indexes unpublished.
|
||||
func TestPartIndexRoutingContractMatchesThePackageOperator(t *testing.T) {
|
||||
source, err := os.ReadFile(filepath.Join("..", "..", "backend", "operators", "builtin", "package", "operator.py"))
|
||||
if err != nil {
|
||||
t.Fatalf("package operator source is unavailable: %v", err)
|
||||
}
|
||||
literal := func(pattern string) string {
|
||||
match := regexp.MustCompile(pattern).FindStringSubmatch(string(source))
|
||||
if match == nil {
|
||||
t.Fatalf("package operator no longer declares %s", pattern)
|
||||
}
|
||||
return match[1]
|
||||
}
|
||||
if got := literal(`_PART_INDEX_SCHEMA = "([^"]+)"`); got != partIndexSchemaVersion {
|
||||
t.Fatalf("schema drift: operator=%q worker=%q", got, partIndexSchemaVersion)
|
||||
}
|
||||
if got := literal(`_PART_INDEX_FILE = "([^"]+)"`); got != partIndexFileName {
|
||||
t.Fatalf("file name drift: operator=%q worker=%q", got, partIndexFileName)
|
||||
}
|
||||
if got := literal(`_PART_INDEX_ARTIFACT = "([^"]+)"`); got != partIndexArtifactName {
|
||||
t.Fatalf("artifact name drift: operator=%q worker=%q", got, partIndexArtifactName)
|
||||
}
|
||||
}
|
||||
|
||||
// A payload declaring a different schema must never reach the frozen delivery
|
||||
// destination, even when its file name matches the convention.
|
||||
func TestPartIndexPayloadRejectsAForeignSchema(t *testing.T) {
|
||||
path := partIndexFixture(t, "part.json", []byte(`{"schemaVersion":"droboticflow.delivery.manifest/v6","delivery":{"kind":"part_index","fileName":"part.json"}}`))
|
||||
if _, err := partIndexPayloadPeek(path); err == nil {
|
||||
t.Fatal("a directory manifest was accepted as a part index")
|
||||
}
|
||||
path = partIndexFixture(t, "part.json", []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"directory","fileName":"part.json"}}`))
|
||||
if _, err := partIndexPayloadPeek(path); err == nil {
|
||||
t.Fatal("a non part-index routing kind was accepted")
|
||||
}
|
||||
}
|
||||
@ -661,6 +661,392 @@ def _collect_unique_package_artifacts(
|
||||
return artifacts
|
||||
|
||||
|
||||
_PART_INDEX_SCHEMA = "droboticflow.delivery.part-index/v1"
|
||||
_PART_INDEX_ARTIFACT = "delivery.part-index"
|
||||
_PART_INDEX_FILE = "part.json"
|
||||
# Only these review fields leave the platform. Reviewer identity, free-text
|
||||
# quality comments and unfiltered acceptance statistics stay internal.
|
||||
_QUALITY_REVIEW_EXPORT_FIELDS = ("taskId", "revision", "decision", "reviewerUserId", "createdAt")
|
||||
_ACCEPTANCE_EXPORT_FIELDS = (
|
||||
"confirmed",
|
||||
"sampled",
|
||||
"samplingDecision",
|
||||
"samplingProbability",
|
||||
"samplingPolicyRevision",
|
||||
"samplingRandomValue",
|
||||
"samplingDecidedAt",
|
||||
"acceptedAt",
|
||||
"confirmedByUserId",
|
||||
"processingProfile",
|
||||
"poseDelivery",
|
||||
)
|
||||
|
||||
|
||||
def _optional_float(value: Any) -> float | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _part_index_capture_record(summary_payload: dict[str, Any], revision_manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
if isinstance(summary_payload.get("capture"), dict):
|
||||
return summary_payload["capture"]
|
||||
if isinstance(revision_manifest.get("capture"), dict):
|
||||
return revision_manifest["capture"]
|
||||
return {}
|
||||
|
||||
|
||||
def _part_index_discarded_slice(
|
||||
row: dict[str, Any],
|
||||
source_root: pathlib.Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Classify a discarded interval without shipping its media or old text.
|
||||
|
||||
History of prior descriptions stays internal: the acceptance contract keeps
|
||||
it as review evidence, not as part of the external deliverable.
|
||||
"""
|
||||
slice_id = str(row.get("sliceId") or "")
|
||||
declared = str(row.get("deliveryDisposition") or "")
|
||||
evidence = row.get("discardEvidence") if isinstance(row.get("discardEvidence"), dict) else {}
|
||||
if not str(evidence.get("reason") or "").strip():
|
||||
raise RuntimeError(f"discarded slice {slice_id} has no reason evidence")
|
||||
relative = pathlib.PurePosixPath(str(row.get("directory") or ""))
|
||||
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
||||
raise RuntimeError("unsafe accepted slice directory")
|
||||
source_slice = source_root / pathlib.Path(*relative.parts)
|
||||
if not source_slice.is_dir():
|
||||
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
|
||||
synchronization_path = source_slice / "synchronized-slice.json"
|
||||
if not synchronization_path.is_file():
|
||||
raise RuntimeError(f"accepted slice {slice_id} has no synchronized-slice.json")
|
||||
try:
|
||||
synchronization = json.loads(synchronization_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
||||
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not valid JSON") from error
|
||||
if not isinstance(synchronization, dict):
|
||||
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not an object")
|
||||
recorded = str(synchronization.get("deliveryDisposition") or declared)
|
||||
if recorded != declared:
|
||||
raise RuntimeError(f"accepted slice {slice_id} disposition disagrees with its synchronization evidence")
|
||||
start = _optional_float(row.get("startSecond")) or 0.0
|
||||
end = _optional_float(row.get("endSecond")) or 0.0
|
||||
return {
|
||||
"sliceId": slice_id,
|
||||
"sliceName": str(row.get("sliceName") or ""),
|
||||
"taskSplit": {
|
||||
"startSecond": start,
|
||||
"endSecond": end,
|
||||
"startTimestampUs": int(row["startTimestampUs"]) if row.get("startTimestampUs") is not None else None,
|
||||
"endTimestampUs": int(row["endTimestampUs"]) if row.get("endTimestampUs") is not None else None,
|
||||
"semantics": "half-open",
|
||||
},
|
||||
"classification": {
|
||||
"deliveryDisposition": declared,
|
||||
"discardEvidence": {
|
||||
"reason": str(evidence.get("reason") or ""),
|
||||
"reasonLabelId": str(evidence.get("reasonLabelId") or ""),
|
||||
"reasonName": str(evidence.get("reasonName") or ""),
|
||||
"note": str(evidence.get("note") or ""),
|
||||
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
|
||||
"discardedAt": str(evidence.get("discardedAt") or ""),
|
||||
},
|
||||
},
|
||||
"segments": [],
|
||||
}
|
||||
|
||||
|
||||
def _part_index_calibration_refs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Reference calibration honestly instead of copying it into a delivery tree.
|
||||
|
||||
The part index records where calibration lives and its checksum; the bytes
|
||||
stay in the accepted materialization revision that this index pins.
|
||||
"""
|
||||
refs: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for artifact in _sidecar_artifacts(inputs, {"calibrations"}):
|
||||
name = str(artifact.get("name") or "").strip()
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
path = str(artifact.get("path") or "").strip()
|
||||
digest = str(artifact.get("sha256") or (artifact.get("metadata") or {}).get("sha256") or "").strip().lower()
|
||||
if path and not _SHA.match(digest):
|
||||
try:
|
||||
digest = _sha256_file(pathlib.Path(path))
|
||||
except OSError as error:
|
||||
raise RuntimeError(f"calibration file is unavailable: {name}") from error
|
||||
if not _SHA.match(digest):
|
||||
raise RuntimeError(f"calibration file has no verifiable checksum: {name}")
|
||||
size_bytes = int(artifact.get("sizeBytes") or 0)
|
||||
if path and not size_bytes:
|
||||
try:
|
||||
size_bytes = pathlib.Path(path).stat().st_size
|
||||
except OSError:
|
||||
size_bytes = 0
|
||||
refs.append({"name": name, "path": path, "sha256": digest, "bytes": size_bytes})
|
||||
if not refs:
|
||||
raise RuntimeError("part index delivery requires camera/IMU calibration files")
|
||||
return refs
|
||||
|
||||
|
||||
def _emit_part_index(
|
||||
run_id: str,
|
||||
node_id: str,
|
||||
output: pathlib.Path,
|
||||
node_data: dict[str, Any],
|
||||
helpers: HelperMap,
|
||||
payload_bytes: bytes,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Emit the part index as a single file artifact.
|
||||
|
||||
The artifact path is the deliverable itself: the worker publishes exactly
|
||||
this file into the frozen capture delivery destination. The directory is
|
||||
only a private staging area so a concurrently written node output cannot be
|
||||
captured by accident. Byte content never carries a build timestamp, so a
|
||||
retry of the same accepted revision republishes identical bytes.
|
||||
"""
|
||||
staging = output / "part-index"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
path = staging / _PART_INDEX_FILE
|
||||
path.write_bytes(payload_bytes)
|
||||
return [
|
||||
helpers["build_artifact"](
|
||||
run_id,
|
||||
node_id,
|
||||
path,
|
||||
"json",
|
||||
metadata={"mimeType": "application/json", "sha256": _sha256_file(path)},
|
||||
output=_output(node_data, helpers, "deliveryPartIndex"),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _description_status(actions: list[dict[str, Any]]) -> str:
|
||||
"""Describe a kept slice without reintroducing the internal annotation state.
|
||||
|
||||
``empty_confirmed`` keeps its platform meaning: no description was written
|
||||
and the submitter confirmed that as a deliberate semantic blank.
|
||||
"""
|
||||
if not actions:
|
||||
return "empty_confirmed"
|
||||
if all(bool(action.get("isInvalid")) for action in actions):
|
||||
return "invalid_only"
|
||||
return "described"
|
||||
|
||||
|
||||
def _pose_section(payload: dict[str, Any] | None, kind: str, start: float, end: float) -> dict[str, Any] | None:
|
||||
if payload is None:
|
||||
return None
|
||||
cropped = _crop_pose(payload, start, end, kind)
|
||||
if cropped.get("frameCount", 0) <= 0:
|
||||
return None
|
||||
return cropped
|
||||
|
||||
|
||||
def _part_index_slice(
|
||||
row: dict[str, Any],
|
||||
index: int,
|
||||
source_slice: pathlib.Path,
|
||||
calibration: list[dict[str, Any]],
|
||||
pose_policy: dict[str, Any],
|
||||
headpose: dict[str, Any] | None,
|
||||
handpose: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
declared = str(row.get("deliveryDisposition") or "")
|
||||
slice_id = str(row.get("sliceId") or f"slice-{index + 1:06d}")
|
||||
start = _optional_float(row.get("startSecond")) or 0.0
|
||||
end = _optional_float(row.get("endSecond")) or 0.0
|
||||
annotation_path = source_slice / "annotation.json"
|
||||
if not annotation_path.is_file():
|
||||
raise RuntimeError(f"accepted slice {slice_id} has no annotation.json")
|
||||
try:
|
||||
annotation = json.loads(annotation_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
||||
raise RuntimeError(f"accepted slice {slice_id} annotation.json is not valid JSON") from error
|
||||
if not isinstance(annotation, dict):
|
||||
raise RuntimeError(f"accepted slice {slice_id} annotation.json is not an object")
|
||||
raw_actions = annotation.get("actions")
|
||||
if raw_actions is None:
|
||||
raw_actions = annotation.get("descriptions")
|
||||
if raw_actions is None:
|
||||
raw_actions = []
|
||||
if not isinstance(raw_actions, list):
|
||||
raise RuntimeError(f"accepted slice {slice_id} annotation actions are not a list")
|
||||
actions: list[dict[str, Any]] = []
|
||||
for action in raw_actions:
|
||||
if not isinstance(action, dict):
|
||||
raise RuntimeError(f"accepted slice {slice_id} has a malformed action segment")
|
||||
start_us = action.get("startTimestampUs")
|
||||
end_us = action.get("endTimestampUs")
|
||||
actions.append({
|
||||
"id": str(action.get("id") or ""),
|
||||
"description": str(action.get("description") or ""),
|
||||
"status": str(action.get("status") or ""),
|
||||
"isInvalid": bool(action.get("isInvalid", False)),
|
||||
"startTimestampUs": int(start_us) if start_us is not None else None,
|
||||
"endTimestampUs": int(end_us) if end_us is not None else None,
|
||||
"startSecond": _optional_float(action.get("clipStartMs")) / 1000.0 if _optional_float(action.get("clipStartMs")) is not None else None,
|
||||
"endSecond": _optional_float(action.get("clipEndMs")) / 1000.0 if _optional_float(action.get("clipEndMs")) is not None else None,
|
||||
"clipStartMs": _optional_float(action.get("clipStartMs")),
|
||||
"clipEndMs": _optional_float(action.get("clipEndMs")),
|
||||
"labelId": str(action.get("descriptionTemplateId") or ""),
|
||||
"phrases": [str(item) for item in action.get("phrases", [])] if isinstance(action.get("phrases"), list) else [],
|
||||
"source": str(action.get("source") or ""),
|
||||
"confidence": _optional_float(action.get("confidence")),
|
||||
})
|
||||
synchronization_path = source_slice / "synchronized-slice.json"
|
||||
if not synchronization_path.is_file():
|
||||
raise RuntimeError(f"accepted slice {slice_id} has no synchronized-slice.json")
|
||||
try:
|
||||
synchronization = json.loads(synchronization_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
||||
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not valid JSON") from error
|
||||
if not isinstance(synchronization, dict):
|
||||
raise RuntimeError(f"accepted slice {slice_id} synchronized-slice.json is not an object")
|
||||
# A disposition that disagrees with the pinned review revision must fail
|
||||
# closed instead of silently exporting the wrong classification.
|
||||
recorded = str(synchronization.get("deliveryDisposition") or declared)
|
||||
if recorded != declared:
|
||||
raise RuntimeError(f"accepted slice {slice_id} disposition disagrees with its synchronization evidence")
|
||||
entry: dict[str, Any] = {
|
||||
"sliceId": slice_id,
|
||||
"sliceName": str(row.get("sliceName") or ""),
|
||||
"taskSplit": {
|
||||
"startSecond": start,
|
||||
"endSecond": end,
|
||||
"startTimestampUs": int(row["startTimestampUs"]) if row.get("startTimestampUs") is not None else None,
|
||||
"endTimestampUs": int(row["endTimestampUs"]) if row.get("endTimestampUs") is not None else None,
|
||||
"semantics": "half-open",
|
||||
},
|
||||
"classification": {
|
||||
"deliveryDisposition": declared,
|
||||
"taskLabel": str(synchronization.get("taskLabel") or ""),
|
||||
"taskLabelId": str(synchronization.get("taskLabelId") or ""),
|
||||
"descriptionStatus": _description_status(actions),
|
||||
},
|
||||
"segments": actions,
|
||||
"synchronization": {
|
||||
"maxStereoAlignmentErrorUs": synchronization.get("maxStereoAlignmentErrorUs"),
|
||||
"boundaryContract": str(synchronization.get("boundaryContract") or ""),
|
||||
"boundaryTimestampUs": [int(value) for value in synchronization.get("boundaryTimestampUs", [])] if isinstance(synchronization.get("boundaryTimestampUs"), list) else [],
|
||||
"calibration": [
|
||||
{"name": item["name"], "path": item["path"], "sha256": item["sha256"]}
|
||||
for item in calibration
|
||||
],
|
||||
},
|
||||
}
|
||||
if declared == "discard":
|
||||
evidence = row.get("discardEvidence") if isinstance(row.get("discardEvidence"), dict) else {}
|
||||
if not str(evidence.get("reason") or "").strip():
|
||||
raise RuntimeError(f"discarded slice {slice_id} has no reason evidence")
|
||||
# Discard classification is exported deliberately: downstream packaging
|
||||
# must know why an interval is excluded.
|
||||
entry["classification"]["discardEvidence"] = {
|
||||
"reason": str(evidence.get("reason") or ""),
|
||||
"reasonLabelId": str(evidence.get("reasonLabelId") or ""),
|
||||
"reasonName": str(evidence.get("reasonName") or ""),
|
||||
"note": str(evidence.get("note") or ""),
|
||||
"discardedByUserId": str(evidence.get("discardedByUserId") or ""),
|
||||
"discardedAt": str(evidence.get("discardedAt") or ""),
|
||||
}
|
||||
if pose_policy.get("headpose") == "include":
|
||||
cropped = _pose_section(headpose, "headpose", start, end)
|
||||
if cropped is None:
|
||||
raise RuntimeError(f"HeadPose has no frames for accepted slice {slice_id}")
|
||||
entry.setdefault("pose", {})["headpose"] = cropped
|
||||
if pose_policy.get("handpose") == "include":
|
||||
cropped = _pose_section(handpose, "handpose", start, end)
|
||||
if cropped is None:
|
||||
raise RuntimeError(f"HandPose has no frames for accepted slice {slice_id}")
|
||||
entry.setdefault("pose", {})["handpose"] = cropped
|
||||
return entry
|
||||
|
||||
|
||||
def _build_part_index(
|
||||
revision_manifest: dict[str, Any],
|
||||
coverage: dict[str, Any],
|
||||
kept_slices: list[dict[str, Any]],
|
||||
source_root: pathlib.Path,
|
||||
calibration: list[dict[str, Any]],
|
||||
pose_policy: dict[str, Any],
|
||||
headpose: dict[str, Any] | None,
|
||||
handpose: dict[str, Any] | None,
|
||||
summary_payload: dict[str, Any],
|
||||
final_acceptance: dict[str, Any],
|
||||
capture_record: dict[str, Any],
|
||||
lineage: dict[str, Any],
|
||||
discarded_slices: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, Any], bytes]:
|
||||
slices: list[dict[str, Any]] = []
|
||||
for index, row in enumerate(kept_slices):
|
||||
if not isinstance(row, dict):
|
||||
raise RuntimeError("invalid accepted slice entry")
|
||||
relative = pathlib.PurePosixPath(str(row.get("directory") or ""))
|
||||
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
||||
raise RuntimeError("unsafe accepted slice directory")
|
||||
source_slice = source_root / pathlib.Path(*relative.parts)
|
||||
if not source_slice.is_dir():
|
||||
raise RuntimeError("accepted slice manifest contains an unavailable slice directory")
|
||||
slices.append(_part_index_slice(row, index, source_slice, calibration, pose_policy, headpose, handpose))
|
||||
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
|
||||
exported_reviews = [
|
||||
{key: review[key] for key in _QUALITY_REVIEW_EXPORT_FIELDS if key in review}
|
||||
for review in quality_reviews
|
||||
if isinstance(review, dict)
|
||||
]
|
||||
payload: dict[str, Any] = {
|
||||
"schemaVersion": _PART_INDEX_SCHEMA,
|
||||
"datasetType": "fanyu-accepted-slice-part-index",
|
||||
# The worker publishes this file unchanged into the part's frozen
|
||||
# delivery destination and needs stable routing facts, not a filename
|
||||
# convention it has to re-derive.
|
||||
"delivery": {
|
||||
"kind": "part_index",
|
||||
"fileName": _PART_INDEX_FILE,
|
||||
"artifactName": _PART_INDEX_ARTIFACT,
|
||||
},
|
||||
"capture": capture_record,
|
||||
"revision": revision_manifest.get("revision"),
|
||||
"coverage": coverage,
|
||||
"slices": slices,
|
||||
"discardedSlices": [
|
||||
_part_index_discarded_slice(row, source_root)
|
||||
for row in discarded_slices
|
||||
if isinstance(row, dict)
|
||||
],
|
||||
"deliveryClassification": {
|
||||
"keptSliceCount": len(slices),
|
||||
"discardedSliceCount": len(discarded_slices),
|
||||
"keptDurationSeconds": sum(float(item["taskSplit"]["endSecond"]) - float(item["taskSplit"]["startSecond"]) for item in slices),
|
||||
"discardedDurationSeconds": sum(
|
||||
(_optional_float(row.get("endSecond")) or 0.0) - (_optional_float(row.get("startSecond")) or 0.0)
|
||||
for row in discarded_slices
|
||||
if isinstance(row, dict)
|
||||
),
|
||||
},
|
||||
"calibration": calibration,
|
||||
"poseDelivery": pose_policy,
|
||||
"acceptance": {
|
||||
"sampling": {
|
||||
"sampled": bool(final_acceptance.get("sampled", True)),
|
||||
"decision": str(final_acceptance.get("samplingDecision") or "selected"),
|
||||
"probability": final_acceptance.get("samplingProbability"),
|
||||
"policyRevision": final_acceptance.get("samplingPolicyRevision"),
|
||||
"randomValue": final_acceptance.get("samplingRandomValue"),
|
||||
"decidedAt": final_acceptance.get("samplingDecidedAt"),
|
||||
},
|
||||
"finalAcceptance": {key: final_acceptance[key] for key in _ACCEPTANCE_EXPORT_FIELDS if key in final_acceptance},
|
||||
"qualityReviews": exported_reviews,
|
||||
},
|
||||
"lineage": lineage,
|
||||
}
|
||||
body = (json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
||||
return payload, body
|
||||
|
||||
|
||||
def _copy_calibrations(inputs: list[dict[str, Any]], packages_root: pathlib.Path, output: pathlib.Path, helpers: HelperMap) -> list[dict[str, Any]]:
|
||||
destination = packages_root / "calibration"
|
||||
result: list[dict[str, Any]] = []
|
||||
@ -1107,6 +1493,7 @@ def _run_accepted_slice_directory(
|
||||
summary_artifact = next((item for item in inputs if _port(item) == "acceptanceSummary"), None)
|
||||
summary_payload = summary_artifact.get("data") if isinstance((summary_artifact or {}).get("data"), dict) else {}
|
||||
final_acceptance = summary_payload.get("finalAcceptance") if isinstance(summary_payload.get("finalAcceptance"), dict) else {}
|
||||
capture_record = _part_index_capture_record(summary_payload, revision_manifest)
|
||||
all_slices = revision_manifest.get("items", [])
|
||||
if not isinstance(all_slices, list) or not all_slices:
|
||||
raise RuntimeError("accepted slice manifest has no review evidence")
|
||||
@ -1151,6 +1538,12 @@ def _run_accepted_slice_directory(
|
||||
payloads = _payload_tree(_payloads_with_full_pose_manifests(inputs, output, helpers))
|
||||
headpose = _first_payload(payloads, "droboticflow.headpose/v1")
|
||||
handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1")
|
||||
params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {}
|
||||
if _bool(params, "PART_INDEX_DELIVERY", False):
|
||||
calibration_refs = _part_index_calibration_refs(inputs)
|
||||
lineage = {"acceptedSliceDirectoryArtifactId": directory_artifact.get("id"), "humanItemId": revision_manifest.get("humanItemId"), "taskId": revision_manifest.get("taskId"), "sourceRunId": revision_manifest.get("runId")}
|
||||
_, body = _build_part_index(revision_manifest, coverage, kept_slices, source_root, calibration_refs, pose_policy, headpose, handpose, summary_payload, final_acceptance, capture_record, lineage, discarded_slices)
|
||||
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
|
||||
delivery_root = pathlib.Path(tempfile.mkdtemp(prefix="delivery-", dir=output)) if disposition_contract else output / "delivery"
|
||||
clips_root = delivery_root / "clips"
|
||||
clips_root.mkdir(parents=True, exist_ok=True)
|
||||
@ -1241,7 +1634,6 @@ def _run_accepted_slice_directory(
|
||||
"randomValue": final_acceptance.get("samplingRandomValue"),
|
||||
"decidedAt": final_acceptance.get("samplingDecidedAt"),
|
||||
}
|
||||
capture_record = summary_payload.get("capture") if isinstance(summary_payload.get("capture"), dict) else revision_manifest.get("capture") if isinstance(revision_manifest.get("capture"), dict) else {}
|
||||
quality_reviews = summary_payload.get("qualityReviews") if isinstance(summary_payload.get("qualityReviews"), list) else []
|
||||
if disposition_contract:
|
||||
quality_reviews = [{k:r[k] for k in ("taskId","revision","decision","reviewerUserId","createdAt") if k in r} for r in quality_reviews if isinstance(r,dict)]
|
||||
@ -1380,6 +1772,12 @@ def _run_multiport(
|
||||
|
||||
|
||||
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
|
||||
if _output(node_data, helpers, "packageDirectory") is not None or _output(node_data, helpers, "deliveryDirectory") is not None:
|
||||
# A part-index delivery intentionally has no directory port; it must still
|
||||
# reach the accepted-slice path instead of the legacy inline contract.
|
||||
if (
|
||||
_output(node_data, helpers, "packageDirectory") is not None
|
||||
or _output(node_data, helpers, "deliveryDirectory") is not None
|
||||
or _output(node_data, helpers, "deliveryPartIndex") is not None
|
||||
):
|
||||
return _run_multiport(run_id, node_id, output_dir, node_data, inputs, helpers)
|
||||
return _run_existing_contract(run_id, node_id, output_dir, node_data, inputs, helpers)
|
||||
|
||||
178
tests/backend/test_part_index_package.py
Normal file
178
tests/backend/test_part_index_package.py
Normal file
@ -0,0 +1,178 @@
|
||||
"""Part-index delivery contract: one JSON per part, no sliced media tree."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from backend.operators.builtin.package import operator as package
|
||||
|
||||
from tests.backend.test_slice_discard_package import NODE as BASE_NODE
|
||||
from tests.backend.test_slice_discard_package import fixture
|
||||
|
||||
|
||||
PART_NODE = {
|
||||
"outputs": [{"id": name, "name": name} for name in ("deliveryPartIndex", "deliverySummary")],
|
||||
"hyperparams": [{"key": "PART_INDEX_DELIVERY", "value": "true"}],
|
||||
}
|
||||
|
||||
|
||||
def helpers():
|
||||
def write(run, node, directory, name, payload, output=None):
|
||||
path = Path(directory) / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload), encoding="utf8")
|
||||
return {"id": run + name, "name": name, "path": str(path), "data": payload, "portId": output["id"]}
|
||||
|
||||
def build(run, node, path, artifact_type, data=None, metadata=None, output=None):
|
||||
return {
|
||||
"id": run + Path(path).stem,
|
||||
"name": Path(path).name,
|
||||
"path": str(path),
|
||||
"type": artifact_type,
|
||||
"metadata": dict(metadata or {}),
|
||||
"portId": output["id"] if output else "",
|
||||
}
|
||||
|
||||
return {
|
||||
"write_json_artifact": write,
|
||||
"build_artifact": build,
|
||||
"build_virtual_artifact": lambda run, node, name, path, kind, size, data=None, output=None, **kw: {
|
||||
"id": run + name,
|
||||
"name": name,
|
||||
"path": path,
|
||||
"data": data,
|
||||
"portId": output["id"] if output else "",
|
||||
},
|
||||
"payloads_from_inputs": lambda items: [i.get("data", {}) for i in items],
|
||||
"hyperparams": lambda node: {str(p.get("key")): str(p.get("value")) for p in node.get("hyperparams", []) if isinstance(p, dict)},
|
||||
"source_path_from_item": lambda item, *args: Path(item.get("path") or item["source"]["path"]),
|
||||
"source_ref_from_artifact": lambda item: {"path": item.get("path")},
|
||||
}
|
||||
|
||||
|
||||
def run_part_index(tmp_path, dispositions, node=None, root=None):
|
||||
inputs = fixture(tmp_path, dispositions)
|
||||
if node is None:
|
||||
# The shared fixture is a raw-human revision; give it the pose decision
|
||||
# the accepted-slice contract requires so the part-index branch runs.
|
||||
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "exclude", "handpose": "exclude"}
|
||||
artifacts = package.run("r", "p", root or tmp_path / "output", node or PART_NODE, inputs, helpers())
|
||||
return inputs, artifacts
|
||||
|
||||
|
||||
def read_index(artifact):
|
||||
return json.loads(Path(artifact["path"]).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_part_index_is_a_single_json_delivery_with_both_split_levels(tmp_path):
|
||||
_, artifacts = run_part_index(tmp_path, ["keep", "discard", "keep"])
|
||||
assert len(artifacts) == 1
|
||||
artifact = artifacts[0]
|
||||
assert artifact["portId"] == "deliveryPartIndex"
|
||||
assert artifact["name"] == "part.json"
|
||||
# The artifact path is the deliverable itself; no sliced media tree exists.
|
||||
assert Path(artifact["path"]).is_file()
|
||||
assert not list((tmp_path / "output").rglob("*.mp4"))
|
||||
|
||||
index = read_index(artifact)
|
||||
assert index["schemaVersion"] == "droboticflow.delivery.part-index/v1"
|
||||
assert index["delivery"] == {
|
||||
"kind": "part_index",
|
||||
"fileName": "part.json",
|
||||
"artifactName": "delivery.part-index",
|
||||
}
|
||||
|
||||
# Task-level split carries every delivered interval with its classification.
|
||||
assert [entry["sliceId"] for entry in index["slices"]] == ["slice-000001", "slice-000003"]
|
||||
assert [entry["classification"]["deliveryDisposition"] for entry in index["slices"]] == ["keep", "keep"]
|
||||
assert all(entry["taskSplit"]["semantics"] == "half-open" for entry in index["slices"])
|
||||
assert index["deliveryClassification"]["keptSliceCount"] == 2
|
||||
assert index["deliveryClassification"]["discardedSliceCount"] == 1
|
||||
# The pinned review revision still covers every reviewed interval.
|
||||
assert index["coverage"]["sliceCount"] == 3
|
||||
|
||||
# Action-level split is part of the same document.
|
||||
first = index["slices"][0]
|
||||
assert first["segments"][0]["description"] == "无效片段"
|
||||
assert first["segments"][0]["isInvalid"] is True
|
||||
assert first["classification"]["descriptionStatus"] == "invalid_only"
|
||||
|
||||
|
||||
def test_discarded_slices_are_delivered_as_their_own_classification(tmp_path):
|
||||
_, artifacts = run_part_index(tmp_path, ["keep", "discard"])
|
||||
index = read_index(artifacts[0])
|
||||
assert [entry["sliceId"] for entry in index["slices"]] == ["slice-000001"]
|
||||
assert [entry["sliceId"] for entry in index["discardedSlices"]] == ["slice-000002"]
|
||||
discarded = index["discardedSlices"][0]
|
||||
assert discarded["classification"]["deliveryDisposition"] == "discard"
|
||||
assert discarded["classification"]["discardEvidence"]["reason"] == "PRIVATE_DISCARD_REASON"
|
||||
# No media members: the index only classifies, it does not ship bytes.
|
||||
assert "files" not in discarded and "mediaKey" not in discarded
|
||||
|
||||
|
||||
def test_calibration_is_referenced_with_checksums(tmp_path):
|
||||
_, artifacts = run_part_index(tmp_path, ["keep"])
|
||||
index = read_index(artifacts[0])
|
||||
assert {entry["name"] for entry in index["calibration"]} == {"calibration_camera.yaml", "calibration_imu.yaml"}
|
||||
for entry in index["calibration"]:
|
||||
assert len(entry["sha256"]) == 64
|
||||
assert entry["bytes"] > 0
|
||||
for entry in index["slices"]:
|
||||
assert len(entry["synchronization"]["calibration"]) == 2
|
||||
|
||||
|
||||
def test_review_comments_and_acceptance_statistics_stay_internal(tmp_path):
|
||||
_, artifacts = run_part_index(tmp_path, ["keep", "discard"])
|
||||
body = Path(artifacts[0]["path"]).read_text(encoding="utf-8")
|
||||
# The quality review comment shares its text with the discard reason here,
|
||||
# so assert on the review projection itself rather than raw text.
|
||||
index = json.loads(body)
|
||||
assert "statistics" not in index["acceptance"]["finalAcceptance"]
|
||||
assert all("comment" not in review for review in index["acceptance"]["qualityReviews"])
|
||||
|
||||
|
||||
def test_part_index_bytes_are_reproducible_for_the_same_revision(tmp_path):
|
||||
inputs, _ = run_part_index(tmp_path, ["keep", "discard"], root=tmp_path / "a")
|
||||
first = package.run("r", "p", tmp_path / "a", PART_NODE, inputs, helpers())
|
||||
second = package.run("r", "p", tmp_path / "b", PART_NODE, inputs, helpers())
|
||||
assert Path(first[0]["path"]).read_bytes() == Path(second[0]["path"]).read_bytes()
|
||||
|
||||
|
||||
def test_disposition_disagreement_fails_closed(tmp_path):
|
||||
inputs = fixture(tmp_path, ["keep"])
|
||||
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "exclude", "handpose": "exclude"}
|
||||
synchronization = Path(inputs[0]["path"]) / "slices/slice-000001/synchronized-slice.json"
|
||||
synchronization.write_text(json.dumps({"schemaVersion": "droboticflow.synchronized-slice/v1", "deliveryDisposition": "discard"}))
|
||||
with pytest.raises(RuntimeError, match="disposition disagrees"):
|
||||
package.run("r", "p", tmp_path / "output", PART_NODE, inputs, helpers())
|
||||
|
||||
|
||||
def test_part_index_keeps_requiring_an_approved_revision(tmp_path):
|
||||
inputs = fixture(tmp_path, ["keep", "discard", "keep"])
|
||||
inputs[1]["data"]["finalAcceptance"]["confirmed"] = False
|
||||
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "exclude", "handpose": "exclude"}
|
||||
with pytest.raises(RuntimeError):
|
||||
package.run("r", "p", tmp_path / "output", PART_NODE, inputs, helpers())
|
||||
|
||||
|
||||
def test_all_discard_still_emits_no_deliverable(tmp_path):
|
||||
_, artifacts = run_part_index(tmp_path, ["discard", "discard"])
|
||||
assert len(artifacts) == 1 and artifacts[0]["portId"] == "deliverySummary"
|
||||
assert artifacts[0]["data"]["outcome"] == "no_deliverables"
|
||||
assert not list((tmp_path / "output").glob("part-index"))
|
||||
|
||||
|
||||
def test_missing_calibration_fails_closed(tmp_path):
|
||||
inputs = fixture(tmp_path, ["keep"])
|
||||
inputs[1]["data"]["finalAcceptance"]["poseDelivery"] = {"headpose": "exclude", "handpose": "exclude"}
|
||||
trimmed = [item for item in inputs if item.get("portId") != "calibrations"]
|
||||
with pytest.raises(RuntimeError, match="calibration"):
|
||||
package.run("r", "p", tmp_path / "output", PART_NODE, trimmed, helpers())
|
||||
|
||||
|
||||
def test_directory_contract_is_unchanged_when_the_switch_is_off(tmp_path, monkeypatch):
|
||||
inputs = fixture(tmp_path, ["keep", "discard", "keep"])
|
||||
monkeypatch.setattr(package, "_ffprobe_frame_count", lambda path: 2)
|
||||
artifacts = package.run("r", "p", tmp_path / "output", BASE_NODE, inputs, helpers())
|
||||
directory = Path(next(a["path"] for a in artifacts if a["portId"] == "deliveryDirectory"))
|
||||
assert {entry.name for entry in (directory / "clips").iterdir()} == {"slice-000001", "slice-000003"}
|
||||
Loading…
x
Reference in New Issue
Block a user