2026-08-25 04:17:45 +08:00

607 lines
34 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Delivery package operator with v5 final-segment and pose-cropping guarantees."""
from __future__ import annotations
import hashlib
import json
import pathlib
import re
import shutil
import subprocess
from typing import Any, Callable
HelperMap = dict[str, Callable[..., Any]]
_SHA = re.compile(r"^[a-f0-9]{64}$")
def _sha256_file(path: pathlib.Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def _bool(params: dict[str, str], key: str, default: bool = False) -> bool:
value = str(params.get(key, str(default))).strip().lower()
if value in {"1", "true", "yes", "on"}:
return True
if value in {"0", "false", "no", "off"}:
return False
raise RuntimeError(f"{key} must be boolean")
def _source_video_name(item: dict[str, Any]) -> str:
direct = str(item.get("sourceVideo", "")).strip()
if direct:
return direct
annotation = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
annotated = str(annotation.get("sourceVideo", "")).strip()
if annotated:
return annotated
source = item.get("source") if isinstance(item.get("source"), dict) else {}
named = str(source.get("name", "")).strip()
if named:
return named
return pathlib.PurePosixPath(str(item.get("sourceClipPath") or item.get("clipPath") or "")).name
def _payload_tree(value: Any) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
if isinstance(value, dict):
items.append(value)
for key in ("upstreamPayloads", "payloads", "inputs"):
nested = value.get(key)
if isinstance(nested, list):
for child in nested:
items.extend(_payload_tree(child))
elif isinstance(value, list):
for child in value:
items.extend(_payload_tree(child))
return items
def _first_payload(payloads: list[dict[str, Any]], schema: str) -> dict[str, Any] | None:
return next((payload for payload in payloads if payload.get("schemaVersion") == schema), None)
def _pose_timestamp(frame: dict[str, Any]) -> float | None:
for key in ("timestampSeconds", "timestamp", "timeSeconds"):
if key in frame:
try:
return float(frame[key])
except (TypeError, ValueError):
return None
if "timestampMs" in frame:
try:
return float(frame["timestampMs"]) / 1000.0
except (TypeError, ValueError):
return None
return None
def _crop_pose(payload: dict[str, Any], start: float, end: float, kind: str) -> dict[str, Any]:
raw_frames: Any = payload.get("frames")
if kind == "handpose" and not isinstance(raw_frames, list):
result = payload.get("result") if isinstance(payload.get("result"), dict) else {}
raw_frames = result.get("frames")
frames: list[dict[str, Any]] = []
seen: dict[tuple[float, str], str] = {}
for frame in raw_frames if isinstance(raw_frames, list) else []:
if not isinstance(frame, dict):
continue
timestamp = _pose_timestamp(frame)
# Final segments are half-open so a frame at a shared boundary belongs
# to the following segment exactly once.
if timestamp is None or timestamp < start or timestamp >= end:
continue
cropped = dict(frame)
relative = round(timestamp - start, 6)
cropped["timestampSeconds"] = relative
cropped["sourceTimestampSeconds"] = timestamp
cropped.pop("timestampMs", None)
identity = (timestamp, str(cropped.get("frameIndex", cropped.get("sourceFrame", ""))))
encoded = json.dumps(cropped, sort_keys=True, ensure_ascii=False)
if identity in seen:
if seen[identity] != encoded:
raise RuntimeError(f"conflicting duplicate {kind} pose frame at source timestamp {timestamp}")
continue
seen[identity] = encoded
frames.append(cropped)
frames.sort(key=lambda item: (float(item["timestampSeconds"]), str(item.get("frameIndex", item.get("sourceFrame", "")))))
return {
"schemaVersion": f"droboticflow.{kind}.segment/v1",
"enabled": bool(payload.get("enabled", True)),
"segmentStartSourceSeconds": start,
"segmentEndSourceSeconds": end,
"frameCount": len(frames),
"frames": frames,
"provenance": {
key: payload.get(key)
for key in ("schemaVersion", "model", "modelSha256", "serviceImageId", "licenseId", "service", "runtimeEvidence", "source")
if key in payload
},
}
def _pose_frames(payload: dict[str, Any], kind: str) -> list[dict[str, Any]]:
frames = payload.get("frames")
if kind == "handpose" and not isinstance(frames, list):
result = payload.get("result") if isinstance(payload.get("result"), dict) else {}
frames = result.get("frames")
return [frame for frame in frames if isinstance(frame, dict)] if isinstance(frames, list) else []
def _camera_collection_id(value: Any) -> str:
if isinstance(value, dict):
data = value.get("data") if isinstance(value.get("data"), dict) else {}
direct = str(value.get("collectionId") or data.get("collectionId") or "").strip()
if direct:
return direct
value = value.get("sourceVideo") or value.get("name") or value.get("path") or ""
name = pathlib.PurePosixPath(str(value or "").strip()).name
match = re.match(r"(.+)_camera_(?:left|right)_part\d+", name, re.IGNORECASE)
return match.group(1) if match else pathlib.PurePosixPath(name).stem
def _right_video_artifacts(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [
artifact
for artifact in inputs
if str(artifact.get("portId") or artifact.get("portName") or "").strip()
in {"rightVideos", "rightRectifiedVideos", "rightSanitizedVideos"}
]
def _matching_right_video(item: dict[str, Any], candidates: list[dict[str, Any]]) -> dict[str, Any] | None:
if not candidates:
return None
collection_id = str(item.get("collectionId") or "").strip() or _camera_collection_id(_source_video_name(item))
matches = [candidate for candidate in candidates if _camera_collection_id(candidate) == collection_id]
if len(matches) == 1:
return matches[0]
if len(candidates) == 1:
return candidates[0]
if not matches:
raise RuntimeError(f"no right-camera video matches collection {collection_id}")
raise RuntimeError(f"multiple right-camera videos match collection {collection_id}")
def _materialize_input_video(
artifact: dict[str, Any],
output: pathlib.Path,
helpers: HelperMap,
) -> pathlib.Path:
materialize = helpers.get("source_path_from_item")
if not callable(materialize):
raise RuntimeError("stereo delivery cannot materialize the right-camera video")
source_ref = helpers.get("source_ref_from_artifact")
source = source_ref(artifact) if callable(source_ref) else artifact.get("source", {})
name = str(artifact.get("name") or "right-camera.mp4").strip()
path = materialize(
{
"source": source if isinstance(source, dict) else {},
"path": str(artifact.get("path", "")),
},
name,
output,
{},
)
resolved = pathlib.Path(path)
if not resolved.is_file() or resolved.stat().st_size <= 0:
raise RuntimeError("right-camera video is unavailable for synchronized cropping")
return resolved
def _write_verified_segment(
source: pathlib.Path,
destination: pathlib.Path,
start: float,
end: float,
helpers: HelperMap,
) -> dict[str, Any]:
_write_precise_final_segment(source, destination, start, end)
metrics = helpers["inspect_video_file"](destination)
actual_duration = float(metrics.get("duration") or 0.0)
expected_duration = end - start
if actual_duration <= 0 or abs(actual_duration - expected_duration) > 0.35:
raise RuntimeError(
f"synchronized stereo segment duration mismatch: expected {expected_duration:.3f}s, got {actual_duration:.3f}s"
)
return {
"name": destination.name,
"sha256": _sha256_file(destination),
"bytes": destination.stat().st_size,
"durationSeconds": actual_duration,
}
def _validate_final_acceptance(item: dict[str, Any]) -> None:
evidence = item.get("finalAcceptance") if isinstance(item.get("finalAcceptance"), dict) else {}
canonical = str(evidence.get("canonical", ""))
digest = str(evidence.get("sha256", "")).lower()
if item.get("finalAcceptanceConfirmed") is not True or not canonical or not _SHA.fullmatch(digest):
raise RuntimeError("v5 package requires verified pre-package acceptance evidence")
if hashlib.sha256(canonical.encode("utf-8")).hexdigest() != digest:
raise RuntimeError("pre-package acceptance evidence SHA-256 does not verify")
if not str(evidence.get("confirmedByUserId", "")).strip():
raise RuntimeError("pre-package acceptance evidence has no confirmer identity")
def _write_precise_final_segment(source: pathlib.Path, destination: pathlib.Path, start: float, end: float) -> None:
if shutil.which("ffmpeg") is None:
raise RuntimeError("final package requires ffmpeg for precise segment materialization")
duration = end - start
destination.parent.mkdir(parents=True, exist_ok=True)
completed = subprocess.run(
[
"ffmpeg",
"-nostdin",
"-hide_banner",
"-loglevel",
"error",
"-y",
"-i",
str(source),
"-ss",
f"{start:.6f}",
"-t",
f"{duration:.6f}",
"-map",
"0:v:0",
"-map",
"0:a?",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"23",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
"-reset_timestamps",
"1",
"-avoid_negative_ts",
"make_zero",
str(destination),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if completed.returncode != 0:
message = (completed.stderr or completed.stdout or "unknown ffmpeg error").strip()
raise RuntimeError(f"precise final segment ffmpeg failed: {message}")
def _materialize_final_mp4(item: dict[str, Any], package_dir: pathlib.Path, helpers: HelperMap) -> pathlib.Path:
name = str(item.get("clipName") or f"{item.get('clipId', 'segment')}.mp4")
destination = package_dir / name
source_path = pathlib.Path(str(item.get("clipPath", "")).strip())
if not source_path.is_file() or source_path.stat().st_size <= 0:
source = item.get("source") if isinstance(item.get("source"), dict) else {}
source_path = helpers["source_path_from_item"](
{"source": source or {"path": str(item.get("clipPath", "")).strip()}},
str(item.get("sourceVideo") or pathlib.PurePosixPath(str(item.get("clipPath", ""))).name or "source.mp4"),
package_dir,
)
if not source_path.is_file() or source_path.stat().st_size <= 0:
raise RuntimeError(f"final segment source video is missing or empty: {source_path}")
start = float(item.get("startSecond", 0.0))
end = float(item.get("endSecond", 0.0))
if not (0 <= start < end):
raise RuntimeError("final segment must contain valid source-relative boundaries")
_write_precise_final_segment(source_path, destination, start, end)
if not destination.is_file() or destination.stat().st_size <= 0:
raise RuntimeError("final segment MP4 was not materialized")
metrics = helpers["inspect_video_file"](destination)
actual_duration = float(metrics.get("duration") or 0.0)
requested_duration = end - start
if actual_duration <= 0 or abs(actual_duration - requested_duration) > 0.35:
raise RuntimeError(f"final segment MP4 duration mismatch: expected {requested_duration:.3f}s, got {actual_duration:.3f}s")
return destination
def _collect_unique_package_artifacts(
run_id: str,
node_id: str,
packages_root: pathlib.Path,
node_data: dict[str, Any],
binding: Any,
helpers: HelperMap,
) -> list[dict[str, Any]]:
artifacts = helpers["collect_output_artifacts"](run_id, node_id, packages_root, node_data=node_data, output=binding)
seen: set[str] = set()
for artifact in artifacts:
path = pathlib.Path(str(artifact.get("path", "")))
try:
relative = path.relative_to(packages_root).as_posix()
except ValueError:
relative = path.name
suffix = hashlib.sha256(relative.encode("utf-8")).hexdigest()[:12]
base_id = str(artifact.get("id") or f"{run_id}-{node_id}-package-artifact")
artifact["id"] = f"{base_id}-{suffix}"
if artifact["id"] in seen:
raise RuntimeError(f"package artifact id collision for {relative}")
seen.add(artifact["id"])
return artifacts
def _run_fanyu_v5(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
output = pathlib.Path(output_dir)
output.mkdir(parents=True, exist_ok=True)
params = helpers["hyperparams"](node_data)
require_headpose = _bool(params, "REQUIRE_HEADPOSE", False)
require_handpose = _bool(params, "REQUIRE_HANDPOSE", False)
require_stereo_delivery = _bool(params, "REQUIRE_STEREO_DELIVERY", False)
pose_free = _bool(params, "POSE_FREE", False)
require_final_acceptance = _bool(params, "REQUIRE_FINAL_ACCEPTANCE", False)
generate_segment_mp4 = _bool(params, "GENERATE_SEGMENT_MP4", False)
payloads = _payload_tree(helpers["payloads_from_inputs"](inputs))
headpose = _first_payload(payloads, "droboticflow.headpose/v1")
handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1")
if pose_free and (headpose is not None or handpose is not None):
raise RuntimeError("pose-free v5 package rejects HeadPose and HandPose payloads")
if require_headpose and (headpose is None or not headpose.get("enabled") or not isinstance(headpose.get("frames"), list)):
raise RuntimeError("package requires a real Headpose manifest with frames")
if require_handpose and (
handpose is None
or handpose.get("enabled") is False
or not _pose_frames(handpose, "handpose")
):
raise RuntimeError("package requires a real WiLoR HandPose manifest with frames")
right_videos = _right_video_artifacts(inputs)
if require_stereo_delivery and not right_videos:
raise RuntimeError("stereo delivery requires a right-camera rectified video")
accepted = next((payload for payload in payloads if isinstance(payload.get("items"), list) and ("annotationCount" in payload or payload.get("schemaVersion") == "droboticflow.annotation.accepted-partition/v1")), None)
if accepted is None:
raise RuntimeError("package requires an accepted annotation partition")
package_items: list[dict[str, Any]] = []
binding = helpers["output_binding"](node_data)
for index, item in enumerate(accepted.get("items", [])):
if not isinstance(item, dict):
continue
if require_final_acceptance:
_validate_final_acceptance(item)
clip_id = str(item.get("clipId") or f"segment-{index + 1:04d}").strip()
package_dir = output / "packages" / f"{clip_id}_package"
package_dir.mkdir(parents=True, exist_ok=True)
start = float(item.get("sourceStartSecond", item.get("startSecond", 0.0)))
end = float(item.get("sourceEndSecond", item.get("endSecond", 0.0)))
if not (0 <= start < end):
raise RuntimeError("accepted package item has invalid final source boundaries")
# 虚拟切片条目没有物理中间片段:无论是否显式要求 MP4,
# 都必须按源视频区间在打包阶段精确物化最终交付 MP4。
if generate_segment_mp4 or item.get("finalSegment") is True or item.get("virtualCut") is True:
video_path = _materialize_final_mp4(item, package_dir, helpers)
else:
source = pathlib.Path(str(item.get("clipPath", "")).strip())
if not source.is_file() or source.stat().st_size <= 0:
raise RuntimeError("legacy package input clip is unavailable")
video_path = package_dir / (str(item.get("clipName") or source.name))
shutil.copy2(source, video_path)
annotation = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
headpose_segment = _crop_pose(headpose, start, end, "headpose") if headpose else None
handpose_segment = _crop_pose(handpose, start, end, "handpose") if handpose and handpose.get("enabled", True) else None
if require_headpose and (headpose_segment is None or headpose_segment["frameCount"] <= 0):
raise RuntimeError("required Headpose has no frame inside the final half-open segment interval")
if require_handpose and (handpose_segment is None or handpose_segment["frameCount"] <= 0):
raise RuntimeError("required HandPose has no frame inside the final half-open segment interval")
right_video = _matching_right_video(item, right_videos)
right_video_manifest = None
if right_video is not None:
right_source = _materialize_input_video(right_video, output, helpers)
right_destination = package_dir / f"{clip_id}_right.mp4"
right_video_manifest = _write_verified_segment(right_source, right_destination, start, end, helpers)
right_video_manifest["cameraSide"] = "right"
right_video_manifest["sourceVideo"] = str(right_video.get("name") or right_source.name)
right_data = right_video.get("data") if isinstance(right_video.get("data"), dict) else {}
right_video_manifest["transformChain"] = (
right_data.get("transformChain") if isinstance(right_data.get("transformChain"), list) else []
)
elif require_stereo_delivery:
raise RuntimeError(f"accepted segment {clip_id} has no matching right-camera video")
helpers["write_json_artifact"](run_id, node_id, package_dir, "semantic.json", annotation, output=binding)
segment_manifest = {
"schemaVersion": "droboticflow.delivery.segment/v2",
"clipId": clip_id,
"video": {"name": video_path.name, "sha256": _sha256_file(video_path), "bytes": video_path.stat().st_size},
"sourceRangeSeconds": {"start": start, "end": end},
"clipRelativeRangeSeconds": {"start": float(item.get("startSecond", 0.0)), "end": float(item.get("endSecond", 0.0))},
"semantic": {"file": "semantic.json", "actionCount": len(annotation.get("actions", []))},
"transformChain": item.get("transformChain", []) if isinstance(item.get("transformChain"), list) else [],
"finalAcceptance": item.get("finalAcceptance") if require_final_acceptance else None,
}
segment_manifest["videos"] = {
"left": {**segment_manifest["video"], "cameraSide": "left"},
**({"right": right_video_manifest} if right_video_manifest else {}),
}
if headpose_segment is not None:
helpers["write_json_artifact"](run_id, node_id, package_dir, "headpose.json", headpose_segment, output=binding)
segment_manifest["headpose"] = {"file": "headpose.json", "frameCount": headpose_segment["frameCount"]}
if handpose_segment is not None:
helpers["write_json_artifact"](run_id, node_id, package_dir, "handpose.json", handpose_segment, output=binding)
segment_manifest["handpose"] = {"file": "handpose.json", "frameCount": handpose_segment["frameCount"]}
segment_manifest["handPrelabel"] = {"file": "handpose.json", "frameCount": handpose_segment["frameCount"], "reviewRequired": True}
helpers["write_json_artifact"](run_id, node_id, package_dir, "segment.json", segment_manifest, output=binding)
source_video = _source_video_name(item)
quality = {"clip_id": clip_id, "source_video": source_video, **(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {})}
package_items.append({"packageName": package_dir.name, "clipId": clip_id, "clipName": video_path.name, "rightClipName": right_video_manifest["name"] if right_video_manifest else "", "sourceVideo": source_video, "sourceRightVideo": right_video_manifest["sourceVideo"] if right_video_manifest else "", "stereo": right_video_manifest is not None, "storagePath": str(package_dir), "startSecond": start, "endSecond": end, "clipDuration": end - start, "duration": end - start, "fileCount": len([path for path in package_dir.iterdir() if path.is_file()]), "quality": quality, "annotation": annotation, "segmentManifest": segment_manifest})
if not package_items:
raise RuntimeError("package produced zero final segment packages")
manifest = {"schemaVersion": "droboticflow.delivery.manifest/v3", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "stereoDelivery": all(item["stereo"] for item in package_items), "headposeIncluded": headpose is not None, "handposeIncluded": handpose is not None and handpose.get("enabled", True), "handPrelabelIncluded": handpose is not None and handpose.get("enabled", True), "items": package_items, "trainingEnabled": False}
artifacts = [helpers["write_json_artifact"](run_id, node_id, output, "packages-manifest.json", manifest, output=binding)]
artifacts.extend(_collect_unique_package_artifacts(run_id, node_id, output / "packages", node_data, binding, helpers))
return artifacts
def _run_legacy(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
"""The pre-v5 contract, intentionally retained byte-for-byte in shape."""
payloads = helpers["payloads_from_inputs"](inputs)
params = helpers["hyperparams"](node_data)
require_headpose = _bool(params, "REQUIRE_HEADPOSE", False)
headpose = _first_payload(payloads, "droboticflow.headpose/v1")
handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1")
if require_headpose and (headpose is None or not headpose.get("enabled")):
raise RuntimeError("成品打包需要必选的 droboticflow.headpose/v1 结果")
headpose_enabled = bool(headpose and headpose.get("enabled"))
handpose_enabled = bool(handpose and handpose.get("enabled", True))
payload = helpers["first_payload_with_keys"](inputs, "annotationCount", "items")
if payload is None:
payload = helpers["first_payload_with_keys"](inputs, "clipCount", "items")
if payload is None:
raise RuntimeError("成品打包需要上游标注或目标裁剪结果输入")
output = pathlib.Path(output_dir)
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
packages: list[dict[str, Any]] = []
binding = helpers["output_binding"](node_data)
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
clip_name = str(item.get("clipName", "")).strip()
clip_id = str(item.get("clipId", "")).strip() or pathlib.Path(clip_name).stem
package_dir = output / "packages" / f"{clip_id}_package"
package_dir.mkdir(parents=True, exist_ok=True)
artifact = clip_artifacts.get(clip_name)
clip_path: pathlib.Path | None = None
if artifact is not None:
try:
clip_path = helpers["source_path_from_item"]({"source": helpers["source_ref_from_artifact"](artifact), "path": artifact.get("path", "")}, clip_name, output, archive_sources)
except RuntimeError:
clip_path = None
if clip_path is None:
direct = str(item.get("clipPath", "")).strip()
if direct and pathlib.Path(direct).exists():
clip_path = pathlib.Path(direct)
if clip_path is None:
clip_path = helpers["materialize_clip_from_source"](item, output, archive_sources)
if clip_path is not None and clip_path.exists():
shutil.copy2(clip_path, package_dir / clip_name)
quality = {"clip_id": clip_id, "source_video": item.get("sourceVideo", ""), **(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {})}
annotation = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
if not annotation and isinstance(item.get("detections"), list):
annotation = {"type": "target_detections", "detectionMode": item.get("detectionMode", payload.get("detectionMode", "")), "targetLabels": item.get("targetLabels", payload.get("targetLabels", [])), "detections": item.get("detections", []), "detectionCount": item.get("detectionCount", 0), "maxConfidence": item.get("maxConfidence", 0)}
config = helpers["package_config_payload"](item, payload)
config["deliveryItems"] = {"headpose": {"enabled": bool(headpose and headpose.get("enabled")), "frameCount": headpose.get("frameCount", 0) if headpose else 0}, "handpose": {"enabled": bool(handpose and handpose.get("enabled", True)), "status": handpose.get("status", "missing") if handpose else "missing"}}
helpers["write_json_artifact"](run_id, node_id, package_dir, f"{clip_id}_quality.json", quality, output=binding)
if headpose_enabled:
helpers["write_json_artifact"](run_id, node_id, package_dir, f"{clip_id}_headpose.json", headpose, output=binding)
if handpose_enabled:
helpers["write_json_artifact"](run_id, node_id, package_dir, f"{clip_id}_handpose.json", handpose, output=binding)
helpers["write_json_artifact"](run_id, node_id, package_dir, f"{clip_id}_annotation.json", annotation, output=binding)
helpers["write_json_artifact"](run_id, node_id, package_dir, f"{clip_id}_config.json", config, output=binding)
package_files = [candidate for candidate in package_dir.rglob("*") if candidate.is_file()]
packages.append({"packageName": package_dir.name, "clipId": clip_id, "clipName": clip_name, "sourceVideo": item.get("sourceVideo", ""), "clipDuration": item.get("duration", 0), "duration": item.get("duration", 0), "startSecond": item.get("startSecond"), "endSecond": item.get("endSecond"), "collectionId": item.get("collectionId", ""), "meta": item.get("meta", {}), "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, "fileCount": len(package_files), "pipelineTags": ["quality_v1", "slicing_v1", "annotation_v1", "headpose_v1", *(["handpose_v1"] if handpose_enabled else [])], "storagePath": str(package_dir), "quality": quality, "annotation": annotation, "deliveryItems": config["deliveryItems"]})
manifest = {"datasetType": "x-platform-packages", "packageCount": len(packages), "deliveryItems": {"headpose": {"enabled": bool(headpose and headpose.get("enabled")), "schemaVersion": headpose.get("schemaVersion") if headpose else ""}, "handpose": {"enabled": bool(handpose and handpose.get("enabled", True)), "schemaVersion": handpose.get("schemaVersion") if handpose else ""}}, "items": packages}
artifacts = [helpers["write_json_artifact"](run_id, node_id, output, "packages-manifest.json", manifest, output=binding)]
artifacts.extend(helpers["collect_output_artifacts"](run_id, node_id, output / "packages", node_data=node_data, output=binding))
return artifacts
def _run_existing_contract(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
if not _bool(params, "REQUIRE_FINAL_ACCEPTANCE", False) and not _bool(params, "GENERATE_SEGMENT_MP4", False):
return _run_legacy(run_id, node_id, output_dir, node_data, inputs, helpers)
return _run_fanyu_v5(run_id, node_id, output_dir, node_data, inputs, helpers)
def _output(node_data: dict[str, Any], helpers: HelperMap, name: str) -> dict[str, Any] | None:
del helpers
for output in node_data.get("outputs", []) if isinstance(node_data.get("outputs"), list) else []:
if isinstance(output, dict) and name in {str(output.get("id", "")).strip(), str(output.get("name", "")).strip()}:
return output
return None
def _port(artifact: dict[str, Any]) -> str:
return str(artifact.get("portId") or artifact.get("portName") or "").strip()
def _materialize_annotation_payload(artifact: dict[str, Any], output: pathlib.Path, helpers: HelperMap) -> dict[str, Any]:
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
if isinstance(data.get("items"), list):
return data
materialize = helpers.get("source_path_from_item")
source_ref = helpers.get("source_ref_from_artifact")
if not callable(materialize):
raise RuntimeError("成品打包无法物化 annotations 引用")
source = source_ref(artifact) if callable(source_ref) else {}
path = materialize({"path": str(artifact.get("path", "")), "source": source if isinstance(source, dict) else {}}, str(artifact.get("name", "annotations.json")), output, {})
decoded = json.loads(pathlib.Path(path).read_text(encoding="utf-8"))
if not isinstance(decoded, dict) or not isinstance(decoded.get("items"), list):
raise RuntimeError("annotations 引用不是可打包的 accepted annotation collection")
return decoded
def _run_multiport(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
output = pathlib.Path(output_dir)
approved = [
item
for item in inputs
if _port(item) == "approvedVideos" or (not _port(item) and str(item.get("type", "")) == "video")
]
annotation_artifact = next((item for item in inputs if _port(item) == "annotations"), None)
if not approved or annotation_artifact is None:
# A newly published package image must still execute historical graphs
# that supply one legacy annotations/manifest input.
return _run_existing_contract(run_id, node_id, output_dir, node_data, inputs, helpers)
accepted = _materialize_annotation_payload(annotation_artifact, output, helpers)
# Existing precise Fanyu packager owns ffmpeg cutting and acceptance
# validation. Feed it a local technical adapter only; business outputs are
# rebound below by their real media/file role.
context_inputs = [item for item in inputs if item not in approved and item is not annotation_artifact]
generated = _run_fanyu_v5(
run_id,
node_id,
output,
node_data,
[{"name": "accepted-annotations-index.json", "data": accepted}, *approved, *context_inputs],
helpers,
)
packages_root = output / "packages"
package_binding = _output(node_data, helpers, "packageDirectory")
result: list[dict[str, Any]] = []
if package_binding is not None:
result.append(helpers["build_virtual_artifact"](run_id, node_id, "delivery-package-directory", str(packages_root), "file", 0, data={"packageCount": len([path for path in packages_root.iterdir() if path.is_dir()])}, output=package_binding))
for artifact in generated:
name = str(artifact.get("name", "")).lower()
binding = None
if name == "packages-manifest.json":
binding = package_binding
elif name.endswith(".mp4"):
binding = _output(node_data, helpers, "finalVideos")
elif name == "semantic.json" or name.endswith("_annotation.json"):
binding = _output(node_data, helpers, "semanticFiles")
elif name == "segment.json" or name.endswith("_segment.json"):
binding = _output(node_data, helpers, "segmentFiles")
elif name == "headpose.json" or name.endswith("_headpose.json"):
binding = _output(node_data, helpers, "headposeFiles")
elif name == "handpose.json" or name.endswith("_handpose.json"):
binding = _output(node_data, helpers, "handposeFiles")
if binding is None:
continue
artifact["portId"] = str(binding.get("id", ""))
artifact["portName"] = str(binding.get("name", ""))
result.append(artifact)
manifest_path = output / "packages-manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.exists() else {}
params = helpers["hyperparams"](node_data)
summary = {"packageCount": int(manifest.get("packageCount", 0) or 0), "trainingEnabled": False, "poseFree": _bool(params, "POSE_FREE", False), "stereoDelivery": bool(manifest.get("stereoDelivery")), "headposeIncluded": bool(manifest.get("headposeIncluded")), "handposeIncluded": bool(manifest.get("handposeIncluded")), "handPrelabelIncluded": bool(manifest.get("handPrelabelIncluded", manifest.get("handposeIncluded"))), "finalAcceptanceRequired": _bool(params, "REQUIRE_FINAL_ACCEPTANCE", False)}
result.append(helpers["write_json_artifact"](run_id, node_id, output, "delivery-summary.json", summary, output=_output(node_data, helpers, "deliverySummary")))
return result
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
if _output(node_data, helpers, "packageDirectory") is not None:
return _run_multiport(run_id, node_id, output_dir, node_data, inputs, helpers)
return _run_existing_contract(run_id, node_id, output_dir, node_data, inputs, helpers)