#!/usr/bin/env python3 """把一个已交付的逐片目录形态交付,增量迁移为 part 级总 JSON 索引。 只新增 `part.json`,不删除 `clips/`、`calibration/` 或任何既有清单。 用法: python3 scripts/part_index_migration.py --delivery-root /path/to/part0001 \\ --revision-root /path/to/human-materializations///revision-000001 \\ --out /tmp/part-index-dry-run 设计边界: - 本脚本**不写对象存储**。发布必须经过 API/Worker 的 `prepareDeliveryTarget` 通道,才能保留 `.delivery-owner.json` 归属校验、条件创建与「绝不覆盖他人交付」 语义;直连 BOS 会绕过它们。本脚本只产出候选产物与迁移计划。 - 复用打包算子自身的 part-index 构建逻辑(同一份源码),不复制第二套实现, 因此迁移产物与在线打包逐字节同构。 - 交付目录里的 `annotation.json` 与 `synchronized-slice.json` 是打包时写出的 对外版本;索引只从它们读取已对外的事实,舍弃原因取自 `revision-manifest.json` 与 `synchronized-slice.json` 的 `discardEvidence`。 - 交付目录已有内容不同的 `part.json` 时直接拒绝,不覆盖、不删除任何对象。 """ from __future__ import annotations import argparse import hashlib import importlib.util import json import pathlib import sys ROOT = pathlib.Path(__file__).resolve().parents[1] OPERATOR_SOURCE = ROOT / "backend" / "operators" / "builtin" / "package" / "operator.py" PART_INDEX_SCHEMA = "droboticflow.delivery.part-index/v1" PART_INDEX_FILE = "part.json" def load_operator_module() -> object: """Load the package operator by path, exactly like the operator runner does. Only module-level helpers are used, so no runtime helper map is required. """ spec = importlib.util.spec_from_file_location("droboticflow_part_index_migration", OPERATOR_SOURCE) if spec is None or spec.loader is None: raise RuntimeError(f"cannot load package operator from {OPERATOR_SOURCE}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def sha256_file(path: pathlib.Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1 << 20), b""): digest.update(block) return digest.hexdigest() def load_json(path: pathlib.Path) -> dict: try: decoded = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as error: raise SystemExit(f"无法读取 {path}:{error}") from error if not isinstance(decoded, dict): raise SystemExit(f"{path} 不是 JSON 对象") return decoded def delivery_completion_manifest(delivery_root: pathlib.Path) -> dict: manifest_path = delivery_root / "manifest.json" if not manifest_path.is_file(): raise SystemExit(f"{delivery_root} 不是已完成的交付目录:缺少 manifest.json") manifest = load_json(manifest_path) kind = str(manifest.get("kind") or "") if kind != "directory": raise SystemExit(f"该交付的 manifest kind={kind!r},只有 directory 形态需要迁移") return manifest def delivered_calibrations(delivery_root: pathlib.Path, operator: object) -> list[dict]: """Reference the calibration bytes that are already delivered. Builds the same artifact shape the operator's calibration helper expects, then reads the checksum from the delivered file itself. """ directory = delivery_root / "calibration" if not directory.is_dir(): raise SystemExit("交付目录缺少 calibration/,无法生成索引") artifacts = [] for path in sorted(directory.iterdir()): if path.is_file(): # The helper accepts either a logical or an absolute location; it # records only the logical one in the index. artifacts.append({"name": path.name, "path": str(path), "portId": "calibrations"}) if not artifacts: raise SystemExit("交付目录的 calibration/ 为空,无法生成索引") return operator._part_index_calibration_refs(artifacts) def resolve_pose_policy(acceptance: dict) -> dict: """Recover the pose decision, or prove it was raw-human with no pose output.""" final = acceptance.get("finalAcceptance") if isinstance(acceptance.get("finalAcceptance"), dict) else {} pose = final.get("poseDelivery") if isinstance(final.get("poseDelivery"), dict) else {} if pose.get("headpose") in {"include", "exclude"} and pose.get("handpose") in {"include", "exclude"}: return pose if final.get("processingProfile") == "raw_human": return {"headpose": "not_generated", "handpose": "not_generated"} raise SystemExit("交付证据里没有可用的姿态交付决策,拒绝猜测 HeadPose/HandPose 策略") def split_slices(revision_manifest: dict) -> tuple[list[dict], list[dict]]: rows = revision_manifest.get("items") if not isinstance(rows, list) or not rows: raise SystemExit("物化修订清单没有切片条目") kept: list[dict] = [] discarded: list[dict] = [] for row in rows: if not isinstance(row, dict): raise SystemExit("物化修订清单包含无效切片条目") disposition = str(row.get("deliveryDisposition") or "") if disposition == "keep": kept.append(row) elif disposition == "discard": if not str((row.get("discardEvidence") or {}).get("reason") or "").strip(): raise SystemExit(f"舍弃切片 {row.get('sliceId')} 没有原因证据") discarded.append(row) else: raise SystemExit(f"切片 {row.get('sliceId')} 的用途未知:{disposition!r}") return kept, discarded def build_index(delivery_root: pathlib.Path, revision_root: pathlib.Path) -> tuple[dict, bytes]: operator = load_operator_module() delivery_completion_manifest(delivery_root) revision_manifest = load_json(revision_root / "revision-manifest.json") schema = str(revision_manifest.get("schemaVersion") or "") if schema not in {"droboticflow.slice-materialization/v2", "droboticflow.slice-materialization/v3"}: raise SystemExit(f"不支持的物化修订清单:{schema!r}") if schema == "droboticflow.slice-materialization/v2": raise SystemExit("v2 修订清单没有逐片用途证据,无法判定保留与舍弃;请先确认该 part 的交付契约") coverage = revision_manifest.get("coverage") if isinstance(revision_manifest.get("coverage"), dict) else {} kept, discarded = split_slices(revision_manifest) if not kept: raise SystemExit("该 part 没有保留切片;迁移不产生交付物") acceptance = load_json(delivery_root / "acceptance-record.json") capture_manifest = load_json(delivery_root / "capture-manifest.json") delivery_manifest = load_json(delivery_root / "packages-manifest.json") capture_record = delivery_manifest.get("capture") or capture_manifest.get("capture") or {} summary_payload = { "decision": "approved", "processingProfile": (acceptance.get("finalAcceptance") or {}).get("processingProfile"), "qualityReviews": (load_json(delivery_root / "quality-review.json") or {}).get("reviews") or [], "capture": capture_record, } lineage = { "acceptedSliceDirectoryArtifactId": delivery_manifest.get("lineage", {}).get("acceptedSliceDirectoryArtifactId"), "humanItemId": revision_manifest.get("humanItemId"), "taskId": revision_manifest.get("taskId"), "sourceRunId": revision_manifest.get("runId"), "migratedFrom": "droboticflow.delivery.manifest/v6", } return operator._build_part_index( revision_manifest, coverage, kept, revision_root, delivered_calibrations(delivery_root, operator), resolve_pose_policy(acceptance), summary_payload, acceptance.get("finalAcceptance") or {}, capture_record, lineage, discarded, ) def part_name_from_prefix(prefix: str) -> str: """The frozen delivery prefix already ends with the part directory.""" name = prefix.rstrip("/").rsplit("/", 1)[-1] return name or "part0001" def run_migration(delivery_root: pathlib.Path, revision_root: pathlib.Path, out_dir: pathlib.Path, part_name: str) -> dict: """Produce the candidate index outside the delivery directory. Unchanged-to-delivered content is a no-op; different content is refused so a migration can never silently replace what the customer already received. """ delivery_root = delivery_root.resolve() revision_root = revision_root.resolve() out_dir = out_dir.resolve() if not delivery_root.is_dir(): raise SystemExit(f"--delivery-root 不存在:{delivery_root}") if not revision_root.is_dir(): raise SystemExit(f"--revision-root 不存在:{revision_root}") if out_dir == delivery_root or delivery_root in out_dir.parents: raise SystemExit("--out 不能位于交付目录内;迁移只新增交付物,不改写本地副本") payload, body = build_index(delivery_root, revision_root) existing = delivery_root / PART_INDEX_FILE if existing.is_file(): if existing.read_bytes() == body: return {"status": "already-migrated", "path": str(existing), "sha256": sha256_file(existing)} raise SystemExit("交付目录已有内容不同的 part.json,拒绝覆盖;请人工确认该 part 的交付状态") out_dir.mkdir(parents=True, exist_ok=True) target = out_dir / PART_INDEX_FILE target.write_bytes(body) return { "status": "dry-run", "schemaVersion": PART_INDEX_SCHEMA, "part": part_name, "written": str(target), "sha256": sha256_file(target), "bytes": len(body), "keptSliceCount": len(payload.get("slices") or []), "discardedSliceCount": len(payload.get("discardedSlices") or []), } def main() -> int: parser = argparse.ArgumentParser(description="增量迁移已交付的逐片目录为 part 级总 JSON 索引(dry run)。") parser.add_argument("--delivery-root", required=True, help="已交付 part 目录的本地副本") parser.add_argument("--revision-root", required=True, help="对应的物化修订目录(含 revision-manifest.json 与 slices/)") parser.add_argument("--out", required=True, help="写出候选 part.json 的目录(不会写入交付目录)") parser.add_argument("--prefix", help="该 part 的冻结交付前缀,仅用于推断目录名") args = parser.parse_args() report = run_migration( pathlib.Path(args.delivery_root), pathlib.Path(args.revision_root), pathlib.Path(args.out), part_name_from_prefix(args.prefix or args.delivery_root), ) print(json.dumps(report, ensure_ascii=False, indent=2)) if report["status"] == "dry-run": print("未上传:发布必须经过 API/Worker 的交付通道。") return 0 if __name__ == "__main__": sys.exit(main())