droboticflow/scripts/part_index_migration.py

244 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""把一个已交付的逐片目录形态交付,增量迁移为 part 级总 JSON 索引。
只新增 `part.json`,不删除 `clips/`、`calibration/` 或任何既有清单。
用法:
python3 scripts/part_index_migration.py --delivery-root /path/to/part0001 \\
--revision-root /path/to/human-materializations/<item>/<task>/revision-000001 \\
--out /tmp/part-index-dry-run
设计边界:
- 本脚本**不写对象存储**。发布必须经过 API/Worker 的 `prepareDeliveryTarget`
通道,才能保留 `.delivery-owner.json` 归属校验、条件创建与「绝不覆盖他人交付」
语义;直连 BOS 会绕过它们。本脚本只产出候选产物与迁移计划。
- 复用打包算子自身的 part-index 构建逻辑(同一份源码),不复制第二套实现,
因此迁移产物与在线打包逐字节同构。
- 交付目录里的 `annotation.json` 与 `synchronized-slice.json` 是打包时写出的
对外版本;索引只从它们读取已对外的事实,舍弃原因取自 `revision-manifest.json`
与 `synchronized-slice.json` 的 `discardEvidence`。
- 交付目录已有内容不同的 `part.json` 时直接拒绝,不覆盖、不删除任何对象。
"""
from __future__ import annotations
import argparse
import hashlib
import importlib.util
import json
import pathlib
import sys
ROOT = pathlib.Path(__file__).resolve().parents[1]
OPERATOR_SOURCE = ROOT / "backend" / "operators" / "builtin" / "package" / "operator.py"
PART_INDEX_SCHEMA = "droboticflow.delivery.part-index/v1"
PART_INDEX_FILE = "part.json"
def load_operator_module() -> object:
"""Load the package operator by path, exactly like the operator runner does.
Only module-level helpers are used, so no runtime helper map is required.
"""
spec = importlib.util.spec_from_file_location("droboticflow_part_index_migration", OPERATOR_SOURCE)
if spec is None or spec.loader is None:
raise RuntimeError(f"cannot load package operator from {OPERATOR_SOURCE}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def sha256_file(path: pathlib.Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def load_json(path: pathlib.Path) -> dict:
try:
decoded = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise SystemExit(f"无法读取 {path}:{error}") from error
if not isinstance(decoded, dict):
raise SystemExit(f"{path} 不是 JSON 对象")
return decoded
def delivery_completion_manifest(delivery_root: pathlib.Path) -> dict:
manifest_path = delivery_root / "manifest.json"
if not manifest_path.is_file():
raise SystemExit(f"{delivery_root} 不是已完成的交付目录:缺少 manifest.json")
manifest = load_json(manifest_path)
kind = str(manifest.get("kind") or "")
if kind != "directory":
raise SystemExit(f"该交付的 manifest kind={kind!r},只有 directory 形态需要迁移")
return manifest
def delivered_calibrations(delivery_root: pathlib.Path, operator: object) -> list[dict]:
"""Reference the calibration bytes that are already delivered.
Builds the same artifact shape the operator's calibration helper expects,
then reads the checksum from the delivered file itself.
"""
directory = delivery_root / "calibration"
if not directory.is_dir():
raise SystemExit("交付目录缺少 calibration/,无法生成索引")
artifacts = []
for path in sorted(directory.iterdir()):
if path.is_file():
# The helper accepts either a logical or an absolute location; it
# records only the logical one in the index.
artifacts.append({"name": path.name, "path": str(path), "portId": "calibrations"})
if not artifacts:
raise SystemExit("交付目录的 calibration/ 为空,无法生成索引")
return operator._part_index_calibration_refs(artifacts)
def resolve_pose_policy(acceptance: dict) -> dict:
"""Recover the pose decision, or prove it was raw-human with no pose output."""
final = acceptance.get("finalAcceptance") if isinstance(acceptance.get("finalAcceptance"), dict) else {}
pose = final.get("poseDelivery") if isinstance(final.get("poseDelivery"), dict) else {}
if pose.get("headpose") in {"include", "exclude"} and pose.get("handpose") in {"include", "exclude"}:
return pose
if final.get("processingProfile") == "raw_human":
return {"headpose": "not_generated", "handpose": "not_generated"}
raise SystemExit("交付证据里没有可用的姿态交付决策,拒绝猜测 HeadPose/HandPose 策略")
def split_slices(revision_manifest: dict) -> tuple[list[dict], list[dict]]:
rows = revision_manifest.get("items")
if not isinstance(rows, list) or not rows:
raise SystemExit("物化修订清单没有切片条目")
kept: list[dict] = []
discarded: list[dict] = []
for row in rows:
if not isinstance(row, dict):
raise SystemExit("物化修订清单包含无效切片条目")
disposition = str(row.get("deliveryDisposition") or "")
if disposition == "keep":
kept.append(row)
elif disposition == "discard":
if not str((row.get("discardEvidence") or {}).get("reason") or "").strip():
raise SystemExit(f"舍弃切片 {row.get('sliceId')} 没有原因证据")
discarded.append(row)
else:
raise SystemExit(f"切片 {row.get('sliceId')} 的用途未知:{disposition!r}")
return kept, discarded
def build_index(delivery_root: pathlib.Path, revision_root: pathlib.Path) -> tuple[dict, bytes]:
operator = load_operator_module()
delivery_completion_manifest(delivery_root)
revision_manifest = load_json(revision_root / "revision-manifest.json")
schema = str(revision_manifest.get("schemaVersion") or "")
if schema not in {"droboticflow.slice-materialization/v2", "droboticflow.slice-materialization/v3"}:
raise SystemExit(f"不支持的物化修订清单:{schema!r}")
if schema == "droboticflow.slice-materialization/v2":
raise SystemExit("v2 修订清单没有逐片用途证据,无法判定保留与舍弃;请先确认该 part 的交付契约")
coverage = revision_manifest.get("coverage") if isinstance(revision_manifest.get("coverage"), dict) else {}
kept, discarded = split_slices(revision_manifest)
if not kept:
raise SystemExit("该 part 没有保留切片;迁移不产生交付物")
acceptance = load_json(delivery_root / "acceptance-record.json")
capture_manifest = load_json(delivery_root / "capture-manifest.json")
delivery_manifest = load_json(delivery_root / "packages-manifest.json")
capture_record = delivery_manifest.get("capture") or capture_manifest.get("capture") or {}
summary_payload = {
"decision": "approved",
"processingProfile": (acceptance.get("finalAcceptance") or {}).get("processingProfile"),
"qualityReviews": (load_json(delivery_root / "quality-review.json") or {}).get("reviews") or [],
"capture": capture_record,
}
lineage = {
"acceptedSliceDirectoryArtifactId": delivery_manifest.get("lineage", {}).get("acceptedSliceDirectoryArtifactId"),
"humanItemId": revision_manifest.get("humanItemId"),
"taskId": revision_manifest.get("taskId"),
"sourceRunId": revision_manifest.get("runId"),
"migratedFrom": "droboticflow.delivery.manifest/v6",
}
return operator._build_part_index(
revision_manifest,
coverage,
kept,
revision_root,
delivered_calibrations(delivery_root, operator),
resolve_pose_policy(acceptance),
summary_payload,
acceptance.get("finalAcceptance") or {},
capture_record,
lineage,
discarded,
)
def part_name_from_prefix(prefix: str) -> str:
"""The frozen delivery prefix already ends with the part directory."""
name = prefix.rstrip("/").rsplit("/", 1)[-1]
return name or "part0001"
def run_migration(delivery_root: pathlib.Path, revision_root: pathlib.Path, out_dir: pathlib.Path, part_name: str) -> dict:
"""Produce the candidate index outside the delivery directory.
Unchanged-to-delivered content is a no-op; different content is refused so a
migration can never silently replace what the customer already received.
"""
delivery_root = delivery_root.resolve()
revision_root = revision_root.resolve()
out_dir = out_dir.resolve()
if not delivery_root.is_dir():
raise SystemExit(f"--delivery-root 不存在:{delivery_root}")
if not revision_root.is_dir():
raise SystemExit(f"--revision-root 不存在:{revision_root}")
if out_dir == delivery_root or delivery_root in out_dir.parents:
raise SystemExit("--out 不能位于交付目录内;迁移只新增交付物,不改写本地副本")
payload, body = build_index(delivery_root, revision_root)
existing = delivery_root / PART_INDEX_FILE
if existing.is_file():
if existing.read_bytes() == body:
return {"status": "already-migrated", "path": str(existing), "sha256": sha256_file(existing)}
raise SystemExit("交付目录已有内容不同的 part.json,拒绝覆盖;请人工确认该 part 的交付状态")
out_dir.mkdir(parents=True, exist_ok=True)
target = out_dir / PART_INDEX_FILE
target.write_bytes(body)
return {
"status": "dry-run",
"schemaVersion": PART_INDEX_SCHEMA,
"part": part_name,
"written": str(target),
"sha256": sha256_file(target),
"bytes": len(body),
"keptSliceCount": len(payload.get("slices") or []),
"discardedSliceCount": len(payload.get("discardedSlices") or []),
}
def main() -> int:
parser = argparse.ArgumentParser(description="增量迁移已交付的逐片目录为 part 级总 JSON 索引(dry run)。")
parser.add_argument("--delivery-root", required=True, help="已交付 part 目录的本地副本")
parser.add_argument("--revision-root", required=True, help="对应的物化修订目录(含 revision-manifest.json 与 slices/)")
parser.add_argument("--out", required=True, help="写出候选 part.json 的目录(不会写入交付目录)")
parser.add_argument("--prefix", help="该 part 的冻结交付前缀,仅用于推断目录名")
args = parser.parse_args()
report = run_migration(
pathlib.Path(args.delivery_root),
pathlib.Path(args.revision_root),
pathlib.Path(args.out),
part_name_from_prefix(args.prefix or args.delivery_root),
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if report["status"] == "dry-run":
print("未上传:发布必须经过 API/Worker 的交付通道。")
return 0
if __name__ == "__main__":
sys.exit(main())