244 lines
11 KiB
Python
244 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""把一个已交付的逐片目录形态交付,增量迁移为 part 级总 JSON 索引。
|
||
|
||
只新增 `part.json`,不删除 `clips/`、`calibration/` 或任何既有清单。
|
||
|
||
用法:
|
||
|
||
python3 scripts/part_index_migration.py --delivery-root /path/to/part0001 \\
|
||
--revision-root /path/to/human-materializations/<item>/<task>/revision-000001 \\
|
||
--out /tmp/part-index-dry-run
|
||
|
||
设计边界:
|
||
|
||
- 本脚本**不写对象存储**。发布必须经过 API/Worker 的 `prepareDeliveryTarget`
|
||
通道,才能保留 `.delivery-owner.json` 归属校验、条件创建与「绝不覆盖他人交付」
|
||
语义;直连 BOS 会绕过它们。本脚本只产出候选产物与迁移计划。
|
||
- 复用打包算子自身的 part-index 构建逻辑(同一份源码),不复制第二套实现,
|
||
因此迁移产物与在线打包逐字节同构。
|
||
- 交付目录里的 `annotation.json` 与 `synchronized-slice.json` 是打包时写出的
|
||
对外版本;索引只从它们读取已对外的事实,舍弃原因取自 `revision-manifest.json`
|
||
与 `synchronized-slice.json` 的 `discardEvidence`。
|
||
- 交付目录已有内容不同的 `part.json` 时直接拒绝,不覆盖、不删除任何对象。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import importlib.util
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
|
||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||
OPERATOR_SOURCE = ROOT / "backend" / "operators" / "builtin" / "package" / "operator.py"
|
||
PART_INDEX_SCHEMA = "droboticflow.delivery.part-index/v1"
|
||
PART_INDEX_FILE = "part.json"
|
||
|
||
|
||
def load_operator_module() -> object:
|
||
"""Load the package operator by path, exactly like the operator runner does.
|
||
|
||
Only module-level helpers are used, so no runtime helper map is required.
|
||
"""
|
||
spec = importlib.util.spec_from_file_location("droboticflow_part_index_migration", OPERATOR_SOURCE)
|
||
if spec is None or spec.loader is None:
|
||
raise RuntimeError(f"cannot load package operator from {OPERATOR_SOURCE}")
|
||
module = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
def sha256_file(path: pathlib.Path) -> str:
|
||
digest = hashlib.sha256()
|
||
with path.open("rb") as handle:
|
||
for block in iter(lambda: handle.read(1 << 20), b""):
|
||
digest.update(block)
|
||
return digest.hexdigest()
|
||
|
||
|
||
def load_json(path: pathlib.Path) -> dict:
|
||
try:
|
||
decoded = json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError) as error:
|
||
raise SystemExit(f"无法读取 {path}:{error}") from error
|
||
if not isinstance(decoded, dict):
|
||
raise SystemExit(f"{path} 不是 JSON 对象")
|
||
return decoded
|
||
|
||
|
||
def delivery_completion_manifest(delivery_root: pathlib.Path) -> dict:
|
||
manifest_path = delivery_root / "manifest.json"
|
||
if not manifest_path.is_file():
|
||
raise SystemExit(f"{delivery_root} 不是已完成的交付目录:缺少 manifest.json")
|
||
manifest = load_json(manifest_path)
|
||
kind = str(manifest.get("kind") or "")
|
||
if kind != "directory":
|
||
raise SystemExit(f"该交付的 manifest kind={kind!r},只有 directory 形态需要迁移")
|
||
return manifest
|
||
|
||
|
||
def delivered_calibrations(delivery_root: pathlib.Path, operator: object) -> list[dict]:
|
||
"""Reference the calibration bytes that are already delivered.
|
||
|
||
Builds the same artifact shape the operator's calibration helper expects,
|
||
then reads the checksum from the delivered file itself.
|
||
"""
|
||
directory = delivery_root / "calibration"
|
||
if not directory.is_dir():
|
||
raise SystemExit("交付目录缺少 calibration/,无法生成索引")
|
||
artifacts = []
|
||
for path in sorted(directory.iterdir()):
|
||
if path.is_file():
|
||
# The helper accepts either a logical or an absolute location; it
|
||
# records only the logical one in the index.
|
||
artifacts.append({"name": path.name, "path": str(path), "portId": "calibrations"})
|
||
if not artifacts:
|
||
raise SystemExit("交付目录的 calibration/ 为空,无法生成索引")
|
||
return operator._part_index_calibration_refs(artifacts)
|
||
|
||
|
||
def resolve_pose_policy(acceptance: dict) -> dict:
|
||
"""Recover the pose decision, or prove it was raw-human with no pose output."""
|
||
final = acceptance.get("finalAcceptance") if isinstance(acceptance.get("finalAcceptance"), dict) else {}
|
||
pose = final.get("poseDelivery") if isinstance(final.get("poseDelivery"), dict) else {}
|
||
if pose.get("headpose") in {"include", "exclude"} and pose.get("handpose") in {"include", "exclude"}:
|
||
return pose
|
||
if final.get("processingProfile") == "raw_human":
|
||
return {"headpose": "not_generated", "handpose": "not_generated"}
|
||
raise SystemExit("交付证据里没有可用的姿态交付决策,拒绝猜测 HeadPose/HandPose 策略")
|
||
|
||
|
||
def split_slices(revision_manifest: dict) -> tuple[list[dict], list[dict]]:
|
||
rows = revision_manifest.get("items")
|
||
if not isinstance(rows, list) or not rows:
|
||
raise SystemExit("物化修订清单没有切片条目")
|
||
kept: list[dict] = []
|
||
discarded: list[dict] = []
|
||
for row in rows:
|
||
if not isinstance(row, dict):
|
||
raise SystemExit("物化修订清单包含无效切片条目")
|
||
disposition = str(row.get("deliveryDisposition") or "")
|
||
if disposition == "keep":
|
||
kept.append(row)
|
||
elif disposition == "discard":
|
||
if not str((row.get("discardEvidence") or {}).get("reason") or "").strip():
|
||
raise SystemExit(f"舍弃切片 {row.get('sliceId')} 没有原因证据")
|
||
discarded.append(row)
|
||
else:
|
||
raise SystemExit(f"切片 {row.get('sliceId')} 的用途未知:{disposition!r}")
|
||
return kept, discarded
|
||
|
||
|
||
def build_index(delivery_root: pathlib.Path, revision_root: pathlib.Path) -> tuple[dict, bytes]:
|
||
operator = load_operator_module()
|
||
delivery_completion_manifest(delivery_root)
|
||
revision_manifest = load_json(revision_root / "revision-manifest.json")
|
||
schema = str(revision_manifest.get("schemaVersion") or "")
|
||
if schema not in {"droboticflow.slice-materialization/v2", "droboticflow.slice-materialization/v3"}:
|
||
raise SystemExit(f"不支持的物化修订清单:{schema!r}")
|
||
if schema == "droboticflow.slice-materialization/v2":
|
||
raise SystemExit("v2 修订清单没有逐片用途证据,无法判定保留与舍弃;请先确认该 part 的交付契约")
|
||
coverage = revision_manifest.get("coverage") if isinstance(revision_manifest.get("coverage"), dict) else {}
|
||
kept, discarded = split_slices(revision_manifest)
|
||
if not kept:
|
||
raise SystemExit("该 part 没有保留切片;迁移不产生交付物")
|
||
acceptance = load_json(delivery_root / "acceptance-record.json")
|
||
capture_manifest = load_json(delivery_root / "capture-manifest.json")
|
||
delivery_manifest = load_json(delivery_root / "packages-manifest.json")
|
||
capture_record = delivery_manifest.get("capture") or capture_manifest.get("capture") or {}
|
||
summary_payload = {
|
||
"decision": "approved",
|
||
"processingProfile": (acceptance.get("finalAcceptance") or {}).get("processingProfile"),
|
||
"qualityReviews": (load_json(delivery_root / "quality-review.json") or {}).get("reviews") or [],
|
||
"capture": capture_record,
|
||
}
|
||
lineage = {
|
||
"acceptedSliceDirectoryArtifactId": delivery_manifest.get("lineage", {}).get("acceptedSliceDirectoryArtifactId"),
|
||
"humanItemId": revision_manifest.get("humanItemId"),
|
||
"taskId": revision_manifest.get("taskId"),
|
||
"sourceRunId": revision_manifest.get("runId"),
|
||
"migratedFrom": "droboticflow.delivery.manifest/v6",
|
||
}
|
||
return operator._build_part_index(
|
||
revision_manifest,
|
||
coverage,
|
||
kept,
|
||
revision_root,
|
||
delivered_calibrations(delivery_root, operator),
|
||
resolve_pose_policy(acceptance),
|
||
summary_payload,
|
||
acceptance.get("finalAcceptance") or {},
|
||
capture_record,
|
||
lineage,
|
||
discarded,
|
||
)
|
||
|
||
|
||
def part_name_from_prefix(prefix: str) -> str:
|
||
"""The frozen delivery prefix already ends with the part directory."""
|
||
name = prefix.rstrip("/").rsplit("/", 1)[-1]
|
||
return name or "part0001"
|
||
|
||
|
||
def run_migration(delivery_root: pathlib.Path, revision_root: pathlib.Path, out_dir: pathlib.Path, part_name: str) -> dict:
|
||
"""Produce the candidate index outside the delivery directory.
|
||
|
||
Unchanged-to-delivered content is a no-op; different content is refused so a
|
||
migration can never silently replace what the customer already received.
|
||
"""
|
||
delivery_root = delivery_root.resolve()
|
||
revision_root = revision_root.resolve()
|
||
out_dir = out_dir.resolve()
|
||
if not delivery_root.is_dir():
|
||
raise SystemExit(f"--delivery-root 不存在:{delivery_root}")
|
||
if not revision_root.is_dir():
|
||
raise SystemExit(f"--revision-root 不存在:{revision_root}")
|
||
if out_dir == delivery_root or delivery_root in out_dir.parents:
|
||
raise SystemExit("--out 不能位于交付目录内;迁移只新增交付物,不改写本地副本")
|
||
|
||
payload, body = build_index(delivery_root, revision_root)
|
||
existing = delivery_root / PART_INDEX_FILE
|
||
if existing.is_file():
|
||
if existing.read_bytes() == body:
|
||
return {"status": "already-migrated", "path": str(existing), "sha256": sha256_file(existing)}
|
||
raise SystemExit("交付目录已有内容不同的 part.json,拒绝覆盖;请人工确认该 part 的交付状态")
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
target = out_dir / PART_INDEX_FILE
|
||
target.write_bytes(body)
|
||
return {
|
||
"status": "dry-run",
|
||
"schemaVersion": PART_INDEX_SCHEMA,
|
||
"part": part_name,
|
||
"written": str(target),
|
||
"sha256": sha256_file(target),
|
||
"bytes": len(body),
|
||
"keptSliceCount": len(payload.get("slices") or []),
|
||
"discardedSliceCount": len(payload.get("discardedSlices") or []),
|
||
}
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="增量迁移已交付的逐片目录为 part 级总 JSON 索引(dry run)。")
|
||
parser.add_argument("--delivery-root", required=True, help="已交付 part 目录的本地副本")
|
||
parser.add_argument("--revision-root", required=True, help="对应的物化修订目录(含 revision-manifest.json 与 slices/)")
|
||
parser.add_argument("--out", required=True, help="写出候选 part.json 的目录(不会写入交付目录)")
|
||
parser.add_argument("--prefix", help="该 part 的冻结交付前缀,仅用于推断目录名")
|
||
args = parser.parse_args()
|
||
|
||
report = run_migration(
|
||
pathlib.Path(args.delivery_root),
|
||
pathlib.Path(args.revision_root),
|
||
pathlib.Path(args.out),
|
||
part_name_from_prefix(args.prefix or args.delivery_root),
|
||
)
|
||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||
if report["status"] == "dry-run":
|
||
print("未上传:发布必须经过 API/Worker 的交付通道。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|