forked from longtao.wu/droboticflow
202 lines
11 KiB
Python
202 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Resume a local-only V4 conversion of inventoried, frozen delivery revisions.
|
|
|
|
Input: inventory.json and packages/<id>/{old/,pinned-revision/,pinned-provenance.json}.
|
|
This produces customer JSON previews, never a complete audit or publishable run.
|
|
No network, current dictionaries, video downloads, or production writes are used.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
|
|
from part_index_migration import load_operator_module, resolve_pose_policy, split_slices
|
|
|
|
|
|
def load(path):
|
|
return json.loads(path.read_text(encoding="utf8"))
|
|
|
|
|
|
def sha(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def convert(package, directory, output):
|
|
if package["status"] not in {"accepted", "no_deliverables"}:
|
|
raise ValueError("no published accepted manifest; frozen revision recovery required")
|
|
revision = directory / "pinned-revision"
|
|
manifest_bytes = (revision / "revision-manifest.json").read_bytes()
|
|
manifest = json.loads(manifest_bytes)
|
|
provenance = load(directory / "pinned-provenance.json")
|
|
delivery = load(directory / "old/packages-manifest.json")
|
|
lineage = delivery["lineage"]
|
|
if (provenance["sha256"] != sha(manifest_bytes)
|
|
or provenance["artifactId"] != lineage["acceptedSliceDirectoryArtifactId"]
|
|
or provenance["revision"] != manifest["revision"]):
|
|
raise ValueError("frozen revision provenance mismatch")
|
|
# Verify every input JSON against its accepted manifest before the builder reads it.
|
|
before = {}
|
|
for row in manifest["items"]:
|
|
relative = Path(row["directory"])
|
|
if relative.is_absolute() or ".." in relative.parts:
|
|
raise ValueError("unsafe accepted slice directory")
|
|
verified = set()
|
|
files = row.get("files", [])
|
|
for entry in files.values() if isinstance(files, dict) else files:
|
|
if entry["name"] not in {"annotation.json", "synchronized-slice.json"}:
|
|
continue
|
|
path = revision / relative / entry["name"]
|
|
if not path.resolve().is_relative_to(revision.resolve()):
|
|
raise ValueError("accepted file escapes revision directory")
|
|
raw = path.read_bytes()
|
|
if not entry.get("sha256") or sha(raw) != entry["sha256"]:
|
|
raise ValueError("frozen annotation checksum mismatch")
|
|
before[path] = sha(raw)
|
|
verified.add(entry["name"])
|
|
required = {"synchronized-slice.json"}
|
|
if row["deliveryDisposition"] == "keep":
|
|
required.add("annotation.json")
|
|
if not required <= verified:
|
|
raise ValueError("missing frozen JSON checksums")
|
|
source_basis = "accepted_revision_manifest"
|
|
recovery_path = directory / "source-selection-recovery.json"
|
|
if not manifest.get("originalSource") and recovery_path.exists():
|
|
recovery = load(recovery_path)
|
|
raw_input = (directory / "source-selection-input.json").read_bytes()
|
|
evidence = json.loads(raw_input)
|
|
run, capture = evidence["run"], evidence["capture"]
|
|
frozen = delivery["capture"]
|
|
refs = [v["ref"] for v in run["inputs"].values()]
|
|
source = recovery["originalSource"]
|
|
if (recovery.get("basis") != "published_capture_snapshot_and_exact_persisted_run_captureUnit_objectKeys"
|
|
or recovery.get("inputSha256") != sha(raw_input)
|
|
or recovery.get("manifestSha256") != sha(manifest_bytes)
|
|
or run["id"] != lineage["sourceRunId"] or run["id"] != manifest["runId"]
|
|
or run["organizationId"] != package["organizationId"]
|
|
or capture["runId"] != run["id"] or capture["organizationId"] != run["organizationId"]
|
|
or capture["id"] != frozen["captureRecordId"]
|
|
or capture["captureId"] != frozen["captureId"] or capture["prefix"] != frozen["prefix"]
|
|
or len(refs) != 1):
|
|
raise ValueError("historical source recovery provenance mismatch")
|
|
ref = refs[0]
|
|
if (ref["provider"] != "BOS" or ref["kind"] != "directory"
|
|
or ref["captureUnit"] != frozen["captureId"]
|
|
or not frozen["prefix"].startswith(ref["prefix"] + "/")
|
|
or source["bucket"] != ref["bucket"] or source["prefix"] != ref["prefix"]
|
|
or source["partPrefix"] != frozen["prefix"] or source["captureUnit"] != frozen["captureId"]
|
|
or source["objectKeys"] != sorted(ref["objectKeys"])
|
|
or source.get("snapshotFingerprint", "") != ref.get("versionId", "")):
|
|
raise ValueError("historical source selection mismatch")
|
|
manifest = copy.deepcopy(manifest)
|
|
manifest["originalSource"] = source
|
|
source_basis = "published_capture_snapshot_and_exact_persisted_run_captureUnit_objectKeys"
|
|
acceptance = load(directory / "old/acceptance-record.json")
|
|
kept, discarded = split_slices(manifest)
|
|
if package["status"] == "no_deliverables":
|
|
proof = load(directory / "empty-acceptance-proof.json")
|
|
accepted = proof.get("acceptedMaterialization", {})
|
|
if (kept or not discarded or acceptance["finalAcceptance"].get("confirmed") is not True
|
|
or proof.get("packageId") != package["id"] or proof.get("confirmed") is not True
|
|
or proof.get("summaryArtifactId") not in package.get("artifactIds", [])
|
|
or accepted.get("artifactId") != lineage["acceptedSliceDirectoryArtifactId"]
|
|
or accepted.get("revision") != manifest["revision"]
|
|
or accepted.get("manifestSha256") != sha(manifest_bytes)):
|
|
raise ValueError("unconfirmed or mismatched all-discard acceptance evidence")
|
|
operator = load_operator_module()
|
|
final = acceptance["finalAcceptance"]
|
|
# Deliberately incomplete audit input: never export it or claim formal readiness.
|
|
preview_context = {"trace": [], "finalAcceptance": final}
|
|
bodies = []
|
|
with tempfile.TemporaryDirectory(prefix="v4-preview-") as temp:
|
|
for attempt in range(2):
|
|
staging = Path(temp) / str(attempt)
|
|
staging.mkdir()
|
|
payload, _ = operator._build_part_index(
|
|
manifest, manifest["coverage"], kept, revision,
|
|
copy.deepcopy(delivery["calibration"]), resolve_pose_policy(acceptance),
|
|
{}, final, copy.deepcopy(delivery["capture"]), lineage, discarded,
|
|
classification_v4=True,
|
|
)
|
|
def internal(*args, **kwargs):
|
|
return {"portId": "deliveryAudit", "previewOnly": True}
|
|
def virtual(run, node, name, path, kind, size, data=None, output=None):
|
|
return {"path": str(path), "portId": output["id"]}
|
|
artifacts = operator._emit_part_index_directory(
|
|
"local-preview", "package", staging,
|
|
{"outputs": [{"id": "deliveryAudit"}, {"id": "deliveryDirectory"}]},
|
|
{"write_json_artifact": internal, "build_virtual_artifact": virtual},
|
|
payload, preview_context, manifest, revision,
|
|
)
|
|
folder = Path(artifacts[0]["path"])
|
|
if {p.name for p in folder.iterdir()} != {"part.json"}:
|
|
raise ValueError("customer directory must contain only part.json")
|
|
bodies.append((folder / "part.json").read_bytes())
|
|
if bodies[0] != bodies[1]:
|
|
raise ValueError("non-deterministic conversion")
|
|
for path, digest in before.items():
|
|
if sha(path.read_bytes()) != digest:
|
|
raise ValueError("source changed during conversion")
|
|
if (revision / "revision-manifest.json").read_bytes() != manifest_bytes:
|
|
raise ValueError("source manifest changed during conversion")
|
|
calibration_proof = directory / "calibration-provenance.json"
|
|
calibration_basis = "published_manifest_checksums"
|
|
if calibration_proof.exists():
|
|
evidence = load(calibration_proof)
|
|
if not all(entry.get("historicalStoredSHAAvailable") is True for entry in evidence):
|
|
calibration_basis = "original_BOS_selection_read_at_conversion; historical_checksum_unavailable"
|
|
document = json.loads(bodies[0])
|
|
target = output / package["id"] / "part.json"
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
if target.exists() and target.read_bytes() != bodies[0]:
|
|
raise ValueError("refusing to replace an existing different preview")
|
|
target.write_bytes(bodies[0])
|
|
return {"packageId": package["id"], "status": "converted-preview", "file": str(target),
|
|
"captureId": document["capture"]["captureId"], "partId": document["partId"],
|
|
"revision": document["revision"], "sha256": sha(bodies[0]), "bytes": len(bodies[0]),
|
|
"keptSlices": len(kept), "discardedSlices": len(discarded),
|
|
"manifestSha256": sha(manifest_bytes), "deterministic": True,
|
|
"calibrationEvidence": calibration_basis,
|
|
"originalSourceEvidence": source_basis,
|
|
"formalPublicationReady": False, "completeInternalAuditValidated": False,
|
|
"onlinePublished": False}
|
|
|
|
|
|
def run(root, output):
|
|
root, output = root.resolve(), output.resolve()
|
|
if output == root or root in output.parents:
|
|
raise ValueError("output must be outside the evidence directory")
|
|
inventory = load(root / "inventory.json")
|
|
ids = [p["id"] for p in inventory]
|
|
if len(ids) != len(set(ids)):
|
|
raise ValueError("duplicate inventory package IDs")
|
|
results = []
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
for package in inventory:
|
|
# Do not use an untrusted ID as a filesystem path.
|
|
if Path(package["id"]).name != package["id"] or package["id"] in {".", ".."}:
|
|
raise ValueError("unsafe package ID")
|
|
try:
|
|
result = convert(package, root / "packages" / package["id"], output)
|
|
except (ValueError, RuntimeError, KeyError, OSError, SystemExit) as error:
|
|
result = {"packageId": package["id"], "status": "blocked", "reason": str(error)}
|
|
results.append(result)
|
|
report = {"total": len(inventory), "processed": len(results),
|
|
"converted": sum(r["status"] == "converted-preview" for r in results),
|
|
"productionWrites": False, "formalPublicationReady": False, "packages": results}
|
|
(output / "conversion-report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
return report
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--evidence", type=Path, required=True)
|
|
parser.add_argument("--out", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
report = run(args.evidence, args.out)
|
|
print(json.dumps({k: v for k, v in report.items() if k != "packages"}, ensure_ascii=False))
|
|
raise SystemExit(0 if report["converted"] == report["total"] else 2)
|