forked from longtao.wu/droboticflow
180 lines
9.5 KiB
Python
180 lines
9.5 KiB
Python
"""Batch conversion must preserve revisions, resume safely and report every failure."""
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tests.backend.test_delivery_v4_package import prepare
|
|
|
|
|
|
def converter(monkeypatch):
|
|
root = Path(__file__).resolve().parents[2]
|
|
monkeypatch.syspath_prepend(str(root / "scripts"))
|
|
spec = importlib.util.spec_from_file_location("preview", root / "scripts/delivery_v4_preview.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def evidence(tmp_path):
|
|
root = tmp_path / "evidence"
|
|
directory = root / "packages/package-accepted"
|
|
directory.mkdir(parents=True)
|
|
inputs = prepare(directory, [{"description": "拿取"}, {"description": "", "isInvalid": True}])
|
|
revision = Path(inputs[0]["path"])
|
|
revision.rename(directory / "pinned-revision")
|
|
revision = directory / "pinned-revision"
|
|
manifest = json.loads((revision / "revision-manifest.json").read_text())
|
|
for row in manifest["items"]:
|
|
row["files"] = [{"name": name, "sha256": hashlib.sha256((revision / row["directory"] / name).read_bytes()).hexdigest()}
|
|
for name in ["annotation.json", "synchronized-slice.json"]]
|
|
raw = json.dumps(manifest).encode()
|
|
(revision / "revision-manifest.json").write_bytes(raw)
|
|
(directory / "pinned-provenance.json").write_text(json.dumps({
|
|
"sha256": hashlib.sha256(raw).hexdigest(), "revision": manifest["revision"], "artifactId": "frozen-artifact"}))
|
|
old = directory / "old"
|
|
old.mkdir()
|
|
(old / "packages-manifest.json").write_text(json.dumps({
|
|
"lineage": {"acceptedSliceDirectoryArtifactId": "frozen-artifact"},
|
|
"capture": inputs[1]["data"]["capture"], "calibration": []}))
|
|
(old / "acceptance-record.json").write_text(json.dumps({"finalAcceptance": inputs[1]["data"]["finalAcceptance"]}))
|
|
inventory = [{"id": "package-accepted", "status": "accepted"}, {"id": "package-empty", "status": "no_deliverables"}]
|
|
(root / "inventory.json").write_text(json.dumps(inventory))
|
|
return root, directory
|
|
|
|
|
|
def test_batch_resumes_exactly_and_keeps_all_blocked_rows(tmp_path, monkeypatch):
|
|
module = converter(monkeypatch)
|
|
root, directory = evidence(tmp_path)
|
|
output = tmp_path / "output"
|
|
before = {p: p.read_bytes() for p in root.rglob("*.json")}
|
|
report = module.run(root, output)
|
|
assert report["processed"] == report["total"] == 2
|
|
assert report["converted"] == 1
|
|
assert report["packages"][1]["status"] == "blocked"
|
|
assert report["productionWrites"] is report["formalPublicationReady"] is False
|
|
customer = output / "package-accepted/part.json"
|
|
doc = json.loads(customer.read_text())
|
|
assert doc["schemaVersion"].endswith("/v4")
|
|
assert not {"lineage", "acceptance", "reviewHistory"} & doc.keys()
|
|
assert not list(output.rglob("review-history.json"))
|
|
assert "PRIVATE_REWORK" not in customer.read_text()
|
|
assert "PRIVATE_BASELINE" not in customer.read_text()
|
|
assert module.run(root, output) == report
|
|
assert all(p.read_bytes() == body for p, body in before.items())
|
|
customer.write_text("other customer bytes")
|
|
blocked = module.run(root, output)
|
|
assert blocked["converted"] == 0
|
|
assert "refusing to replace" in blocked["packages"][0]["reason"]
|
|
assert customer.read_text() == "other customer bytes"
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["annotation", "provenance", "source", "classification", "checksums"])
|
|
def test_corrupt_or_missing_frozen_evidence_blocks_without_guessing(tmp_path, monkeypatch, mutation):
|
|
module = converter(monkeypatch)
|
|
root, directory = evidence(tmp_path)
|
|
revision = directory / "pinned-revision"
|
|
path = revision / "revision-manifest.json"
|
|
manifest = json.loads(path.read_text())
|
|
if mutation == "annotation":
|
|
(revision / manifest["items"][0]["directory"] / "annotation.json").write_text("{}")
|
|
elif mutation == "provenance":
|
|
(directory / "pinned-provenance.json").write_text('{}')
|
|
else:
|
|
if mutation == "source":
|
|
manifest.pop("originalSource")
|
|
elif mutation == "classification":
|
|
manifest["items"][0].pop("dictionary", None)
|
|
sync = revision / manifest["items"][0]["directory"] / "synchronized-slice.json"
|
|
value = json.loads(sync.read_text()); value.pop("dictionary", None); sync.write_text(json.dumps(value))
|
|
manifest["items"][0]["files"][1]["sha256"] = hashlib.sha256(sync.read_bytes()).hexdigest()
|
|
else:
|
|
manifest["items"][0].pop("files")
|
|
path.write_text(json.dumps(manifest))
|
|
provenance = json.loads((directory / "pinned-provenance.json").read_text())
|
|
provenance["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
(directory / "pinned-provenance.json").write_text(json.dumps(provenance))
|
|
report = module.run(root, tmp_path / "output")
|
|
assert report["processed"] == 2
|
|
assert report["converted"] == 0
|
|
assert not list((tmp_path / "output").rglob("part.json"))
|
|
|
|
|
|
def test_all_discard_preview_requires_matching_fixed_acceptance_proof(tmp_path, monkeypatch):
|
|
import copy
|
|
module = converter(monkeypatch)
|
|
root, directory = evidence(tmp_path)
|
|
revision = directory / "pinned-revision"
|
|
path = revision / "revision-manifest.json"
|
|
manifest = json.loads(path.read_text())
|
|
row = manifest["items"][0]
|
|
row["deliveryDisposition"] = "discard"
|
|
row["discardEvidence"] = copy.deepcopy(manifest["items"][1]["discardEvidence"])
|
|
sync = revision / row["directory"] / "synchronized-slice.json"
|
|
doc = json.loads(sync.read_text())
|
|
doc.update(deliveryDisposition="discard", discardEvidence=row["discardEvidence"])
|
|
sync.write_text(json.dumps(doc))
|
|
row["files"][1]["sha256"] = hashlib.sha256(sync.read_bytes()).hexdigest()
|
|
path.write_text(json.dumps(manifest))
|
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
provenance = json.loads((directory / "pinned-provenance.json").read_text())
|
|
provenance["sha256"] = digest
|
|
(directory / "pinned-provenance.json").write_text(json.dumps(provenance))
|
|
package = {"id": "package-accepted", "status": "no_deliverables", "artifactIds": ["frozen-summary"]}
|
|
(root / "inventory.json").write_text(json.dumps([package]))
|
|
assert module.run(root, tmp_path / "out")["converted"] == 0
|
|
proof = {"packageId": package["id"], "confirmed": True, "summaryArtifactId": "frozen-summary",
|
|
"acceptedMaterialization": {"artifactId": "frozen-artifact", "revision": manifest["revision"], "manifestSha256": digest}}
|
|
(directory / "empty-acceptance-proof.json").write_text(json.dumps(proof))
|
|
report = module.run(root, tmp_path / "out")
|
|
assert report["converted"] == 1
|
|
doc = json.loads(Path(report["packages"][0]["file"]).read_text())
|
|
assert doc["slices"] == [] and len(doc["discardedSlices"]) == 2
|
|
assert all(not item["segments"] for item in doc["discardedSlices"])
|
|
proof["acceptedMaterialization"]["manifestSha256"] = "wrong-revision"
|
|
(directory / "empty-acceptance-proof.json").write_text(json.dumps(proof))
|
|
assert module.run(root, tmp_path / "other")["converted"] == 0
|
|
|
|
|
|
def test_recovers_source_only_from_matching_published_capture_and_fixed_run(tmp_path, monkeypatch):
|
|
module = converter(monkeypatch)
|
|
root, directory = evidence(tmp_path)
|
|
manifest_path = directory / "pinned-revision/revision-manifest.json"
|
|
manifest = json.loads(manifest_path.read_text())
|
|
original = manifest.pop("originalSource")
|
|
original["objectKeys"] = sorted(original["objectKeys"])
|
|
manifest_path.write_text(json.dumps(manifest))
|
|
digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
|
|
provenance = json.loads((directory / "pinned-provenance.json").read_text())
|
|
provenance["sha256"] = digest
|
|
(directory / "pinned-provenance.json").write_text(json.dumps(provenance))
|
|
delivery_path = directory / "old/packages-manifest.json"
|
|
delivery = json.loads(delivery_path.read_text())
|
|
delivery["lineage"]["sourceRunId"] = manifest["runId"]
|
|
delivery_path.write_text(json.dumps(delivery))
|
|
package = {"id": "package-accepted", "status": "accepted", "organizationId": "org"}
|
|
(root / "inventory.json").write_text(json.dumps([package]))
|
|
frozen = delivery["capture"]
|
|
source = {"run": {"id": manifest["runId"], "organizationId": "org", "inputs": {"source": {"ref": {
|
|
"provider": "BOS", "kind": "directory", "bucket": original["bucket"], "prefix": original["prefix"],
|
|
"objectKeys": original["objectKeys"], "captureUnit": original["captureUnit"], "versionId": original["snapshotFingerprint"]}}}},
|
|
"capture": {"id": frozen["captureRecordId"], "organizationId": "org", "runId": manifest["runId"],
|
|
"captureId": frozen["captureId"], "prefix": frozen["prefix"]}}
|
|
recovery = {"basis": "published_capture_snapshot_and_exact_persisted_run_captureUnit_objectKeys",
|
|
"manifestSha256": digest, "originalSource": original}
|
|
def save():
|
|
raw = json.dumps(source).encode()
|
|
(directory / "source-selection-input.json").write_bytes(raw)
|
|
recovery["inputSha256"] = hashlib.sha256(raw).hexdigest()
|
|
(directory / "source-selection-recovery.json").write_text(json.dumps(recovery))
|
|
save()
|
|
report = module.run(root, tmp_path / "out")
|
|
assert report["converted"] == 1
|
|
assert report["packages"][0]["originalSourceEvidence"] == recovery["basis"]
|
|
assert "originalSource" not in json.loads(manifest_path.read_text())
|
|
source["run"]["organizationId"] = "different-tenant"
|
|
save()
|
|
assert module.run(root, tmp_path / "foreign")["converted"] == 0
|