droboticflow/scripts/fanyu_v58_capture_migration.py
2026-08-31 21:25:47 +08:00

175 lines
6.9 KiB
Python

#!/usr/bin/env python3
"""Audit/apply the Fanyu v5.8 Capture-policy migration.
Credentials are read only from FANYU_MONGO_URI and never emitted. Dry-run is
the default; --apply performs idempotent updates after the caller has created a
Mongo backup.
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import sys
from typing import Any
COLLECTIONS_WITH_LEGACY_ASSIGNMENT = (
"annotationTasks",
"annotationWorklogEvents",
"humanItems",
"deliveryPackages",
)
def now_iso() -> str:
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
def redact_mongo_uri(value: str) -> str:
if "@" not in value:
return "mongodb://***"
scheme, rest = value.split("://", 1) if "://" in value else ("mongodb", value)
return f"{scheme}://***@{rest.split('@', 1)[1]}"
def build_report(db: Any) -> dict[str, Any]:
legacy = {
name: {
"workgroupId": db[name].count_documents({"workgroupId": {"$exists": True}}),
"vendorId": db[name].count_documents({"vendorId": {"$exists": True}}),
"missingSupplier": db[name].count_documents({"$or": [{"supplierId": {"$exists": False}}, {"supplierId": ""}]}),
}
for name in COLLECTIONS_WITH_LEGACY_ASSIGNMENT
}
return {
"generatedAt": now_iso(),
"captureWorkflows": db.workflows.count_documents({"captureAutomation.enabled": True}),
"runtimePolicies": db.workflowRuntimePolicies.count_documents({}),
"captureRecords": db.captureRecords.count_documents({}),
"captureSamplingDecisions": db.captureRecords.count_documents({"samplingDecision": {"$exists": True}}),
"acceptanceBatches": db.humanBatches.count_documents({"sourceBatchId": {"$nin": ["", None]}}),
"annotationWorkgroups": db.annotationWorkgroups.count_documents({}),
"annotationMemberships": db.annotationMemberships.count_documents({}),
"legacyReferences": legacy,
}
def apply_migration(db: Any) -> dict[str, int]:
now = now_iso()
counters = {"policiesCreated": 0, "samplingBackfilled": 0, "legacyFieldsUnset": 0}
for workflow in db.workflows.find({"captureAutomation.enabled": True}, {"id": 1, "organizationId": 1, "projectId": 1}):
workflow_id = str(workflow.get("id") or "").strip()
if not workflow_id:
continue
result = db.workflowRuntimePolicies.update_one(
{"workflowId": workflow_id},
{"$setOnInsert": {
"id": f"workflow-runtime-policy-{workflow_id}",
"organizationId": workflow.get("organizationId") or "org-default",
"projectId": workflow.get("projectId") or "",
"workflowId": workflow_id,
"revision": 1,
"maxConcurrentCaptureRuns": 2,
"acceptanceSampleProbability": 0.5,
"updatedByUserId": "system-migration",
"updatedAt": now,
"history": [{
"revision": 1,
"maxConcurrentCaptureRuns": 2,
"acceptanceSampleProbability": 0.5,
"updatedByUserId": "system-migration",
"updatedAt": now,
}],
}},
upsert=True,
)
counters["policiesCreated"] += int(result.upserted_id is not None)
acceptance_batches = db.humanBatches.find({"sourceBatchId": {"$nin": ["", None]}}, {"sourceBatchId": 1})
for acceptance_batch in acceptance_batches:
source_id = str(acceptance_batch.get("sourceBatchId") or "").strip()
source = db.humanBatches.find_one({"id": source_id}, {"runId": 1})
if not source:
continue
capture = db.captureRecords.find_one({"runId": source.get("runId")}, {"id": 1, "samplingDecision": 1})
if not capture or capture.get("samplingDecision"):
continue
result = db.captureRecords.update_one(
{"id": capture["id"], "samplingDecision": {"$exists": False}},
{"$set": {
"samplingDecision": {
"decision": "selected",
"probability": 1.0,
"policyRevision": 0,
"randomValue": 0.0,
"decidedAt": now,
"decidedByUserId": "system-migration",
"sourceHumanBatchId": source_id,
},
"businessStage": "waiting_acceptance",
"updatedAt": now,
}},
)
counters["samplingBackfilled"] += int(result.modified_count)
for name in COLLECTIONS_WITH_LEGACY_ASSIGNMENT:
collection = db[name]
collection.update_many(
{"legacyAssignment": {"$exists": False}, "workgroupId": {"$exists": True}},
[{"$set": {"legacyAssignment": {"workgroupId": "$workgroupId", "vendorId": "$vendorId"}}}],
)
result = collection.update_many({}, {"$unset": {"workgroupId": "", "vendorId": ""}})
counters["legacyFieldsUnset"] += int(result.modified_count)
for source in ("annotationWorkgroups", "annotationMemberships"):
for document in db[source].find({}):
source_id = str(document.get("id") or document.get("_id"))
db.legacyWorkgroupAudit.update_one(
{"source": source, "sourceId": source_id},
{"$set": {"source": source, "sourceId": source_id, "payload": document, "archivedAt": now}},
upsert=True,
)
db[source].drop()
return counters
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Audit/apply Fanyu v5.8 Capture migration")
parser.add_argument("--apply", action="store_true", help="perform idempotent migration updates")
parser.add_argument("--report", help="optional JSON report output path")
args = parser.parse_args(argv)
uri = os.environ.get("FANYU_MONGO_URI", "").strip()
database = os.environ.get("FANYU_MONGO_DATABASE", "drobodata-fanyu").strip()
if not uri:
parser.error("FANYU_MONGO_URI is required")
try:
import pymongo
except ImportError as error:
raise SystemExit("pymongo is required") from error
client = pymongo.MongoClient(uri, serverSelectionTimeoutMS=10_000)
client.admin.command("ping")
db = client[database]
before = build_report(db)
result: dict[str, Any] = {
"mode": "apply" if args.apply else "dry-run",
"database": database,
"connection": redact_mongo_uri(uri),
"before": before,
}
if args.apply:
result["changes"] = apply_migration(db)
result["after"] = build_report(db)
encoded = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
if args.report:
from pathlib import Path
Path(args.report).write_text(encoded, encoding="utf-8")
sys.stdout.write(encoded)
return 0
if __name__ == "__main__":
raise SystemExit(main())