#!/usr/bin/env python3 """Audit/apply the Fanyu v5.8 Capture-policy migration. Credentials are read only from FANYU_MONGO_URI and never emitted. Dry-run is the default; --apply performs idempotent updates after the caller has created a Mongo backup. """ from __future__ import annotations import argparse import datetime as dt import json import os import sys from typing import Any COLLECTIONS_WITH_LEGACY_ASSIGNMENT = ( "annotationTasks", "annotationWorklogEvents", "humanItems", "deliveryPackages", ) def now_iso() -> str: return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") def redact_mongo_uri(value: str) -> str: if "@" not in value: return "mongodb://***" scheme, rest = value.split("://", 1) if "://" in value else ("mongodb", value) return f"{scheme}://***@{rest.split('@', 1)[1]}" def build_report(db: Any) -> dict[str, Any]: legacy = { name: { "workgroupId": db[name].count_documents({"workgroupId": {"$exists": True}}), "vendorId": db[name].count_documents({"vendorId": {"$exists": True}}), "missingSupplier": db[name].count_documents({"$or": [{"supplierId": {"$exists": False}}, {"supplierId": ""}]}), } for name in COLLECTIONS_WITH_LEGACY_ASSIGNMENT } return { "generatedAt": now_iso(), "captureWorkflows": db.workflows.count_documents({"captureAutomation.enabled": True}), "runtimePolicies": db.workflowRuntimePolicies.count_documents({}), "captureRecords": db.captureRecords.count_documents({}), "captureSamplingDecisions": db.captureRecords.count_documents({"samplingDecision": {"$exists": True}}), "acceptanceBatches": db.humanBatches.count_documents({"sourceBatchId": {"$nin": ["", None]}}), "annotationWorkgroups": db.annotationWorkgroups.count_documents({}), "annotationMemberships": db.annotationMemberships.count_documents({}), "legacyReferences": legacy, } def apply_migration(db: Any) -> dict[str, int]: now = now_iso() counters = {"policiesCreated": 0, "samplingBackfilled": 0, "legacyFieldsUnset": 0} for workflow in db.workflows.find({"captureAutomation.enabled": True}, {"id": 1, "organizationId": 1, "projectId": 1}): workflow_id = str(workflow.get("id") or "").strip() if not workflow_id: continue result = db.workflowRuntimePolicies.update_one( {"workflowId": workflow_id}, {"$setOnInsert": { "id": f"workflow-runtime-policy-{workflow_id}", "organizationId": workflow.get("organizationId") or "org-default", "projectId": workflow.get("projectId") or "", "workflowId": workflow_id, "revision": 1, "maxConcurrentCaptureRuns": 2, "acceptanceSampleProbability": 0.5, "updatedByUserId": "system-migration", "updatedAt": now, "history": [{ "revision": 1, "maxConcurrentCaptureRuns": 2, "acceptanceSampleProbability": 0.5, "updatedByUserId": "system-migration", "updatedAt": now, }], }}, upsert=True, ) counters["policiesCreated"] += int(result.upserted_id is not None) acceptance_batches = db.humanBatches.find({"sourceBatchId": {"$nin": ["", None]}}, {"sourceBatchId": 1}) for acceptance_batch in acceptance_batches: source_id = str(acceptance_batch.get("sourceBatchId") or "").strip() source = db.humanBatches.find_one({"id": source_id}, {"runId": 1}) if not source: continue capture = db.captureRecords.find_one({"runId": source.get("runId")}, {"id": 1, "samplingDecision": 1}) if not capture or capture.get("samplingDecision"): continue result = db.captureRecords.update_one( {"id": capture["id"], "samplingDecision": {"$exists": False}}, {"$set": { "samplingDecision": { "decision": "selected", "probability": 1.0, "policyRevision": 0, "randomValue": 0.0, "decidedAt": now, "decidedByUserId": "system-migration", "sourceHumanBatchId": source_id, }, "businessStage": "waiting_acceptance", "updatedAt": now, }}, ) counters["samplingBackfilled"] += int(result.modified_count) for name in COLLECTIONS_WITH_LEGACY_ASSIGNMENT: collection = db[name] collection.update_many( {"legacyAssignment": {"$exists": False}, "workgroupId": {"$exists": True}}, [{"$set": {"legacyAssignment": {"workgroupId": "$workgroupId", "vendorId": "$vendorId"}}}], ) result = collection.update_many({}, {"$unset": {"workgroupId": "", "vendorId": ""}}) counters["legacyFieldsUnset"] += int(result.modified_count) for source in ("annotationWorkgroups", "annotationMemberships"): for document in db[source].find({}): source_id = str(document.get("id") or document.get("_id")) db.legacyWorkgroupAudit.update_one( {"source": source, "sourceId": source_id}, {"$set": {"source": source, "sourceId": source_id, "payload": document, "archivedAt": now}}, upsert=True, ) db[source].drop() return counters def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Audit/apply Fanyu v5.8 Capture migration") parser.add_argument("--apply", action="store_true", help="perform idempotent migration updates") parser.add_argument("--report", help="optional JSON report output path") args = parser.parse_args(argv) uri = os.environ.get("FANYU_MONGO_URI", "").strip() database = os.environ.get("FANYU_MONGO_DATABASE", "drobodata-fanyu").strip() if not uri: parser.error("FANYU_MONGO_URI is required") try: import pymongo except ImportError as error: raise SystemExit("pymongo is required") from error client = pymongo.MongoClient(uri, serverSelectionTimeoutMS=10_000) client.admin.command("ping") db = client[database] before = build_report(db) result: dict[str, Any] = { "mode": "apply" if args.apply else "dry-run", "database": database, "connection": redact_mongo_uri(uri), "before": before, } if args.apply: result["changes"] = apply_migration(db) result["after"] = build_report(db) encoded = json.dumps(result, ensure_ascii=False, indent=2) + "\n" if args.report: from pathlib import Path Path(args.report).write_text(encoded, encoding="utf-8") sys.stdout.write(encoded) return 0 if __name__ == "__main__": raise SystemExit(main())