forked from longtao.wu/droboticflow
175 lines
6.9 KiB
Python
175 lines
6.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit/apply the Fanyu v5.8 Capture-policy migration.
|
|
|
|
Credentials are read only from FANYU_MONGO_URI and never emitted. Dry-run is
|
|
the default; --apply performs idempotent updates after the caller has created a
|
|
Mongo backup.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
COLLECTIONS_WITH_LEGACY_ASSIGNMENT = (
|
|
"annotationTasks",
|
|
"annotationWorklogEvents",
|
|
"humanItems",
|
|
"deliveryPackages",
|
|
)
|
|
|
|
|
|
def now_iso() -> str:
|
|
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def redact_mongo_uri(value: str) -> str:
|
|
if "@" not in value:
|
|
return "mongodb://***"
|
|
scheme, rest = value.split("://", 1) if "://" in value else ("mongodb", value)
|
|
return f"{scheme}://***@{rest.split('@', 1)[1]}"
|
|
|
|
|
|
def build_report(db: Any) -> dict[str, Any]:
|
|
legacy = {
|
|
name: {
|
|
"workgroupId": db[name].count_documents({"workgroupId": {"$exists": True}}),
|
|
"vendorId": db[name].count_documents({"vendorId": {"$exists": True}}),
|
|
"missingSupplier": db[name].count_documents({"$or": [{"supplierId": {"$exists": False}}, {"supplierId": ""}]}),
|
|
}
|
|
for name in COLLECTIONS_WITH_LEGACY_ASSIGNMENT
|
|
}
|
|
return {
|
|
"generatedAt": now_iso(),
|
|
"captureWorkflows": db.workflows.count_documents({"captureAutomation.enabled": True}),
|
|
"runtimePolicies": db.workflowRuntimePolicies.count_documents({}),
|
|
"captureRecords": db.captureRecords.count_documents({}),
|
|
"captureSamplingDecisions": db.captureRecords.count_documents({"samplingDecision": {"$exists": True}}),
|
|
"acceptanceBatches": db.humanBatches.count_documents({"sourceBatchId": {"$nin": ["", None]}}),
|
|
"annotationWorkgroups": db.annotationWorkgroups.count_documents({}),
|
|
"annotationMemberships": db.annotationMemberships.count_documents({}),
|
|
"legacyReferences": legacy,
|
|
}
|
|
|
|
|
|
def apply_migration(db: Any) -> dict[str, int]:
|
|
now = now_iso()
|
|
counters = {"policiesCreated": 0, "samplingBackfilled": 0, "legacyFieldsUnset": 0}
|
|
for workflow in db.workflows.find({"captureAutomation.enabled": True}, {"id": 1, "organizationId": 1, "projectId": 1}):
|
|
workflow_id = str(workflow.get("id") or "").strip()
|
|
if not workflow_id:
|
|
continue
|
|
result = db.workflowRuntimePolicies.update_one(
|
|
{"workflowId": workflow_id},
|
|
{"$setOnInsert": {
|
|
"id": f"workflow-runtime-policy-{workflow_id}",
|
|
"organizationId": workflow.get("organizationId") or "org-default",
|
|
"projectId": workflow.get("projectId") or "",
|
|
"workflowId": workflow_id,
|
|
"revision": 1,
|
|
"maxConcurrentCaptureRuns": 2,
|
|
"acceptanceSampleProbability": 0.5,
|
|
"updatedByUserId": "system-migration",
|
|
"updatedAt": now,
|
|
"history": [{
|
|
"revision": 1,
|
|
"maxConcurrentCaptureRuns": 2,
|
|
"acceptanceSampleProbability": 0.5,
|
|
"updatedByUserId": "system-migration",
|
|
"updatedAt": now,
|
|
}],
|
|
}},
|
|
upsert=True,
|
|
)
|
|
counters["policiesCreated"] += int(result.upserted_id is not None)
|
|
|
|
acceptance_batches = db.humanBatches.find({"sourceBatchId": {"$nin": ["", None]}}, {"sourceBatchId": 1})
|
|
for acceptance_batch in acceptance_batches:
|
|
source_id = str(acceptance_batch.get("sourceBatchId") or "").strip()
|
|
source = db.humanBatches.find_one({"id": source_id}, {"runId": 1})
|
|
if not source:
|
|
continue
|
|
capture = db.captureRecords.find_one({"runId": source.get("runId")}, {"id": 1, "samplingDecision": 1})
|
|
if not capture or capture.get("samplingDecision"):
|
|
continue
|
|
result = db.captureRecords.update_one(
|
|
{"id": capture["id"], "samplingDecision": {"$exists": False}},
|
|
{"$set": {
|
|
"samplingDecision": {
|
|
"decision": "selected",
|
|
"probability": 1.0,
|
|
"policyRevision": 0,
|
|
"randomValue": 0.0,
|
|
"decidedAt": now,
|
|
"decidedByUserId": "system-migration",
|
|
"sourceHumanBatchId": source_id,
|
|
},
|
|
"businessStage": "waiting_acceptance",
|
|
"updatedAt": now,
|
|
}},
|
|
)
|
|
counters["samplingBackfilled"] += int(result.modified_count)
|
|
|
|
for name in COLLECTIONS_WITH_LEGACY_ASSIGNMENT:
|
|
collection = db[name]
|
|
collection.update_many(
|
|
{"legacyAssignment": {"$exists": False}, "workgroupId": {"$exists": True}},
|
|
[{"$set": {"legacyAssignment": {"workgroupId": "$workgroupId", "vendorId": "$vendorId"}}}],
|
|
)
|
|
result = collection.update_many({}, {"$unset": {"workgroupId": "", "vendorId": ""}})
|
|
counters["legacyFieldsUnset"] += int(result.modified_count)
|
|
|
|
for source in ("annotationWorkgroups", "annotationMemberships"):
|
|
for document in db[source].find({}):
|
|
source_id = str(document.get("id") or document.get("_id"))
|
|
db.legacyWorkgroupAudit.update_one(
|
|
{"source": source, "sourceId": source_id},
|
|
{"$set": {"source": source, "sourceId": source_id, "payload": document, "archivedAt": now}},
|
|
upsert=True,
|
|
)
|
|
db[source].drop()
|
|
return counters
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Audit/apply Fanyu v5.8 Capture migration")
|
|
parser.add_argument("--apply", action="store_true", help="perform idempotent migration updates")
|
|
parser.add_argument("--report", help="optional JSON report output path")
|
|
args = parser.parse_args(argv)
|
|
uri = os.environ.get("FANYU_MONGO_URI", "").strip()
|
|
database = os.environ.get("FANYU_MONGO_DATABASE", "drobodata-fanyu").strip()
|
|
if not uri:
|
|
parser.error("FANYU_MONGO_URI is required")
|
|
try:
|
|
import pymongo
|
|
except ImportError as error:
|
|
raise SystemExit("pymongo is required") from error
|
|
client = pymongo.MongoClient(uri, serverSelectionTimeoutMS=10_000)
|
|
client.admin.command("ping")
|
|
db = client[database]
|
|
before = build_report(db)
|
|
result: dict[str, Any] = {
|
|
"mode": "apply" if args.apply else "dry-run",
|
|
"database": database,
|
|
"connection": redact_mongo_uri(uri),
|
|
"before": before,
|
|
}
|
|
if args.apply:
|
|
result["changes"] = apply_migration(db)
|
|
result["after"] = build_report(db)
|
|
encoded = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
|
if args.report:
|
|
from pathlib import Path
|
|
Path(args.report).write_text(encoded, encoding="utf-8")
|
|
sys.stdout.write(encoded)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|