droboticflow/scripts/lib/drosync_real_order.py

186 lines
10 KiB
Python

#!/usr/bin/env python3
"""Read actual drosync text logs to check start order and metadata requests.
file_upload_start follows reservation, so parallel starts can reorder. This
reports that observation explicitly, rather than inventing a dispatch event.
Client /sessions/{id}/items GETs are counted independently of BOS SDK LISTs.
"""
from __future__ import annotations
import argparse
from collections import Counter
from datetime import datetime
import json
from pathlib import Path
import re
import sys
from drosync_real_evidence import _record
_ITEMS = re.compile(r"^/api/external/object-sync/sessions/([^/]+)/items$")
_STATUS = re.compile(r"^/api/external/object-sync/sessions/([^/]+)/status$")
_MANIFEST = re.compile(r"^/api/external/object-sync/sessions/([^/]+)/manifest-pages$")
def capture_order(path: str) -> tuple[str, str]:
"""Match state::capture_upload_order's first valid date component."""
for component in path.split("/"):
for length, formats in ((19, ("%Y_%m_%d-%H_%M_%S", "%Y-%m-%d-%H-%M-%S")),
(10, ("%Y_%m_%d", "%Y-%m-%d"))):
if len(component) != length:
continue
for fmt in formats:
try:
value = datetime.strptime(component, fmt)
except ValueError:
continue
return value.strftime("%Y%m%d%H%M%S"), path
return "99999999999999", path
def _order(starts: list[dict], concurrency: int, expected_paths: list[str] | None) -> dict:
paths = [entry["path"] for entry in starts]
duplicates = sorted(path for path, count in Counter(paths).items() if count > 1)
initial_paths = list(dict.fromkeys(paths))
expected = sorted(set(expected_paths if expected_paths is not None else initial_paths), key=capture_order)
missing = sorted(set(expected) - set(initial_paths), key=capture_order)
unexpected = sorted(set(initial_paths) - set(expected), key=capture_order)
remaining = expected.copy()
violations = []
max_ahead = 0
for observed_index, path in enumerate(initial_paths):
if path not in remaining:
continue
ahead = remaining.index(path)
max_ahead = max(max_ahead, ahead)
if ahead >= concurrency:
violations.append({"observedIndex": observed_index, "path": path,
"earliestUnobserved": remaining[0], "aheadOfEarliest": ahead})
remaining.remove(path)
strict = initial_paths == expected
complete = not missing and not unexpected and not duplicates
passed = complete and (strict if concurrency == 1 else not violations)
return {"status": "PASS" if passed else "FAIL",
"criterion": "strict start order" if concurrency == 1 else "start within earliest unobserved concurrency window",
"evidenceLimit": "file_upload_start follows HTTP reservation, not task admission; parallel HTTP timing can exceed this observational window",
"coverage": "provided expected ready files" if expected_paths is not None else "observed start events only",
"observedStartCount": len(starts), "uniqueStartCount": len(initial_paths),
"strictlyOrdered": strict, "maxAheadOfEarliestUnobserved": max_ahead,
"missingExpectedPaths": missing, "unexpectedPaths": unexpected,
"duplicatePaths": duplicates, "violations": violations[:32],
"firstStarts": starts[:max(8, concurrency)],
"captureTimeRanges": dict(Counter(capture_order(path)[0] for path in initial_paths))}
def analyze_order_and_metadata(text: str, concurrency: int = 4,
expected_paths: list[str] | None = None,
run_id: str | None = None) -> dict:
if not 1 <= concurrency <= 64:
raise ValueError("invalid concurrency")
records = []
for index, line in enumerate(text.splitlines()):
timestamp, level, fields = _record(line)
if timestamp and fields.get("event"):
records.append((timestamp, index, level, fields))
# Rotation files may be concatenated by name; restore their timestamps.
records.sort(key=lambda record: (record[0], record[1]))
phases: dict[str, str] = {}
runs: dict[str, dict] = {}
for timestamp, _, level, fields in records:
rid = str(fields.get("run_id") or "")
if not rid or (run_id is not None and rid != run_id):
continue
run = runs.setdefault(rid, {"starts": [], "itemsGetPagesByPhase": Counter(),
"itemsGetResponseBytesByPhase": Counter(),
"manifestPostPages": 0, "statusGetByPhase": Counter(),
"successfulControlRequests": 0, "sessions": set(),
"completedPhases": set(), "outcome": None, "manifestFiles": None})
event = fields.get("event")
phase = str(fields.get("phase") or "")
# Concurrent per-file work phases must not replace the run's phase.
if event == "phase_start" and not fields.get("item_id"):
phases[rid] = phase
elif event == "phase_end" and not fields.get("item_id"):
run["completedPhases"].add(phase)
if phases.get(rid) == phase:
phases.pop(rid, None)
if fields.get("session_id"):
run["sessions"].add(str(fields["session_id"]))
if event == "file_upload_start":
run["starts"].append({"timestamp": timestamp, "path": str(fields.get("path") or ""),
"itemId": str(fields.get("item_id") or ""),
"sessionId": str(fields.get("session_id") or "")})
if event == "sync_summary":
run["outcome"] = fields.get("outcome")
if event == "session_manifest_bound":
run["manifestFiles"] = int(fields.get("manifest_files") or 0)
if event != "request_completed" or fields.get("endpoint_kind") != "control":
continue
run["successfulControlRequests"] += 1
endpoint = str(fields.get("endpoint") or "").split("?", 1)[0]
active_phase = phases.get(rid, "outside_phase")
if fields.get("method") == "GET" and (match := _ITEMS.fullmatch(endpoint)):
run["itemsGetPagesByPhase"][active_phase] += 1
run["itemsGetResponseBytesByPhase"][active_phase] += int(fields.get("response_bytes") or 0)
run["sessions"].add(match.group(1))
elif fields.get("method") == "GET" and _STATUS.fullmatch(endpoint):
run["statusGetByPhase"][active_phase] += 1
elif fields.get("method") == "POST" and _MANIFEST.fullmatch(endpoint):
run["manifestPostPages"] += 1
results = []
for rid, run in runs.items():
pages = dict(run["itemsGetPagesByPhase"])
commit_completed = "commit_wait" in run["completedPhases"]
diagnostic_available = run["successfulControlRequests"] > 0
commit_gets = pages.get("commit_wait", 0)
metadata_status = ("PASS" if commit_gets == 0 else "FAIL") if diagnostic_available and commit_completed else "NOT_RUN"
manifest_files = run["manifestFiles"]
expected_pages = (manifest_files + 999) // 1000 if manifest_files is not None else None
bounded_status = "NOT_RUN"
if expected_pages is not None and diagnostic_available and run["outcome"] == "completed":
bounded_status = "PASS" if pages.get("upload_pipeline", 0) <= expected_pages and pages.get("terminal_details", 0) <= expected_pages else "FAIL"
if bounded_status == "FAIL":
metadata_status = "FAIL"
result = {"runId": rid, "sessionIds": sorted(run["sessions"]), "outcome": run["outcome"],
"ordering": _order(run["starts"], concurrency, expected_paths) if run["starts"] else {"status": "NOT_RUN", "observedStartCount": 0},
"metadata": {"status": metadata_status,
"source": "client request_completed control events; no SDK LIST events",
"itemsGetPagesTotal": sum(pages.values()), "itemsGetPagesByPhase": pages,
"itemsGetResponseBytesByPhase": dict(run["itemsGetResponseBytesByPhase"]),
"manifestPostPages": run["manifestPostPages"],
"statusGetByPhase": dict(run["statusGetByPhase"]),
"commitWaitItemsGetPages": commit_gets,
"commitWaitCompleted": commit_completed,
"requestDiagnosticsAvailable": diagnostic_available,
"manifestFiles": manifest_files,
"expectedPagesPerInventoryPass": expected_pages,
"boundedInventoryPaginationStatus": bounded_status}}
results.append(result)
statuses = [r[k]["status"] for r in results for k in ("ordering", "metadata")]
return {"status": "FAIL" if "FAIL" in statuses else "PASS" if "PASS" in statuses else "NOT_RUN",
"concurrency": concurrency, "runs": results}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--log", action="append", required=True, help="Actual daemon text log; repeat for rotations")
parser.add_argument("--concurrency", type=int, choices=range(1, 65), default=4)
parser.add_argument("--expected-paths", help="JSON string array of all initially ready relative paths, including mapping remote prefix")
parser.add_argument("--run-id")
args = parser.parse_args()
try:
text = "\n".join(Path(path).read_text(errors="replace") for path in args.log)
expected = json.loads(Path(args.expected_paths).read_text()) if args.expected_paths else None
if expected is not None and (not isinstance(expected, list) or not all(isinstance(path, str) for path in expected)):
raise ValueError("invalid expected paths")
result = analyze_order_and_metadata(text, args.concurrency, expected, args.run_id)
except (OSError, ValueError, TypeError):
print(json.dumps({"status": "FAIL", "classification": "invalid_log_or_arguments"}))
return 2
print(json.dumps(result, ensure_ascii=False))
return 1 if result["status"] == "FAIL" else 0
if __name__ == "__main__":
sys.exit(main())