forked from longtao.wu/droboticflow
186 lines
10 KiB
Python
186 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Read actual drosync text logs to check start order and metadata requests.
|
|
|
|
file_upload_start follows reservation, so parallel starts can reorder. This
|
|
reports that observation explicitly, rather than inventing a dispatch event.
|
|
Client /sessions/{id}/items GETs are counted independently of BOS SDK LISTs.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
from datetime import datetime
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
|
|
from drosync_real_evidence import _record
|
|
|
|
_ITEMS = re.compile(r"^/api/external/object-sync/sessions/([^/]+)/items$")
|
|
_STATUS = re.compile(r"^/api/external/object-sync/sessions/([^/]+)/status$")
|
|
_MANIFEST = re.compile(r"^/api/external/object-sync/sessions/([^/]+)/manifest-pages$")
|
|
|
|
|
|
def capture_order(path: str) -> tuple[str, str]:
|
|
"""Match state::capture_upload_order's first valid date component."""
|
|
for component in path.split("/"):
|
|
for length, formats in ((19, ("%Y_%m_%d-%H_%M_%S", "%Y-%m-%d-%H-%M-%S")),
|
|
(10, ("%Y_%m_%d", "%Y-%m-%d"))):
|
|
if len(component) != length:
|
|
continue
|
|
for fmt in formats:
|
|
try:
|
|
value = datetime.strptime(component, fmt)
|
|
except ValueError:
|
|
continue
|
|
return value.strftime("%Y%m%d%H%M%S"), path
|
|
return "99999999999999", path
|
|
|
|
|
|
def _order(starts: list[dict], concurrency: int, expected_paths: list[str] | None) -> dict:
|
|
paths = [entry["path"] for entry in starts]
|
|
duplicates = sorted(path for path, count in Counter(paths).items() if count > 1)
|
|
initial_paths = list(dict.fromkeys(paths))
|
|
expected = sorted(set(expected_paths if expected_paths is not None else initial_paths), key=capture_order)
|
|
missing = sorted(set(expected) - set(initial_paths), key=capture_order)
|
|
unexpected = sorted(set(initial_paths) - set(expected), key=capture_order)
|
|
remaining = expected.copy()
|
|
violations = []
|
|
max_ahead = 0
|
|
for observed_index, path in enumerate(initial_paths):
|
|
if path not in remaining:
|
|
continue
|
|
ahead = remaining.index(path)
|
|
max_ahead = max(max_ahead, ahead)
|
|
if ahead >= concurrency:
|
|
violations.append({"observedIndex": observed_index, "path": path,
|
|
"earliestUnobserved": remaining[0], "aheadOfEarliest": ahead})
|
|
remaining.remove(path)
|
|
strict = initial_paths == expected
|
|
complete = not missing and not unexpected and not duplicates
|
|
passed = complete and (strict if concurrency == 1 else not violations)
|
|
return {"status": "PASS" if passed else "FAIL",
|
|
"criterion": "strict start order" if concurrency == 1 else "start within earliest unobserved concurrency window",
|
|
"evidenceLimit": "file_upload_start follows HTTP reservation, not task admission; parallel HTTP timing can exceed this observational window",
|
|
"coverage": "provided expected ready files" if expected_paths is not None else "observed start events only",
|
|
"observedStartCount": len(starts), "uniqueStartCount": len(initial_paths),
|
|
"strictlyOrdered": strict, "maxAheadOfEarliestUnobserved": max_ahead,
|
|
"missingExpectedPaths": missing, "unexpectedPaths": unexpected,
|
|
"duplicatePaths": duplicates, "violations": violations[:32],
|
|
"firstStarts": starts[:max(8, concurrency)],
|
|
"captureTimeRanges": dict(Counter(capture_order(path)[0] for path in initial_paths))}
|
|
|
|
|
|
def analyze_order_and_metadata(text: str, concurrency: int = 4,
|
|
expected_paths: list[str] | None = None,
|
|
run_id: str | None = None) -> dict:
|
|
if not 1 <= concurrency <= 64:
|
|
raise ValueError("invalid concurrency")
|
|
records = []
|
|
for index, line in enumerate(text.splitlines()):
|
|
timestamp, level, fields = _record(line)
|
|
if timestamp and fields.get("event"):
|
|
records.append((timestamp, index, level, fields))
|
|
# Rotation files may be concatenated by name; restore their timestamps.
|
|
records.sort(key=lambda record: (record[0], record[1]))
|
|
phases: dict[str, str] = {}
|
|
runs: dict[str, dict] = {}
|
|
for timestamp, _, level, fields in records:
|
|
rid = str(fields.get("run_id") or "")
|
|
if not rid or (run_id is not None and rid != run_id):
|
|
continue
|
|
run = runs.setdefault(rid, {"starts": [], "itemsGetPagesByPhase": Counter(),
|
|
"itemsGetResponseBytesByPhase": Counter(),
|
|
"manifestPostPages": 0, "statusGetByPhase": Counter(),
|
|
"successfulControlRequests": 0, "sessions": set(),
|
|
"completedPhases": set(), "outcome": None, "manifestFiles": None})
|
|
event = fields.get("event")
|
|
phase = str(fields.get("phase") or "")
|
|
# Concurrent per-file work phases must not replace the run's phase.
|
|
if event == "phase_start" and not fields.get("item_id"):
|
|
phases[rid] = phase
|
|
elif event == "phase_end" and not fields.get("item_id"):
|
|
run["completedPhases"].add(phase)
|
|
if phases.get(rid) == phase:
|
|
phases.pop(rid, None)
|
|
if fields.get("session_id"):
|
|
run["sessions"].add(str(fields["session_id"]))
|
|
if event == "file_upload_start":
|
|
run["starts"].append({"timestamp": timestamp, "path": str(fields.get("path") or ""),
|
|
"itemId": str(fields.get("item_id") or ""),
|
|
"sessionId": str(fields.get("session_id") or "")})
|
|
if event == "sync_summary":
|
|
run["outcome"] = fields.get("outcome")
|
|
if event == "session_manifest_bound":
|
|
run["manifestFiles"] = int(fields.get("manifest_files") or 0)
|
|
if event != "request_completed" or fields.get("endpoint_kind") != "control":
|
|
continue
|
|
run["successfulControlRequests"] += 1
|
|
endpoint = str(fields.get("endpoint") or "").split("?", 1)[0]
|
|
active_phase = phases.get(rid, "outside_phase")
|
|
if fields.get("method") == "GET" and (match := _ITEMS.fullmatch(endpoint)):
|
|
run["itemsGetPagesByPhase"][active_phase] += 1
|
|
run["itemsGetResponseBytesByPhase"][active_phase] += int(fields.get("response_bytes") or 0)
|
|
run["sessions"].add(match.group(1))
|
|
elif fields.get("method") == "GET" and _STATUS.fullmatch(endpoint):
|
|
run["statusGetByPhase"][active_phase] += 1
|
|
elif fields.get("method") == "POST" and _MANIFEST.fullmatch(endpoint):
|
|
run["manifestPostPages"] += 1
|
|
results = []
|
|
for rid, run in runs.items():
|
|
pages = dict(run["itemsGetPagesByPhase"])
|
|
commit_completed = "commit_wait" in run["completedPhases"]
|
|
diagnostic_available = run["successfulControlRequests"] > 0
|
|
commit_gets = pages.get("commit_wait", 0)
|
|
metadata_status = ("PASS" if commit_gets == 0 else "FAIL") if diagnostic_available and commit_completed else "NOT_RUN"
|
|
manifest_files = run["manifestFiles"]
|
|
expected_pages = (manifest_files + 999) // 1000 if manifest_files is not None else None
|
|
bounded_status = "NOT_RUN"
|
|
if expected_pages is not None and diagnostic_available and run["outcome"] == "completed":
|
|
bounded_status = "PASS" if pages.get("upload_pipeline", 0) <= expected_pages and pages.get("terminal_details", 0) <= expected_pages else "FAIL"
|
|
if bounded_status == "FAIL":
|
|
metadata_status = "FAIL"
|
|
result = {"runId": rid, "sessionIds": sorted(run["sessions"]), "outcome": run["outcome"],
|
|
"ordering": _order(run["starts"], concurrency, expected_paths) if run["starts"] else {"status": "NOT_RUN", "observedStartCount": 0},
|
|
"metadata": {"status": metadata_status,
|
|
"source": "client request_completed control events; no SDK LIST events",
|
|
"itemsGetPagesTotal": sum(pages.values()), "itemsGetPagesByPhase": pages,
|
|
"itemsGetResponseBytesByPhase": dict(run["itemsGetResponseBytesByPhase"]),
|
|
"manifestPostPages": run["manifestPostPages"],
|
|
"statusGetByPhase": dict(run["statusGetByPhase"]),
|
|
"commitWaitItemsGetPages": commit_gets,
|
|
"commitWaitCompleted": commit_completed,
|
|
"requestDiagnosticsAvailable": diagnostic_available,
|
|
"manifestFiles": manifest_files,
|
|
"expectedPagesPerInventoryPass": expected_pages,
|
|
"boundedInventoryPaginationStatus": bounded_status}}
|
|
results.append(result)
|
|
statuses = [r[k]["status"] for r in results for k in ("ordering", "metadata")]
|
|
return {"status": "FAIL" if "FAIL" in statuses else "PASS" if "PASS" in statuses else "NOT_RUN",
|
|
"concurrency": concurrency, "runs": results}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--log", action="append", required=True, help="Actual daemon text log; repeat for rotations")
|
|
parser.add_argument("--concurrency", type=int, choices=range(1, 65), default=4)
|
|
parser.add_argument("--expected-paths", help="JSON string array of all initially ready relative paths, including mapping remote prefix")
|
|
parser.add_argument("--run-id")
|
|
args = parser.parse_args()
|
|
try:
|
|
text = "\n".join(Path(path).read_text(errors="replace") for path in args.log)
|
|
expected = json.loads(Path(args.expected_paths).read_text()) if args.expected_paths else None
|
|
if expected is not None and (not isinstance(expected, list) or not all(isinstance(path, str) for path in expected)):
|
|
raise ValueError("invalid expected paths")
|
|
result = analyze_order_and_metadata(text, args.concurrency, expected, args.run_id)
|
|
except (OSError, ValueError, TypeError):
|
|
print(json.dumps({"status": "FAIL", "classification": "invalid_log_or_arguments"}))
|
|
return 2
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
return 1 if result["status"] == "FAIL" else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|