droboticflow/scripts/lib/drosync_real_evidence.py

399 lines
19 KiB
Python

"""Summarize captured drosync text logs without accessing a service or state DB.
The BOS peak is an estimate of *completed instrumented calls*, including stream
verification. Legacy records also include permit queueing; new records report
queue_wait_ms separately. It is not a count of sockets or HTTP requests in flight.
Missing DEBUG request records are reported as unavailable, not zero.
The result deliberately excludes raw lines, error messages, credentials, URLs
with queries, and request bodies. Secret findings contain classifications only.
"""
from __future__ import annotations
import argparse
from collections import Counter, defaultdict
from datetime import datetime
import json
from pathlib import Path
import re
import sys
from typing import Any
_ANSI = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
_HEADER = re.compile(
r"(?P<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2}))"
r"\s+(?P<level>TRACE|DEBUG|INFO|WARN|ERROR)\s+"
)
_KEY = re.compile(r"(?<![\w.-])([A-Za-z_][\w.-]*)\s*=")
_SPAN = re.compile(r"\b[A-Za-z_][\w:.-]*\{")
_SAFE_SUMMARY = (
"elapsed_ms", "phase_wall_ms", "phase_work_ms", "file_tasks_attempted",
"file_tasks_finished", "file_tasks_failed", "file_tasks_cancelled",
"file_tasks_unfinished", "manifest_files", "deferred_files",
"confirmed_files_skipped", "historical_confirmed_bytes",
"local_confirmed_skip_bytes", "historical_terminal_item_bytes",
"historical_multipart_part_bytes", "already_committing_items",
"successful_put_bytes", "retry_count", "session_counts",
)
_REDACTED = re.compile(r"^(?:\[?redacted\]?|\*+|<redacted>|null|none|)$", re.I)
_SENSITIVE_KEY = re.compile(
r"^(?:authorization|proxy-authorization|password|passwd|secret|secret_access_key|"
r"secretaccesskey|accesskeyid|access_key_id|access_token|accesstoken|refresh_token|"
r"refreshtoken|token|api_token|apitoken|cookie|set-cookie|x-bce-security-token)$",
re.I,
)
_LEGACY_BOS_METRIC = (
"estimated_completed_bos_put_call_overlap_including_permit_wait_and_stream_verification"
)
_BOS_METRIC = "estimated_completed_bos_put_and_stream_verification_overlap_excluding_permit_wait"
def _balanced_end(text: str, start: int) -> int:
"""Return the index after a balanced JSON/span value, respecting quotes."""
pairs = {"{": "}", "[": "]", "(": ")"}
stack = [pairs[text[start]]]
quoted = False
escaped = False
for index in range(start + 1, len(text)):
char = text[index]
if quoted:
if escaped:
escaped = False
elif char == "\\":
escaped = True
elif char == '"':
quoted = False
elif char == '"':
quoted = True
elif char in pairs:
stack.append(pairs[char])
elif char == stack[-1]:
stack.pop()
if not stack:
return index + 1
return len(text)
def _quoted_end(text: str, start: int) -> int:
escaped = False
for index in range(start + 1, len(text)):
char = text[index]
if escaped:
escaped = False
elif char == "\\":
escaped = True
elif char == '"':
return index + 1
return len(text)
def _decode(value: str) -> Any:
value = value.strip()
if value == "None":
return None
if value.startswith("Some(") and value.endswith(")"):
return _decode(value[5:-1])
try:
return json.loads(value)
except (ValueError, TypeError):
return value
def _fields(text: str) -> dict[str, Any]:
"""Parse tracing values, including quoted paths and JSON maps with spaces."""
fields: dict[str, Any] = {}
position = 0
while match := _KEY.search(text, position):
start = match.end()
while start < len(text) and text[start].isspace():
start += 1
if start >= len(text):
end = start
elif text[start] == '"':
end = _quoted_end(text, start)
elif text[start] in "{[(":
end = _balanced_end(text, start)
else:
next_key = _KEY.search(text, start)
end = next_key.start() if next_key else len(text)
fields[match.group(1)] = _decode(text[start:end])
position = max(end, match.end())
return fields
def _record(line: str) -> tuple[str | None, str | None, dict[str, Any]]:
header = _HEADER.search(line)
if not header:
return None, None, {}
remaining = line[header.end():]
fields: dict[str, Any] = {}
# Span values precede the target/module and event body. Process each span
# separately so an inner path or JSON value cannot eat the next span.
while span := _SPAN.match(remaining):
opening = span.end() - 1
end = _balanced_end(remaining, opening)
fields.update(_fields(remaining[opening + 1:end - 1]))
remaining = remaining[end:].lstrip(": ")
fields.update(_fields(remaining))
return header.group("time"), header.group("level"), fields
def _number(value: Any) -> int | None:
return value if isinstance(value, int) and not isinstance(value, bool) else None
def _endpoint(value: Any) -> str:
# Only relative API paths are retained, with query and fragment removed.
path = str(value or "").split("?", 1)[0].split("#", 1)[0]
return path if path.startswith("/") and not path.startswith("//") else ""
def _risks(line: str, fields: dict[str, Any]) -> set[str]:
categories: set[str] = set()
for key, value in fields.items():
if _SENSITIVE_KEY.fullmatch(key) and not _REDACTED.fullmatch(str(value)):
categories.add("sensitive_field")
if re.search(r"\bBearer\s+(?!\[?redacted\]?\b|<redacted>)[A-Za-z0-9_.~+/=-]{8,}", line, re.I):
categories.add("bearer_credential")
if re.search(r"\bbce-auth-v1/[A-Za-z0-9_-]{8,}/", line, re.I):
categories.add("bos_signature")
if re.search(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", line):
categories.add("jwt_credential")
if re.search(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----", line):
categories.add("private_key")
for url in re.findall(r"https?://[^\s\"<>]+", line):
if re.search(r"https?://[^/]+@", url):
categories.add("url_userinfo")
if "?" in url:
query = url.split("?", 1)[1].rstrip("),}]")
if query and not _REDACTED.fullmatch(query):
categories.add("url_query")
return categories
def _new_request(kind: str, endpoint: str) -> dict[str, Any]:
return {"endpoint_kind": kind, "endpoint": endpoint,
"completed": 0, "failed": 0, "retry": 0,
"response_bytes": 0, "bytes": 0, "status_counts": Counter(),
"error_codes": Counter(), "backoff_ms": [], "attempts": [],
"retryable_counts": Counter(), "retry_exhausted_count": 0,
"queue_wait_ms": [], "methods": Counter(), "request_ids": []}
def _request_update(group: dict[str, Any], event: str, fields: dict[str, Any]) -> None:
group[{"request_completed": "completed", "request_failed": "failed",
"request_retry": "retry"}[event]] += 1
if (status := _number(fields.get("status"))) is not None:
group["status_counts"][str(status)] += 1
if fields.get("method") in ("GET", "POST", "PUT", "DELETE", "HEAD", "PATCH", "OPTIONS"):
group["methods"][fields["method"]] += 1
request_id = str(fields.get("request_id") or "")
if re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", request_id) and request_id not in group["request_ids"]:
group["request_ids"].append(request_id)
if event == "request_completed":
group["response_bytes"] += _number(fields.get("response_bytes")) or 0
group["bytes"] += _number(fields.get("bytes")) or 0
if fields.get("error_code"):
code = str(fields["error_code"])
group["error_codes"][code if re.fullmatch(r"[a-z][a-z0-9_]{0,63}", code) else "other"] += 1
if isinstance(fields.get("retryable"), bool):
group["retryable_counts"][str(fields["retryable"]).lower()] += 1
if fields.get("retry_exhausted") is True:
group["retry_exhausted_count"] += 1
for field, collection in [("backoff_ms", "backoff_ms"), ("attempt", "attempts"),
("queue_wait_ms", "queue_wait_ms")]:
if (value := _number(fields.get(field))) is not None:
group[collection].append(value)
def _put_intervals(records: list[dict[str, Any]]) -> dict[str, Any]:
intervals: list[dict[str, Any]] = []
boundaries: list[tuple[int, int]] = []
for record in records:
elapsed = _number(record.get("elapsed_ms"))
if elapsed is None or elapsed < 0:
continue
try:
timestamp = datetime.fromisoformat(record["timestamp"].replace("Z", "+00:00"))
end_us = round(timestamp.timestamp() * 1_000_000)
except (ValueError, KeyError, OverflowError):
continue
start_us = end_us - elapsed * 1000
interval = dict(record, start_epoch_us=start_us, end_epoch_us=end_us)
intervals.append(interval)
if end_us > start_us:
boundaries.extend(((start_us, 1), (end_us, -1)))
active = peak = 0
# End before start on equal timestamps: adjacent calls do not overlap.
for _, change in sorted(boundaries):
active += change
peak = max(peak, active)
queue_separated = bool(records) and all(_number(record.get("queue_wait_ms")) is not None for record in records)
return {"metricLabel": _BOS_METRIC if queue_separated else _LEGACY_BOS_METRIC,
"queueWaitSeparated": queue_separated if records else None,
"estimatedCompletedCallPeak": max(1, peak) if intervals else None,
"completedCallCount": len(records), "intervalCount": len(intervals),
"zeroDurationCount": sum(record.get("elapsed_ms") == 0 for record in intervals),
"successfulBytes": sum(_number(r.get("bytes")) or 0 for r in records),
"intervals": intervals,
"limitations": ["Successful DEBUG completion records only; failed, cancelled and unfinished calls are omitted.",
"Duration includes source checks and stream verification; legacy records also include permit queueing. This is not socket concurrency.",
"Text wall-clock timestamps and millisecond durations introduce rounding and clock-adjustment error."]}
def analyze_logs(text: str) -> dict[str, Any]:
"""Return JSON-serializable diagnostics from existing real captured logs.
Counts are event-record counts, not deduplicated server request counts. A
received HTTP success whose JSON cannot be decoded may also produce a retry
event. A run with no summary remains incomplete (for example after SIGKILL).
"""
events: Counter[str] = Counter()
levels: Counter[str] = Counter()
runs: dict[str, dict[str, Any]] = {}
request_groups: dict[tuple[str, str], dict[str, Any]] = {}
waits: dict[tuple[str, str], dict[str, Any]] = {}
risk_lines: dict[str, list[int]] = defaultdict(list)
missing_run: Counter[str] = Counter()
puts: list[dict[str, Any]] = []
startup: list[dict[str, Any]] = []
state_ready: list[dict[str, Any]] = []
upload_retry_backoffs: list[dict[str, Any]] = []
starts: list[dict[str, Any]] = []
recoveries: list[dict[str, Any]] = []
total = parsed = 0
for total, original in enumerate(text.splitlines(), 1):
line = _ANSI.sub("", original)
timestamp, level, fields = _record(line)
for category in _risks(line, fields):
risk_lines[category].append(total)
if level:
levels[level] += 1
event = fields.get("event")
if not isinstance(event, str) or not event:
continue
parsed += 1
events[event] += 1
run_id = str(fields.get("run_id") or "")
session_id = str(fields.get("session_id") or "")
run = None
if run_id:
run = runs.setdefault(run_id, {
"run_id": run_id, "mapping_id": "", "session_ids": [],
"summaryCount": 0, "status": "incomplete", "elapsed_ms": None,
"phase_wall_ms": {}, "phase_work_ms": {},
"successful_put_bytes": None, "historical_confirmed_bytes": None,
"first_timestamp": timestamp, "last_timestamp": timestamp,
"events": Counter(), "requests": {}, "phase_end_records": [],
"requestFailedCount": 0, "requestRetryCount": 0, "uploadRetryCount": 0,
})
run["last_timestamp"] = timestamp
run["events"][event] += 1
if fields.get("mapping_id"):
run["mapping_id"] = str(fields["mapping_id"])
if session_id and session_id not in run["session_ids"]:
run["session_ids"].append(session_id)
if event == "sync_summary":
run["summaryCount"] += 1
run["status"] = fields.get("outcome", "unknown")
for field in _SAFE_SUMMARY:
if field in fields:
run[field] = fields[field]
elif event == "phase_end":
run["phase_end_records"].append({
"phase": fields.get("phase"), "elapsed_ms": fields.get("elapsed_ms"),
"outcome": fields.get("outcome"), "timestamp": timestamp,
})
elif event not in ("daemon_start", "daemon_state_ready"):
missing_run[event] += 1
if event == "file_sync_retry" and run is not None:
run["uploadRetryCount"] += 1
if event == "upload_retry_backoff":
upload_retry_backoffs.append({
"timestamp": timestamp, "run_id": run_id, "session_id": session_id,
**{key: fields[key] for key in ("item_id", "part_number", "attempt", "next_attempt", "backoff_ms")
if key in fields},
})
if event in ("request_completed", "request_failed", "request_retry"):
kind = str(fields.get("endpoint_kind") or "unknown")
endpoint = _endpoint(fields.get("endpoint"))
key = (kind, endpoint)
group = request_groups.setdefault(key, _new_request(kind, endpoint))
_request_update(group, event, fields)
if run is not None:
run_group = run["requests"].setdefault(key, _new_request(kind, endpoint))
_request_update(run_group, event, fields)
if event == "request_failed":
run["requestFailedCount"] += 1
elif event == "request_retry":
run["requestRetryCount"] += 1
if event == "request_completed" and kind == "bos_put":
puts.append({"timestamp": timestamp, "run_id": run_id,
"session_id": session_id, "item_id": fields.get("item_id"),
"part_number": fields.get("part_number"), "offset": fields.get("offset"),
"bytes": fields.get("bytes"), "elapsed_ms": fields.get("elapsed_ms"),
"status": fields.get("status"), "queue_wait_ms": fields.get("queue_wait_ms")})
if event == "commit_wait":
wait = waits.setdefault((run_id, session_id), {
"run_id": run_id, "session_id": session_id, "elapsed_ms": [],
"timestamps": [], "statuses": [], "request_in_flight": [],
})
wait["elapsed_ms"].append(fields.get("elapsed_ms"))
wait["timestamps"].append(timestamp)
wait["statuses"].append(fields.get("last_observed_status"))
wait["request_in_flight"].append(fields.get("request_in_flight"))
if event == "file_upload_start":
starts.append({"timestamp": timestamp, "run_id": run_id,
"session_id": session_id, "item_id": fields.get("item_id"),
"path": fields.get("path"), "bytes": fields.get("bytes")})
if event in ("session_recovery", "multipart_recovery"):
recoveries.append({"event": event, "timestamp": timestamp, "run_id": run_id,
"session_id": session_id, "item_id": fields.get("item_id"),
**{key: fields[key] for key in ("completed_parts", "remaining_parts",
"historical_part_bytes", "prior_status") if key in fields}})
if event == "daemon_start":
safe = ("version", "protocol_version", "state_schema", "file_concurrency",
"put_concurrency", "part_concurrency", "mapping_concurrency", "hash_concurrency",
"log_level", "log_max_bytes", "log_keep_files")
startup.append({"timestamp": timestamp, **{key: fields[key] for key in safe if key in fields}})
if event == "daemon_state_ready":
state_ready.append({"timestamp": timestamp,
**{key: fields[key] for key in ("version", "protocol_version", "state_schema")
if key in fields}})
for run in runs.values():
run["requests"] = list(run["requests"].values())
run["bosPut"] = _put_intervals([put for put in puts if put["run_id"] == run["run_id"]])
result = {
"schemaVersion": 1, "lineCount": total, "eventLineCount": parsed,
"levels": dict(levels), "events": dict(events), "runs": list(runs.values()),
"requests": list(request_groups.values()), "bosPut": _put_intervals(puts),
"commitWait": list(waits.values()), "requestFailedCount": events["request_failed"],
"requestRetryCount": events["request_retry"], "uploadRetryCount": events["file_sync_retry"],
"uploadRetryBackoffs": upload_retry_backoffs, "fileUploadStartOrder": starts,
"recoveries": recoveries, "startup": startup, "stateReady": state_ready,
"missingRunEventCounts": dict(missing_run),
"secretRisks": [{"category": category, "lineNumbers": lines, "count": len(lines)}
for category, lines in sorted(risk_lines.items())],
"requestMetricLabel": "diagnostic_event_records_not_deduplicated_server_requests",
}
# Counter and tuple-key maps are converted without exposing source text.
return json.loads(json.dumps(result, ensure_ascii=False))
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("logs", type=Path, nargs="*", help="Captured text logs; default stdin")
parser.add_argument("--output", type=Path, help="Write sanitized JSON instead of stdout")
args = parser.parse_args()
text = "\n".join(path.read_text(errors="replace") for path in args.logs) if args.logs else sys.stdin.read()
encoded = json.dumps(analyze_logs(text), ensure_ascii=False, indent=2) + "\n"
if args.output:
args.output.write_text(encoded)
else:
sys.stdout.write(encoded)
if __name__ == "__main__":
main()