forked from longtao.wu/droboticflow
399 lines
19 KiB
Python
399 lines
19 KiB
Python
"""Summarize captured drosync text logs without accessing a service or state DB.
|
|
|
|
The BOS peak is an estimate of *completed instrumented calls*, including stream
|
|
verification. Legacy records also include permit queueing; new records report
|
|
queue_wait_ms separately. It is not a count of sockets or HTTP requests in flight.
|
|
Missing DEBUG request records are reported as unavailable, not zero.
|
|
The result deliberately excludes raw lines, error messages, credentials, URLs
|
|
with queries, and request bodies. Secret findings contain classifications only.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
_ANSI = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
|
|
_HEADER = re.compile(
|
|
r"(?P<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2}))"
|
|
r"\s+(?P<level>TRACE|DEBUG|INFO|WARN|ERROR)\s+"
|
|
)
|
|
_KEY = re.compile(r"(?<![\w.-])([A-Za-z_][\w.-]*)\s*=")
|
|
_SPAN = re.compile(r"\b[A-Za-z_][\w:.-]*\{")
|
|
_SAFE_SUMMARY = (
|
|
"elapsed_ms", "phase_wall_ms", "phase_work_ms", "file_tasks_attempted",
|
|
"file_tasks_finished", "file_tasks_failed", "file_tasks_cancelled",
|
|
"file_tasks_unfinished", "manifest_files", "deferred_files",
|
|
"confirmed_files_skipped", "historical_confirmed_bytes",
|
|
"local_confirmed_skip_bytes", "historical_terminal_item_bytes",
|
|
"historical_multipart_part_bytes", "already_committing_items",
|
|
"successful_put_bytes", "retry_count", "session_counts",
|
|
)
|
|
_REDACTED = re.compile(r"^(?:\[?redacted\]?|\*+|<redacted>|null|none|)$", re.I)
|
|
_SENSITIVE_KEY = re.compile(
|
|
r"^(?:authorization|proxy-authorization|password|passwd|secret|secret_access_key|"
|
|
r"secretaccesskey|accesskeyid|access_key_id|access_token|accesstoken|refresh_token|"
|
|
r"refreshtoken|token|api_token|apitoken|cookie|set-cookie|x-bce-security-token)$",
|
|
re.I,
|
|
)
|
|
_LEGACY_BOS_METRIC = (
|
|
"estimated_completed_bos_put_call_overlap_including_permit_wait_and_stream_verification"
|
|
)
|
|
_BOS_METRIC = "estimated_completed_bos_put_and_stream_verification_overlap_excluding_permit_wait"
|
|
|
|
|
|
def _balanced_end(text: str, start: int) -> int:
|
|
"""Return the index after a balanced JSON/span value, respecting quotes."""
|
|
pairs = {"{": "}", "[": "]", "(": ")"}
|
|
stack = [pairs[text[start]]]
|
|
quoted = False
|
|
escaped = False
|
|
for index in range(start + 1, len(text)):
|
|
char = text[index]
|
|
if quoted:
|
|
if escaped:
|
|
escaped = False
|
|
elif char == "\\":
|
|
escaped = True
|
|
elif char == '"':
|
|
quoted = False
|
|
elif char == '"':
|
|
quoted = True
|
|
elif char in pairs:
|
|
stack.append(pairs[char])
|
|
elif char == stack[-1]:
|
|
stack.pop()
|
|
if not stack:
|
|
return index + 1
|
|
return len(text)
|
|
|
|
|
|
def _quoted_end(text: str, start: int) -> int:
|
|
escaped = False
|
|
for index in range(start + 1, len(text)):
|
|
char = text[index]
|
|
if escaped:
|
|
escaped = False
|
|
elif char == "\\":
|
|
escaped = True
|
|
elif char == '"':
|
|
return index + 1
|
|
return len(text)
|
|
|
|
|
|
def _decode(value: str) -> Any:
|
|
value = value.strip()
|
|
if value == "None":
|
|
return None
|
|
if value.startswith("Some(") and value.endswith(")"):
|
|
return _decode(value[5:-1])
|
|
try:
|
|
return json.loads(value)
|
|
except (ValueError, TypeError):
|
|
return value
|
|
|
|
|
|
def _fields(text: str) -> dict[str, Any]:
|
|
"""Parse tracing values, including quoted paths and JSON maps with spaces."""
|
|
fields: dict[str, Any] = {}
|
|
position = 0
|
|
while match := _KEY.search(text, position):
|
|
start = match.end()
|
|
while start < len(text) and text[start].isspace():
|
|
start += 1
|
|
if start >= len(text):
|
|
end = start
|
|
elif text[start] == '"':
|
|
end = _quoted_end(text, start)
|
|
elif text[start] in "{[(":
|
|
end = _balanced_end(text, start)
|
|
else:
|
|
next_key = _KEY.search(text, start)
|
|
end = next_key.start() if next_key else len(text)
|
|
fields[match.group(1)] = _decode(text[start:end])
|
|
position = max(end, match.end())
|
|
return fields
|
|
|
|
|
|
def _record(line: str) -> tuple[str | None, str | None, dict[str, Any]]:
|
|
header = _HEADER.search(line)
|
|
if not header:
|
|
return None, None, {}
|
|
remaining = line[header.end():]
|
|
fields: dict[str, Any] = {}
|
|
# Span values precede the target/module and event body. Process each span
|
|
# separately so an inner path or JSON value cannot eat the next span.
|
|
while span := _SPAN.match(remaining):
|
|
opening = span.end() - 1
|
|
end = _balanced_end(remaining, opening)
|
|
fields.update(_fields(remaining[opening + 1:end - 1]))
|
|
remaining = remaining[end:].lstrip(": ")
|
|
fields.update(_fields(remaining))
|
|
return header.group("time"), header.group("level"), fields
|
|
|
|
|
|
def _number(value: Any) -> int | None:
|
|
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
|
|
|
|
|
def _endpoint(value: Any) -> str:
|
|
# Only relative API paths are retained, with query and fragment removed.
|
|
path = str(value or "").split("?", 1)[0].split("#", 1)[0]
|
|
return path if path.startswith("/") and not path.startswith("//") else ""
|
|
|
|
|
|
def _risks(line: str, fields: dict[str, Any]) -> set[str]:
|
|
categories: set[str] = set()
|
|
for key, value in fields.items():
|
|
if _SENSITIVE_KEY.fullmatch(key) and not _REDACTED.fullmatch(str(value)):
|
|
categories.add("sensitive_field")
|
|
if re.search(r"\bBearer\s+(?!\[?redacted\]?\b|<redacted>)[A-Za-z0-9_.~+/=-]{8,}", line, re.I):
|
|
categories.add("bearer_credential")
|
|
if re.search(r"\bbce-auth-v1/[A-Za-z0-9_-]{8,}/", line, re.I):
|
|
categories.add("bos_signature")
|
|
if re.search(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", line):
|
|
categories.add("jwt_credential")
|
|
if re.search(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----", line):
|
|
categories.add("private_key")
|
|
for url in re.findall(r"https?://[^\s\"<>]+", line):
|
|
if re.search(r"https?://[^/]+@", url):
|
|
categories.add("url_userinfo")
|
|
if "?" in url:
|
|
query = url.split("?", 1)[1].rstrip("),}]")
|
|
if query and not _REDACTED.fullmatch(query):
|
|
categories.add("url_query")
|
|
return categories
|
|
|
|
|
|
def _new_request(kind: str, endpoint: str) -> dict[str, Any]:
|
|
return {"endpoint_kind": kind, "endpoint": endpoint,
|
|
"completed": 0, "failed": 0, "retry": 0,
|
|
"response_bytes": 0, "bytes": 0, "status_counts": Counter(),
|
|
"error_codes": Counter(), "backoff_ms": [], "attempts": [],
|
|
"retryable_counts": Counter(), "retry_exhausted_count": 0,
|
|
"queue_wait_ms": [], "methods": Counter(), "request_ids": []}
|
|
|
|
|
|
def _request_update(group: dict[str, Any], event: str, fields: dict[str, Any]) -> None:
|
|
group[{"request_completed": "completed", "request_failed": "failed",
|
|
"request_retry": "retry"}[event]] += 1
|
|
if (status := _number(fields.get("status"))) is not None:
|
|
group["status_counts"][str(status)] += 1
|
|
if fields.get("method") in ("GET", "POST", "PUT", "DELETE", "HEAD", "PATCH", "OPTIONS"):
|
|
group["methods"][fields["method"]] += 1
|
|
request_id = str(fields.get("request_id") or "")
|
|
if re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", request_id) and request_id not in group["request_ids"]:
|
|
group["request_ids"].append(request_id)
|
|
if event == "request_completed":
|
|
group["response_bytes"] += _number(fields.get("response_bytes")) or 0
|
|
group["bytes"] += _number(fields.get("bytes")) or 0
|
|
if fields.get("error_code"):
|
|
code = str(fields["error_code"])
|
|
group["error_codes"][code if re.fullmatch(r"[a-z][a-z0-9_]{0,63}", code) else "other"] += 1
|
|
if isinstance(fields.get("retryable"), bool):
|
|
group["retryable_counts"][str(fields["retryable"]).lower()] += 1
|
|
if fields.get("retry_exhausted") is True:
|
|
group["retry_exhausted_count"] += 1
|
|
for field, collection in [("backoff_ms", "backoff_ms"), ("attempt", "attempts"),
|
|
("queue_wait_ms", "queue_wait_ms")]:
|
|
if (value := _number(fields.get(field))) is not None:
|
|
group[collection].append(value)
|
|
|
|
|
|
def _put_intervals(records: list[dict[str, Any]]) -> dict[str, Any]:
|
|
intervals: list[dict[str, Any]] = []
|
|
boundaries: list[tuple[int, int]] = []
|
|
for record in records:
|
|
elapsed = _number(record.get("elapsed_ms"))
|
|
if elapsed is None or elapsed < 0:
|
|
continue
|
|
try:
|
|
timestamp = datetime.fromisoformat(record["timestamp"].replace("Z", "+00:00"))
|
|
end_us = round(timestamp.timestamp() * 1_000_000)
|
|
except (ValueError, KeyError, OverflowError):
|
|
continue
|
|
start_us = end_us - elapsed * 1000
|
|
interval = dict(record, start_epoch_us=start_us, end_epoch_us=end_us)
|
|
intervals.append(interval)
|
|
if end_us > start_us:
|
|
boundaries.extend(((start_us, 1), (end_us, -1)))
|
|
active = peak = 0
|
|
# End before start on equal timestamps: adjacent calls do not overlap.
|
|
for _, change in sorted(boundaries):
|
|
active += change
|
|
peak = max(peak, active)
|
|
queue_separated = bool(records) and all(_number(record.get("queue_wait_ms")) is not None for record in records)
|
|
return {"metricLabel": _BOS_METRIC if queue_separated else _LEGACY_BOS_METRIC,
|
|
"queueWaitSeparated": queue_separated if records else None,
|
|
"estimatedCompletedCallPeak": max(1, peak) if intervals else None,
|
|
"completedCallCount": len(records), "intervalCount": len(intervals),
|
|
"zeroDurationCount": sum(record.get("elapsed_ms") == 0 for record in intervals),
|
|
"successfulBytes": sum(_number(r.get("bytes")) or 0 for r in records),
|
|
"intervals": intervals,
|
|
"limitations": ["Successful DEBUG completion records only; failed, cancelled and unfinished calls are omitted.",
|
|
"Duration includes source checks and stream verification; legacy records also include permit queueing. This is not socket concurrency.",
|
|
"Text wall-clock timestamps and millisecond durations introduce rounding and clock-adjustment error."]}
|
|
|
|
|
|
def analyze_logs(text: str) -> dict[str, Any]:
|
|
"""Return JSON-serializable diagnostics from existing real captured logs.
|
|
|
|
Counts are event-record counts, not deduplicated server request counts. A
|
|
received HTTP success whose JSON cannot be decoded may also produce a retry
|
|
event. A run with no summary remains incomplete (for example after SIGKILL).
|
|
"""
|
|
events: Counter[str] = Counter()
|
|
levels: Counter[str] = Counter()
|
|
runs: dict[str, dict[str, Any]] = {}
|
|
request_groups: dict[tuple[str, str], dict[str, Any]] = {}
|
|
waits: dict[tuple[str, str], dict[str, Any]] = {}
|
|
risk_lines: dict[str, list[int]] = defaultdict(list)
|
|
missing_run: Counter[str] = Counter()
|
|
puts: list[dict[str, Any]] = []
|
|
startup: list[dict[str, Any]] = []
|
|
state_ready: list[dict[str, Any]] = []
|
|
upload_retry_backoffs: list[dict[str, Any]] = []
|
|
starts: list[dict[str, Any]] = []
|
|
recoveries: list[dict[str, Any]] = []
|
|
total = parsed = 0
|
|
for total, original in enumerate(text.splitlines(), 1):
|
|
line = _ANSI.sub("", original)
|
|
timestamp, level, fields = _record(line)
|
|
for category in _risks(line, fields):
|
|
risk_lines[category].append(total)
|
|
if level:
|
|
levels[level] += 1
|
|
event = fields.get("event")
|
|
if not isinstance(event, str) or not event:
|
|
continue
|
|
parsed += 1
|
|
events[event] += 1
|
|
run_id = str(fields.get("run_id") or "")
|
|
session_id = str(fields.get("session_id") or "")
|
|
run = None
|
|
if run_id:
|
|
run = runs.setdefault(run_id, {
|
|
"run_id": run_id, "mapping_id": "", "session_ids": [],
|
|
"summaryCount": 0, "status": "incomplete", "elapsed_ms": None,
|
|
"phase_wall_ms": {}, "phase_work_ms": {},
|
|
"successful_put_bytes": None, "historical_confirmed_bytes": None,
|
|
"first_timestamp": timestamp, "last_timestamp": timestamp,
|
|
"events": Counter(), "requests": {}, "phase_end_records": [],
|
|
"requestFailedCount": 0, "requestRetryCount": 0, "uploadRetryCount": 0,
|
|
})
|
|
run["last_timestamp"] = timestamp
|
|
run["events"][event] += 1
|
|
if fields.get("mapping_id"):
|
|
run["mapping_id"] = str(fields["mapping_id"])
|
|
if session_id and session_id not in run["session_ids"]:
|
|
run["session_ids"].append(session_id)
|
|
if event == "sync_summary":
|
|
run["summaryCount"] += 1
|
|
run["status"] = fields.get("outcome", "unknown")
|
|
for field in _SAFE_SUMMARY:
|
|
if field in fields:
|
|
run[field] = fields[field]
|
|
elif event == "phase_end":
|
|
run["phase_end_records"].append({
|
|
"phase": fields.get("phase"), "elapsed_ms": fields.get("elapsed_ms"),
|
|
"outcome": fields.get("outcome"), "timestamp": timestamp,
|
|
})
|
|
elif event not in ("daemon_start", "daemon_state_ready"):
|
|
missing_run[event] += 1
|
|
if event == "file_sync_retry" and run is not None:
|
|
run["uploadRetryCount"] += 1
|
|
if event == "upload_retry_backoff":
|
|
upload_retry_backoffs.append({
|
|
"timestamp": timestamp, "run_id": run_id, "session_id": session_id,
|
|
**{key: fields[key] for key in ("item_id", "part_number", "attempt", "next_attempt", "backoff_ms")
|
|
if key in fields},
|
|
})
|
|
if event in ("request_completed", "request_failed", "request_retry"):
|
|
kind = str(fields.get("endpoint_kind") or "unknown")
|
|
endpoint = _endpoint(fields.get("endpoint"))
|
|
key = (kind, endpoint)
|
|
group = request_groups.setdefault(key, _new_request(kind, endpoint))
|
|
_request_update(group, event, fields)
|
|
if run is not None:
|
|
run_group = run["requests"].setdefault(key, _new_request(kind, endpoint))
|
|
_request_update(run_group, event, fields)
|
|
if event == "request_failed":
|
|
run["requestFailedCount"] += 1
|
|
elif event == "request_retry":
|
|
run["requestRetryCount"] += 1
|
|
if event == "request_completed" and kind == "bos_put":
|
|
puts.append({"timestamp": timestamp, "run_id": run_id,
|
|
"session_id": session_id, "item_id": fields.get("item_id"),
|
|
"part_number": fields.get("part_number"), "offset": fields.get("offset"),
|
|
"bytes": fields.get("bytes"), "elapsed_ms": fields.get("elapsed_ms"),
|
|
"status": fields.get("status"), "queue_wait_ms": fields.get("queue_wait_ms")})
|
|
if event == "commit_wait":
|
|
wait = waits.setdefault((run_id, session_id), {
|
|
"run_id": run_id, "session_id": session_id, "elapsed_ms": [],
|
|
"timestamps": [], "statuses": [], "request_in_flight": [],
|
|
})
|
|
wait["elapsed_ms"].append(fields.get("elapsed_ms"))
|
|
wait["timestamps"].append(timestamp)
|
|
wait["statuses"].append(fields.get("last_observed_status"))
|
|
wait["request_in_flight"].append(fields.get("request_in_flight"))
|
|
if event == "file_upload_start":
|
|
starts.append({"timestamp": timestamp, "run_id": run_id,
|
|
"session_id": session_id, "item_id": fields.get("item_id"),
|
|
"path": fields.get("path"), "bytes": fields.get("bytes")})
|
|
if event in ("session_recovery", "multipart_recovery"):
|
|
recoveries.append({"event": event, "timestamp": timestamp, "run_id": run_id,
|
|
"session_id": session_id, "item_id": fields.get("item_id"),
|
|
**{key: fields[key] for key in ("completed_parts", "remaining_parts",
|
|
"historical_part_bytes", "prior_status") if key in fields}})
|
|
if event == "daemon_start":
|
|
safe = ("version", "protocol_version", "state_schema", "file_concurrency",
|
|
"put_concurrency", "part_concurrency", "mapping_concurrency", "hash_concurrency",
|
|
"log_level", "log_max_bytes", "log_keep_files")
|
|
startup.append({"timestamp": timestamp, **{key: fields[key] for key in safe if key in fields}})
|
|
if event == "daemon_state_ready":
|
|
state_ready.append({"timestamp": timestamp,
|
|
**{key: fields[key] for key in ("version", "protocol_version", "state_schema")
|
|
if key in fields}})
|
|
for run in runs.values():
|
|
run["requests"] = list(run["requests"].values())
|
|
run["bosPut"] = _put_intervals([put for put in puts if put["run_id"] == run["run_id"]])
|
|
result = {
|
|
"schemaVersion": 1, "lineCount": total, "eventLineCount": parsed,
|
|
"levels": dict(levels), "events": dict(events), "runs": list(runs.values()),
|
|
"requests": list(request_groups.values()), "bosPut": _put_intervals(puts),
|
|
"commitWait": list(waits.values()), "requestFailedCount": events["request_failed"],
|
|
"requestRetryCount": events["request_retry"], "uploadRetryCount": events["file_sync_retry"],
|
|
"uploadRetryBackoffs": upload_retry_backoffs, "fileUploadStartOrder": starts,
|
|
"recoveries": recoveries, "startup": startup, "stateReady": state_ready,
|
|
"missingRunEventCounts": dict(missing_run),
|
|
"secretRisks": [{"category": category, "lineNumbers": lines, "count": len(lines)}
|
|
for category, lines in sorted(risk_lines.items())],
|
|
"requestMetricLabel": "diagnostic_event_records_not_deduplicated_server_requests",
|
|
}
|
|
# Counter and tuple-key maps are converted without exposing source text.
|
|
return json.loads(json.dumps(result, ensure_ascii=False))
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("logs", type=Path, nargs="*", help="Captured text logs; default stdin")
|
|
parser.add_argument("--output", type=Path, help="Write sanitized JSON instead of stdout")
|
|
args = parser.parse_args()
|
|
text = "\n".join(path.read_text(errors="replace") for path in args.logs) if args.logs else sys.stdin.read()
|
|
encoded = json.dumps(analyze_logs(text), ensure_ascii=False, indent=2) + "\n"
|
|
if args.output:
|
|
args.output.write_text(encoded)
|
|
else:
|
|
sys.stdout.write(encoded)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|