droboticflow/scripts/local/query-shadow-work-health.py
2026-11-14 20:59:54 -08:00

147 lines
9.4 KiB
Python

#!/usr/bin/env python3
"""Read-only shadow stop signal from complete API slow/error log windows.
No customer requests, bodies, task IDs, log lines or credentials are persisted.
This measures HTTP slow/error tails, not total throughput, full p95, or async completion.
Run only with a reviewed hash-bound identity/policy file; it never mutates Kubernetes.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import time
from concurrent.futures import ThreadPoolExecutor
OPS = ('save', 'slice', 'submit', 'review', 'acceptance')
TRACE = re.compile(r'\brequest_trace id=([a-f0-9]{32}) kind=(slow|failed) method=(PUT|POST) path=(/api/\S+) status=(\d{3}) elapsed_ms=(\d+) observations=')
def operations(method, route):
prefix = '/api/plugins/annotation-qa/tasks/{id}/'
legacy = '/api/annotation-tasks/{id}/'
if method == 'PUT' and (route == prefix + 'drafts' or route == '/api/human-items/{id}/review-draft'):
return ('save',)
if method != 'POST': return ()
if route in [prefix + s for s in ('slice-confirmations', 'slice-edit-applications', 'slice-edit-restorations', 'slice-dispositions', 'slice-reset', 'slice-revisions')] + [legacy + 'slice-revisions']:
return ('slice',)
if route in (prefix + 'combined-submissions', legacy + 'combined-submissions', prefix + 'annotation-confirmations', prefix + 'revisions', legacy + 'revisions', '/api/human-items/{id}/submissions'):
return ('submit',)
if route in (prefix + 'reviews', legacy + 'reviews'):
return ('review',)
# Shared human decision route does not identify role/stage. Both categories
# receive the same event, enforcing the stricter of their reviewed budgets.
if route == '/api/human-items/{id}/decisions': return ('review', 'acceptance')
if route == '/api/delivery-packages/{id}/decisions': return ('acceptance',)
return ()
def summarize(logs, limits, start, end, max_lines, received_at=None):
if set(limits) != set(OPS) or any(type(v) is not int or not 1000 <= v <= 20000 for v in limits.values()):
raise ValueError('invalid fixed HTTP tail limits')
received_at = received_at or dt.datetime.now(dt.timezone.utc)
result = {op: dict(slowOrFailed=0, serverErrors=0, maxObservedMs=0, aboveLimit=0) for op in OPS}
seen = set()
for text in logs:
lines = text.splitlines()
if len(lines) >= max_lines: raise ValueError('log window truncated')
for line in lines:
# kubectl --timestamps prepends server-generated RFC3339Nano.
stamp, _, body = line.partition(' ')
try: at = dt.datetime.fromisoformat(stamp.replace('Z', '+00:00'))
except ValueError: raise ValueError('log clock unavailable') from None
if at > received_at + dt.timedelta(seconds=2): raise ValueError('log clock in future')
if at < start or at > end: continue
m = TRACE.search(body)
if not m:
if 'request_trace ' in body and 'kind=diagnostic' in body:
status = re.search(r' status=(\d{3})', body)
elapsed = re.search(r' elapsed_ms=(\d+)', body)
if not status or not elapsed or int(status[1]) >= 500 or int(elapsed[1]) > min(limits.values()):
raise ValueError('unclassified diagnostic request failure or latency')
if 'request_trace ' in body and ('kind=slow' in body or 'kind=failed' in body):
# Unknown trace formats cannot silently claim monitored coverage.
if 'method=PUT ' in body or 'method=POST ' in body: raise ValueError('write trace format changed')
if any(token in body.lower() for token in ('panic:', 'fatal error:', 'write conflict retry exhausted')):
raise ValueError('API fatal/storage failure')
continue
request_id, _, method, route, status, elapsed = m.groups()
if request_id in seen: continue
seen.add(request_id)
for op in operations(method, route):
row = result[op]; ms = int(elapsed)
row['slowOrFailed'] += 1; row['serverErrors'] += int(int(status) >= 500)
row['maxObservedMs'] = max(row['maxObservedMs'], ms)
row['aboveLimit'] += int(ms > limits[op])
return result
def call(k, *args):
p = subprocess.run(k + list(args), capture_output=True, text=True, timeout=20)
if p.returncode: raise RuntimeError('bounded read-only metrics unavailable')
return p.stdout
def collect(k, binding, seconds=30, max_lines=4000, caller=None, log_workers=1):
caller = caller or call
if not 10 <= seconds <= 30: raise ValueError('invalid observation window')
if not 1 <= log_workers <= 4: raise ValueError('invalid log concurrency')
now = dt.datetime.now(dt.timezone.utc); start = now - dt.timedelta(seconds=seconds)
ns = binding['sourceNamespace']
namespace = json.loads(caller(k, 'get', 'namespace', ns, '-o', 'json'))
if namespace['metadata']['uid'] != binding['sourceNamespaceUID']: raise ValueError('namespace replaced')
d = json.loads(caller(k, '-n', ns, 'get', 'deployment', 'droboticflow-api', '-o', 'json'))
if d['metadata']['uid'] != binding['apiDeploymentUID']: raise ValueError('API replaced')
c = d['spec']['template']['spec']['containers'][0]
if c['image'] != binding['apiImage'] or d.get('status', {}).get('updatedReplicas') != binding['apiReplicas'] or d.get('status', {}).get('availableReplicas') != binding['apiReplicas']:
raise ValueError('API identity or coverage changed')
pods = json.loads(caller(k, '-n', ns, 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
active = [p for p in pods if p['status'].get('phase') not in ('Failed', 'Succeeded')]
if {p['metadata']['uid'] for p in active} != set(binding['apiPodUIDs']): raise ValueError('API pod set changed')
for p in active:
statuses = p['status'].get('containerStatuses', [])
if (p['metadata'].get('deletionTimestamp') or p['status'].get('phase') != 'Running'
or not statuses or not all(s.get('ready') and s.get('restartCount', 0) == binding['podRestarts'][p['metadata']['uid']] for s in statuses)):
raise ValueError('API pod health changed')
def read_log(p):
return caller(k, '-n', ns, 'logs', p['metadata']['name'], '-c', c['name'], '--timestamps=true', '--since-time=' + start.isoformat(), '--tail=' + str(max_lines))
with ThreadPoolExecutor(max_workers=log_workers) as pool:
logs = list(pool.map(read_log, active))
rows = summarize(logs, binding['tailLimitsMs'], start, now, max_lines)
# Mark the start of collection, not publication, so slow reads cannot refresh
# evidence which is already stale. Normal requests below1s remain uncounted.
return dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance',
unexpectedErrors=any(r['serverErrors'] for r in rows.values()), latencyRegressed=any(r['aboveLimit'] for r in rows.values()),
windowSeconds=seconds, monitoredPods=len(active), metrics=rows, tailLimitsMs=binding['tailLimitsMs'],
metricKind='http-slow-error-tail', fullP95Available=False, normalRequestCountsAvailable=False,
asyncCompletionVerified=False, idleOrFastOnly=not any(r['slowOrFailed'] for r in rows.values()))
def atomic_write(path, value):
path = Path(path); temporary = path.with_name(path.name + '.tmp')
with open(temporary, 'w', opener=lambda name, flags: os.open(name, flags, 0o600)) as f:
json.dump(value, f, indent=2); f.write('\n'); f.flush(); os.fsync(f.fileno())
os.replace(temporary, path)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True); p.add_argument('--bindings-sha256', required=True)
p.add_argument('--output', required=True); p.add_argument('--once', action='store_true')
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
binding = json.loads(raw); k = ['kubectl', '--kubeconfig', args.kubeconfig]
while True:
try: value = collect(k, binding)
except Exception as error:
reasons = {'invalid fixed HTTP tail limits', 'log window truncated', 'log clock unavailable', 'log clock in future', 'write trace format changed', 'unclassified diagnostic request failure or latency', 'API fatal/storage failure', 'namespace replaced', 'API replaced', 'API identity or coverage changed', 'API pod set changed', 'API pod health changed'}
reason = str(error) if type(error) is ValueError and str(error) in reasons else 'read-only collection unavailable'
atomic_write(args.output, dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=True, latencyRegressed=True, available=False, failureReason=reason))
raise RuntimeError('work-health collection stopped; no raw logs persisted') from None
value['bindingSHA256'] = args.bindings_sha256
atomic_write(args.output, value)
if args.once: return
time.sleep(10)
if __name__ == '__main__':
try: main()
except KeyboardInterrupt: raise SystemExit(0)
except Exception: raise SystemExit('work-health unavailable; inspect private aggregate evidence')