forked from longtao.wu/droboticflow
147 lines
9.4 KiB
Python
147 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only shadow stop signal from complete API slow/error log windows.
|
|
|
|
No customer requests, bodies, task IDs, log lines or credentials are persisted.
|
|
This measures HTTP slow/error tails, not total throughput, full p95, or async completion.
|
|
Run only with a reviewed hash-bound identity/policy file; it never mutates Kubernetes.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
OPS = ('save', 'slice', 'submit', 'review', 'acceptance')
|
|
TRACE = re.compile(r'\brequest_trace id=([a-f0-9]{32}) kind=(slow|failed) method=(PUT|POST) path=(/api/\S+) status=(\d{3}) elapsed_ms=(\d+) observations=')
|
|
|
|
def operations(method, route):
|
|
prefix = '/api/plugins/annotation-qa/tasks/{id}/'
|
|
legacy = '/api/annotation-tasks/{id}/'
|
|
if method == 'PUT' and (route == prefix + 'drafts' or route == '/api/human-items/{id}/review-draft'):
|
|
return ('save',)
|
|
if method != 'POST': return ()
|
|
if route in [prefix + s for s in ('slice-confirmations', 'slice-edit-applications', 'slice-edit-restorations', 'slice-dispositions', 'slice-reset', 'slice-revisions')] + [legacy + 'slice-revisions']:
|
|
return ('slice',)
|
|
if route in (prefix + 'combined-submissions', legacy + 'combined-submissions', prefix + 'annotation-confirmations', prefix + 'revisions', legacy + 'revisions', '/api/human-items/{id}/submissions'):
|
|
return ('submit',)
|
|
if route in (prefix + 'reviews', legacy + 'reviews'):
|
|
return ('review',)
|
|
# Shared human decision route does not identify role/stage. Both categories
|
|
# receive the same event, enforcing the stricter of their reviewed budgets.
|
|
if route == '/api/human-items/{id}/decisions': return ('review', 'acceptance')
|
|
if route == '/api/delivery-packages/{id}/decisions': return ('acceptance',)
|
|
return ()
|
|
|
|
def summarize(logs, limits, start, end, max_lines, received_at=None):
|
|
if set(limits) != set(OPS) or any(type(v) is not int or not 1000 <= v <= 20000 for v in limits.values()):
|
|
raise ValueError('invalid fixed HTTP tail limits')
|
|
received_at = received_at or dt.datetime.now(dt.timezone.utc)
|
|
result = {op: dict(slowOrFailed=0, serverErrors=0, maxObservedMs=0, aboveLimit=0) for op in OPS}
|
|
seen = set()
|
|
for text in logs:
|
|
lines = text.splitlines()
|
|
if len(lines) >= max_lines: raise ValueError('log window truncated')
|
|
for line in lines:
|
|
# kubectl --timestamps prepends server-generated RFC3339Nano.
|
|
stamp, _, body = line.partition(' ')
|
|
try: at = dt.datetime.fromisoformat(stamp.replace('Z', '+00:00'))
|
|
except ValueError: raise ValueError('log clock unavailable') from None
|
|
if at > received_at + dt.timedelta(seconds=2): raise ValueError('log clock in future')
|
|
if at < start or at > end: continue
|
|
m = TRACE.search(body)
|
|
if not m:
|
|
if 'request_trace ' in body and 'kind=diagnostic' in body:
|
|
status = re.search(r' status=(\d{3})', body)
|
|
elapsed = re.search(r' elapsed_ms=(\d+)', body)
|
|
if not status or not elapsed or int(status[1]) >= 500 or int(elapsed[1]) > min(limits.values()):
|
|
raise ValueError('unclassified diagnostic request failure or latency')
|
|
if 'request_trace ' in body and ('kind=slow' in body or 'kind=failed' in body):
|
|
# Unknown trace formats cannot silently claim monitored coverage.
|
|
if 'method=PUT ' in body or 'method=POST ' in body: raise ValueError('write trace format changed')
|
|
if any(token in body.lower() for token in ('panic:', 'fatal error:', 'write conflict retry exhausted')):
|
|
raise ValueError('API fatal/storage failure')
|
|
continue
|
|
request_id, _, method, route, status, elapsed = m.groups()
|
|
if request_id in seen: continue
|
|
seen.add(request_id)
|
|
for op in operations(method, route):
|
|
row = result[op]; ms = int(elapsed)
|
|
row['slowOrFailed'] += 1; row['serverErrors'] += int(int(status) >= 500)
|
|
row['maxObservedMs'] = max(row['maxObservedMs'], ms)
|
|
row['aboveLimit'] += int(ms > limits[op])
|
|
return result
|
|
|
|
def call(k, *args):
|
|
p = subprocess.run(k + list(args), capture_output=True, text=True, timeout=20)
|
|
if p.returncode: raise RuntimeError('bounded read-only metrics unavailable')
|
|
return p.stdout
|
|
|
|
def collect(k, binding, seconds=30, max_lines=4000, caller=None, log_workers=1):
|
|
caller = caller or call
|
|
if not 10 <= seconds <= 30: raise ValueError('invalid observation window')
|
|
if not 1 <= log_workers <= 4: raise ValueError('invalid log concurrency')
|
|
now = dt.datetime.now(dt.timezone.utc); start = now - dt.timedelta(seconds=seconds)
|
|
ns = binding['sourceNamespace']
|
|
namespace = json.loads(caller(k, 'get', 'namespace', ns, '-o', 'json'))
|
|
if namespace['metadata']['uid'] != binding['sourceNamespaceUID']: raise ValueError('namespace replaced')
|
|
d = json.loads(caller(k, '-n', ns, 'get', 'deployment', 'droboticflow-api', '-o', 'json'))
|
|
if d['metadata']['uid'] != binding['apiDeploymentUID']: raise ValueError('API replaced')
|
|
c = d['spec']['template']['spec']['containers'][0]
|
|
if c['image'] != binding['apiImage'] or d.get('status', {}).get('updatedReplicas') != binding['apiReplicas'] or d.get('status', {}).get('availableReplicas') != binding['apiReplicas']:
|
|
raise ValueError('API identity or coverage changed')
|
|
pods = json.loads(caller(k, '-n', ns, 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
|
active = [p for p in pods if p['status'].get('phase') not in ('Failed', 'Succeeded')]
|
|
if {p['metadata']['uid'] for p in active} != set(binding['apiPodUIDs']): raise ValueError('API pod set changed')
|
|
for p in active:
|
|
statuses = p['status'].get('containerStatuses', [])
|
|
if (p['metadata'].get('deletionTimestamp') or p['status'].get('phase') != 'Running'
|
|
or not statuses or not all(s.get('ready') and s.get('restartCount', 0) == binding['podRestarts'][p['metadata']['uid']] for s in statuses)):
|
|
raise ValueError('API pod health changed')
|
|
def read_log(p):
|
|
return caller(k, '-n', ns, 'logs', p['metadata']['name'], '-c', c['name'], '--timestamps=true', '--since-time=' + start.isoformat(), '--tail=' + str(max_lines))
|
|
with ThreadPoolExecutor(max_workers=log_workers) as pool:
|
|
logs = list(pool.map(read_log, active))
|
|
rows = summarize(logs, binding['tailLimitsMs'], start, now, max_lines)
|
|
# Mark the start of collection, not publication, so slow reads cannot refresh
|
|
# evidence which is already stale. Normal requests below1s remain uncounted.
|
|
return dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance',
|
|
unexpectedErrors=any(r['serverErrors'] for r in rows.values()), latencyRegressed=any(r['aboveLimit'] for r in rows.values()),
|
|
windowSeconds=seconds, monitoredPods=len(active), metrics=rows, tailLimitsMs=binding['tailLimitsMs'],
|
|
metricKind='http-slow-error-tail', fullP95Available=False, normalRequestCountsAvailable=False,
|
|
asyncCompletionVerified=False, idleOrFastOnly=not any(r['slowOrFailed'] for r in rows.values()))
|
|
|
|
def atomic_write(path, value):
|
|
path = Path(path); temporary = path.with_name(path.name + '.tmp')
|
|
with open(temporary, 'w', opener=lambda name, flags: os.open(name, flags, 0o600)) as f:
|
|
json.dump(value, f, indent=2); f.write('\n'); f.flush(); os.fsync(f.fileno())
|
|
os.replace(temporary, path)
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True); p.add_argument('--bindings-sha256', required=True)
|
|
p.add_argument('--output', required=True); p.add_argument('--once', action='store_true')
|
|
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
|
|
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
|
|
binding = json.loads(raw); k = ['kubectl', '--kubeconfig', args.kubeconfig]
|
|
while True:
|
|
try: value = collect(k, binding)
|
|
except Exception as error:
|
|
reasons = {'invalid fixed HTTP tail limits', 'log window truncated', 'log clock unavailable', 'log clock in future', 'write trace format changed', 'unclassified diagnostic request failure or latency', 'API fatal/storage failure', 'namespace replaced', 'API replaced', 'API identity or coverage changed', 'API pod set changed', 'API pod health changed'}
|
|
reason = str(error) if type(error) is ValueError and str(error) in reasons else 'read-only collection unavailable'
|
|
atomic_write(args.output, dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=True, latencyRegressed=True, available=False, failureReason=reason))
|
|
raise RuntimeError('work-health collection stopped; no raw logs persisted') from None
|
|
value['bindingSHA256'] = args.bindings_sha256
|
|
atomic_write(args.output, value)
|
|
if args.once: return
|
|
time.sleep(10)
|
|
|
|
if __name__ == '__main__':
|
|
try: main()
|
|
except KeyboardInterrupt: raise SystemExit(0)
|
|
except Exception: raise SystemExit('work-health unavailable; inspect private aggregate evidence')
|