📊 Bind shadow safety to live customer operation traces
This commit is contained in:
parent
f73665cfa0
commit
ae9e33a3bc
@ -13,6 +13,7 @@ import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
@ -48,8 +49,10 @@ def full_io_pressure(text):
|
||||
raise ValueError('protected IO evidence missing')
|
||||
|
||||
|
||||
def business_health(path, now=None):
|
||||
def business_health(path, now=None, binding_sha256=None):
|
||||
feed = json.loads(Path(path).read_text())
|
||||
if binding_sha256 is not None and (feed.get('bindingSHA256') != binding_sha256 or feed.get('metricKind') != 'http-slow-error-tail'):
|
||||
raise ValueError('business-health binding changed')
|
||||
at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00'))
|
||||
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
|
||||
if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance'
|
||||
@ -220,7 +223,7 @@ class Observer:
|
||||
io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path)))
|
||||
# A trusted business-health feed is required; readiness/log absence is
|
||||
# not evidence that saves/submits meet their latency/error baseline.
|
||||
work_errors = business_health(self.args.work_health)
|
||||
work_errors = business_health(self.args.work_health, binding_sha256=self.b.get('workHealthBindingSHA256'))
|
||||
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
||||
for p in pods:
|
||||
if p['metadata'].get('deletionTimestamp'): continue
|
||||
@ -287,7 +290,10 @@ def main():
|
||||
p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true')
|
||||
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
|
||||
b = json.loads(raw); check_bindings(b); Observer(args, b).run()
|
||||
b = json.loads(raw); check_bindings(b)
|
||||
if not re.fullmatch(r'[a-f0-9]{64}', b.get('workHealthBindingSHA256', '')):
|
||||
raise ValueError('reviewed business-health binding required')
|
||||
Observer(args, b).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
141
scripts/local/query-shadow-work-health.py
Normal file
141
scripts/local/query-shadow-work-health.py
Normal file
@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only shadow stop signal from complete API slow/error log windows.
|
||||
|
||||
No customer requests, bodies, task IDs, log lines or credentials are persisted.
|
||||
This measures HTTP slow/error tails, not total throughput, full p95, or async completion.
|
||||
Run only with a reviewed hash-bound identity/policy file; it never mutates Kubernetes.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
OPS = ('save', 'slice', 'submit', 'review', 'acceptance')
|
||||
TRACE = re.compile(r'\brequest_trace id=([a-f0-9]{32}) kind=(slow|failed) method=(PUT|POST) path=(/api/\S+) status=(\d{3}) elapsed_ms=(\d+) observations=')
|
||||
|
||||
def operations(method, route):
|
||||
prefix = '/api/plugins/annotation-qa/tasks/{id}/'
|
||||
legacy = '/api/annotation-tasks/{id}/'
|
||||
if method == 'PUT' and (route == prefix + 'drafts' or route == '/api/human-items/{id}/review-draft'):
|
||||
return ('save',)
|
||||
if method != 'POST': return ()
|
||||
if route in [prefix + s for s in ('slice-confirmations', 'slice-edit-applications', 'slice-edit-restorations', 'slice-dispositions', 'slice-reset', 'slice-revisions')] + [legacy + 'slice-revisions']:
|
||||
return ('slice',)
|
||||
if route in (prefix + 'combined-submissions', legacy + 'combined-submissions', prefix + 'annotation-confirmations', prefix + 'revisions', legacy + 'revisions', '/api/human-items/{id}/submissions'):
|
||||
return ('submit',)
|
||||
if route in (prefix + 'reviews', legacy + 'reviews'):
|
||||
return ('review',)
|
||||
# Shared human decision route does not identify role/stage. Both categories
|
||||
# receive the same event, enforcing the stricter of their reviewed budgets.
|
||||
if route == '/api/human-items/{id}/decisions': return ('review', 'acceptance')
|
||||
if route == '/api/delivery-packages/{id}/decisions': return ('acceptance',)
|
||||
return ()
|
||||
|
||||
def summarize(logs, limits, start, end, max_lines, received_at=None):
|
||||
if set(limits) != set(OPS) or any(type(v) is not int or not 1000 <= v <= 20000 for v in limits.values()):
|
||||
raise ValueError('invalid fixed HTTP tail limits')
|
||||
received_at = received_at or dt.datetime.now(dt.timezone.utc)
|
||||
result = {op: dict(slowOrFailed=0, serverErrors=0, maxObservedMs=0, aboveLimit=0) for op in OPS}
|
||||
seen = set()
|
||||
for text in logs:
|
||||
lines = text.splitlines()
|
||||
if len(lines) >= max_lines: raise ValueError('log window truncated')
|
||||
for line in lines:
|
||||
# kubectl --timestamps prepends server-generated RFC3339Nano.
|
||||
stamp, _, body = line.partition(' ')
|
||||
try: at = dt.datetime.fromisoformat(stamp.replace('Z', '+00:00'))
|
||||
except ValueError: raise ValueError('log clock unavailable') from None
|
||||
if at > received_at + dt.timedelta(seconds=2): raise ValueError('log clock in future')
|
||||
if at < start or at > end: continue
|
||||
m = TRACE.search(body)
|
||||
if not m:
|
||||
if 'request_trace ' in body and 'kind=diagnostic' in body:
|
||||
status = re.search(r' status=(\d{3})', body)
|
||||
elapsed = re.search(r' elapsed_ms=(\d+)', body)
|
||||
if not status or not elapsed or int(status[1]) >= 500 or int(elapsed[1]) > min(limits.values()):
|
||||
raise ValueError('unclassified diagnostic request failure or latency')
|
||||
if 'request_trace ' in body and ('kind=slow' in body or 'kind=failed' in body):
|
||||
# Unknown trace formats cannot silently claim monitored coverage.
|
||||
if 'method=PUT ' in body or 'method=POST ' in body: raise ValueError('write trace format changed')
|
||||
if any(token in body.lower() for token in ('panic:', 'fatal error:', 'write conflict retry exhausted')):
|
||||
raise ValueError('API fatal/storage failure')
|
||||
continue
|
||||
request_id, _, method, route, status, elapsed = m.groups()
|
||||
if request_id in seen: continue
|
||||
seen.add(request_id)
|
||||
for op in operations(method, route):
|
||||
row = result[op]; ms = int(elapsed)
|
||||
row['slowOrFailed'] += 1; row['serverErrors'] += int(int(status) >= 500)
|
||||
row['maxObservedMs'] = max(row['maxObservedMs'], ms)
|
||||
row['aboveLimit'] += int(ms > limits[op])
|
||||
return result
|
||||
|
||||
def call(k, *args):
|
||||
p = subprocess.run(k + list(args), capture_output=True, text=True, timeout=20)
|
||||
if p.returncode: raise RuntimeError('bounded read-only metrics unavailable')
|
||||
return p.stdout
|
||||
|
||||
def collect(k, binding, seconds=30, max_lines=4000):
|
||||
if not 10 <= seconds <= 30: raise ValueError('invalid observation window')
|
||||
now = dt.datetime.now(dt.timezone.utc); start = now - dt.timedelta(seconds=seconds)
|
||||
ns = binding['sourceNamespace']
|
||||
namespace = json.loads(call(k, 'get', 'namespace', ns, '-o', 'json'))
|
||||
if namespace['metadata']['uid'] != binding['sourceNamespaceUID']: raise ValueError('namespace replaced')
|
||||
d = json.loads(call(k, '-n', ns, 'get', 'deployment', 'droboticflow-api', '-o', 'json'))
|
||||
if d['metadata']['uid'] != binding['apiDeploymentUID']: raise ValueError('API replaced')
|
||||
c = d['spec']['template']['spec']['containers'][0]
|
||||
if c['image'] != binding['apiImage'] or d.get('status', {}).get('updatedReplicas') != binding['apiReplicas'] or d.get('status', {}).get('availableReplicas') != binding['apiReplicas']:
|
||||
raise ValueError('API identity or coverage changed')
|
||||
pods = json.loads(call(k, '-n', ns, 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
||||
active = [p for p in pods if p['status'].get('phase') not in ('Failed', 'Succeeded')]
|
||||
if {p['metadata']['uid'] for p in active} != set(binding['apiPodUIDs']): raise ValueError('API pod set changed')
|
||||
logs = []
|
||||
for p in active:
|
||||
statuses = p['status'].get('containerStatuses', [])
|
||||
if (p['metadata'].get('deletionTimestamp') or p['status'].get('phase') != 'Running'
|
||||
or not statuses or not all(s.get('ready') and s.get('restartCount', 0) == binding['podRestarts'][p['metadata']['uid']] for s in statuses)):
|
||||
raise ValueError('API pod health changed')
|
||||
logs.append(call(k, '-n', ns, 'logs', p['metadata']['name'], '-c', c['name'], '--timestamps=true', '--since-time=' + start.isoformat(), '--tail=' + str(max_lines)))
|
||||
rows = summarize(logs, binding['tailLimitsMs'], start, now, max_lines)
|
||||
# Mark the start of collection, not publication, so slow reads cannot refresh
|
||||
# evidence which is already stale. Normal requests below1s remain uncounted.
|
||||
return dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance',
|
||||
unexpectedErrors=any(r['serverErrors'] for r in rows.values()), latencyRegressed=any(r['aboveLimit'] for r in rows.values()),
|
||||
windowSeconds=seconds, monitoredPods=len(active), metrics=rows, tailLimitsMs=binding['tailLimitsMs'],
|
||||
metricKind='http-slow-error-tail', fullP95Available=False, normalRequestCountsAvailable=False,
|
||||
asyncCompletionVerified=False, idleOrFastOnly=not any(r['slowOrFailed'] for r in rows.values()))
|
||||
|
||||
def atomic_write(path, value):
|
||||
path = Path(path); temporary = path.with_name(path.name + '.tmp')
|
||||
with open(temporary, 'w', opener=lambda name, flags: os.open(name, flags, 0o600)) as f:
|
||||
json.dump(value, f, indent=2); f.write('\n'); f.flush(); os.fsync(f.fileno())
|
||||
os.replace(temporary, path)
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True); p.add_argument('--bindings-sha256', required=True)
|
||||
p.add_argument('--output', required=True); p.add_argument('--once', action='store_true')
|
||||
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
|
||||
binding = json.loads(raw); k = ['kubectl', '--kubeconfig', args.kubeconfig]
|
||||
while True:
|
||||
try: value = collect(k, binding)
|
||||
except Exception as error:
|
||||
reasons = {'invalid fixed HTTP tail limits', 'log window truncated', 'log clock unavailable', 'log clock in future', 'write trace format changed', 'unclassified diagnostic request failure or latency', 'API fatal/storage failure', 'namespace replaced', 'API replaced', 'API identity or coverage changed', 'API pod set changed', 'API pod health changed'}
|
||||
reason = str(error) if type(error) is ValueError and str(error) in reasons else 'read-only collection unavailable'
|
||||
atomic_write(args.output, dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=True, latencyRegressed=True, available=False, failureReason=reason))
|
||||
raise RuntimeError('work-health collection stopped; no raw logs persisted') from None
|
||||
value['bindingSHA256'] = args.bindings_sha256
|
||||
atomic_write(args.output, value)
|
||||
if args.once: return
|
||||
time.sleep(10)
|
||||
|
||||
if __name__ == '__main__':
|
||||
try: main()
|
||||
except KeyboardInterrupt: raise SystemExit(0)
|
||||
except Exception: raise SystemExit('work-health unavailable; inspect private aggregate evidence')
|
||||
@ -56,6 +56,16 @@ class ObserverTests(unittest.TestCase):
|
||||
feed['at'] = now.isoformat(); feed['coverage'] = 'readiness-only'
|
||||
path.write_text(json.dumps(feed))
|
||||
with self.assertRaises(ValueError): m.business_health(path, now)
|
||||
def test_business_health_exact_binding_required(self):
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / 'health.json'
|
||||
feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a'*64, metricKind='http-slow-error-tail')
|
||||
path.write_text(json.dumps(feed))
|
||||
self.assertFalse(m.business_health(path, now, 'a'*64))
|
||||
with self.assertRaises(ValueError): m.business_health(path, now, 'b'*64)
|
||||
feed['metricKind'] = 'manual-constant'; path.write_text(json.dumps(feed))
|
||||
with self.assertRaises(ValueError): m.business_health(path, now, 'a'*64)
|
||||
def test_unknown_io_is_not_zero(self):
|
||||
with self.assertRaises(ValueError): m.full_io_pressure('no evidence')
|
||||
self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25)
|
||||
|
||||
81
scripts/local/test_query_shadow_work_health.py
Normal file
81
scripts/local/test_query_shadow_work_health.py
Normal file
@ -0,0 +1,81 @@
|
||||
import datetime as dt
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location('work_health', Path(__file__).with_name('query-shadow-work-health.py'))
|
||||
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
|
||||
class WorkHealthTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.end = dt.datetime.now(dt.timezone.utc); self.start = self.end - dt.timedelta(seconds=30)
|
||||
self.limits = dict(save=5000,slice=10000,submit=15000,review=5000,acceptance=15000)
|
||||
def line(self, route, ms=2000, status=200, method='POST', request_id='a'*32):
|
||||
return (self.end - dt.timedelta(seconds=1)).isoformat() + ' 2026/11/14 request_trace id=' + request_id + ' kind=slow method=' + method + ' path=' + route + ' status=' + str(status) + ' elapsed_ms=' + str(ms) + ' observations={"sensitive":"must never survive"}'
|
||||
def summarize(self, *logs):
|
||||
return m.summarize(logs,self.limits,self.start,self.end,4000)
|
||||
def test_five_flows_legacy_and_shared_decision(self):
|
||||
routes=[('PUT','/api/plugins/annotation-qa/tasks/{id}/drafts',('save',)),('POST','/api/annotation-tasks/{id}/slice-revisions',('slice',)),('POST','/api/annotation-tasks/{id}/combined-submissions',('submit',)),('POST','/api/plugins/annotation-qa/tasks/{id}/reviews',('review',)),('POST','/api/human-items/{id}/decisions',('review','acceptance')),('POST','/api/delivery-packages/{id}/decisions',('acceptance',))]
|
||||
for method,route,ops in routes:
|
||||
self.assertEqual(m.operations(method,route),ops)
|
||||
self.assertEqual(m.operations('GET','/api/human-items/{id}/decisions'),())
|
||||
self.assertEqual(m.operations('POST','/api/unrecognized/private-id'),())
|
||||
def test_real_errors_dedup_and_limits_without_payload(self):
|
||||
row=self.line('/api/human-items/{id}/decisions',6000,500)
|
||||
value=self.summarize(row,row)
|
||||
self.assertEqual(value['review'],dict(slowOrFailed=1,serverErrors=1,maxObservedMs=6000,aboveLimit=1))
|
||||
self.assertEqual(value['acceptance']['aboveLimit'],0)
|
||||
for secret in ['sensitive','must never survive','{id}','a'*32]: self.assertNotIn(secret,json.dumps(value))
|
||||
def test_idle_is_not_fabricated_success_count(self):
|
||||
rows=self.summarize('')
|
||||
self.assertTrue(all(r['slowOrFailed']==0 for r in rows.values()))
|
||||
self.assertTrue(all('successCount' not in r for r in rows.values()))
|
||||
def test_missing_malformed_truncated_and_future_evidence(self):
|
||||
for text in ['no-time log',self.line('/api/human-items/{id}/decisions').replace('elapsed_ms=2000','elapsed_ms=unknown')]:
|
||||
with self.assertRaises(ValueError): self.summarize(text)
|
||||
with self.assertRaises(ValueError): m.summarize(['x\nx'],self.limits,self.start,self.end,2)
|
||||
with self.assertRaises(ValueError): self.summarize((self.end+dt.timedelta(seconds=3)).isoformat()+' log')
|
||||
with self.assertRaises(ValueError): self.summarize(self.line('/api/human-items/{id}/decisions',status=500).replace('kind=slow method=POST path=/api/human-items/{id}/decisions','kind=diagnostic'))
|
||||
def test_old_window_and_unrelated_reads_not_customer_write_tail(self):
|
||||
old=self.line('/api/human-items/{id}/decisions').split(' ',1)[1]
|
||||
rows=self.summarize((self.start-dt.timedelta(seconds=1)).isoformat()+' '+old,self.line('/api/dashboard/slow',20000,500,method='GET'))
|
||||
self.assertTrue(all(r['slowOrFailed']==0 for r in rows.values()))
|
||||
def test_requests_finishing_during_collection_do_not_look_like_clock_skew(self):
|
||||
text = self.line('/api/human-items/{id}/decisions')
|
||||
old_time = text.split(' ',1)[0]
|
||||
text = text.replace(old_time,(self.end+dt.timedelta(seconds=3)).isoformat(),1)
|
||||
rows=m.summarize([text],self.limits,self.start,self.end,4000,received_at=self.end+dt.timedelta(seconds=5))
|
||||
self.assertEqual(rows['review']['slowOrFailed'],0)
|
||||
def fixture(self):
|
||||
b=dict(sourceNamespace='authority',sourceNamespaceUID='ns',apiDeploymentUID='api',apiImage='image@sha256:fixed',apiReplicas=1,apiPodUIDs=['pod'],podRestarts={'pod':0},tailLimitsMs=self.limits)
|
||||
deployment=dict(metadata=dict(uid='api'),spec=dict(template=dict(spec=dict(containers=[dict(name='api',image=b['apiImage'])]))),status=dict(updatedReplicas=1,availableReplicas=1))
|
||||
pod=dict(metadata=dict(uid='pod',name='api-1'),status=dict(phase='Running',containerStatuses=[dict(ready=True,restartCount=0)]))
|
||||
return b,[json.dumps(dict(metadata=dict(uid='ns'))),json.dumps(deployment),json.dumps(dict(items=[pod])),'']
|
||||
def test_collect_all_pods_and_no_false_p95(self):
|
||||
b,replies=self.fixture()
|
||||
with patch.object(m,'call',side_effect=replies) as called:
|
||||
result=m.collect(['kubectl'],b)
|
||||
self.assertFalse(result['fullP95Available']); self.assertFalse(result['asyncCompletionVerified'])
|
||||
self.assertTrue(result['idleOrFastOnly']); self.assertEqual(result['monitoredPods'],1)
|
||||
self.assertIn('--timestamps=true',called.call_args.args)
|
||||
self.assertNotIn('pod',result)
|
||||
def test_collect_replaced_identity_and_restart_fail_closed(self):
|
||||
for which in ('namespace','image','pod','restart'):
|
||||
b,replies=self.fixture()
|
||||
if which=='namespace': b['sourceNamespaceUID']='other'
|
||||
if which=='image': b['apiImage']='other'
|
||||
if which=='pod': b['apiPodUIDs']=['other']
|
||||
if which=='restart': b['podRestarts']={'pod':1}
|
||||
with patch.object(m,'call',side_effect=replies):
|
||||
with self.assertRaises(ValueError): m.collect(['kubectl'],b)
|
||||
def test_collection_denied_cannot_be_a_healthy_empty_window(self):
|
||||
b,_=self.fixture()
|
||||
with patch.object(m,'call',side_effect=RuntimeError('denied')):
|
||||
with self.assertRaises(RuntimeError): m.collect(['kubectl'],b)
|
||||
def test_fixed_thresholds_cannot_be_raised_beyond_cap(self):
|
||||
self.limits['submit']=20001
|
||||
with self.assertRaises(ValueError): self.summarize('')
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@ -1015,10 +1015,40 @@
|
||||
],
|
||||
"warnings": ["仅执行离线安全回归;节点写回归属、Pod重建、满盘及容量仍须已授权的独立合成实测,不能以该检查替代。"]
|
||||
},
|
||||
{
|
||||
"name": "shadow-work-health",
|
||||
"paths": [
|
||||
"^scripts/local/(query-shadow-(observer|work-health)|test_query_shadow_(observer|work_health))\\.py$"
|
||||
],
|
||||
"frontend": [],
|
||||
"go": {
|
||||
"./internal/api": [
|
||||
"^TestDiagnosticTrace"
|
||||
]
|
||||
},
|
||||
"task": [
|
||||
{
|
||||
"argv": [
|
||||
"python3",
|
||||
"-m",
|
||||
"unittest",
|
||||
"discover",
|
||||
"-s",
|
||||
"scripts/local",
|
||||
"-p",
|
||||
"test_query_shadow*.py",
|
||||
"-v"
|
||||
],
|
||||
"tests": true
|
||||
}
|
||||
],
|
||||
"warnings": [
|
||||
"HTTP slow/error tails only; no full p95 or async job completion claims. Production collectors are read-only and must bind current API identities."
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "independent-query-replica",
|
||||
"paths": [
|
||||
"^scripts/local/(query-shadow-observer|test_query_shadow_observer)\\.py$",
|
||||
"^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$",
|
||||
"^internal/api/query_replica.*\\.go$",
|
||||
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user