📊 Bind shadow safety to live customer operation traces
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions

This commit is contained in:
d-robotics 2026-11-14 08:03:43 -08:00
parent f73665cfa0
commit ae9e33a3bc
5 changed files with 272 additions and 4 deletions

View File

@ -13,6 +13,7 @@ import datetime as dt
import hashlib
import json
import os
import re
from pathlib import Path
import socket
import subprocess
@ -48,8 +49,10 @@ def full_io_pressure(text):
raise ValueError('protected IO evidence missing')
def business_health(path, now=None):
def business_health(path, now=None, binding_sha256=None):
feed = json.loads(Path(path).read_text())
if binding_sha256 is not None and (feed.get('bindingSHA256') != binding_sha256 or feed.get('metricKind') != 'http-slow-error-tail'):
raise ValueError('business-health binding changed')
at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00'))
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance'
@ -220,7 +223,7 @@ class Observer:
io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path)))
# A trusted business-health feed is required; readiness/log absence is
# not evidence that saves/submits meet their latency/error baseline.
work_errors = business_health(self.args.work_health)
work_errors = business_health(self.args.work_health, binding_sha256=self.b.get('workHealthBindingSHA256'))
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
for p in pods:
if p['metadata'].get('deletionTimestamp'): continue
@ -287,7 +290,10 @@ def main():
p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true')
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
b = json.loads(raw); check_bindings(b); Observer(args, b).run()
b = json.loads(raw); check_bindings(b)
if not re.fullmatch(r'[a-f0-9]{64}', b.get('workHealthBindingSHA256', '')):
raise ValueError('reviewed business-health binding required')
Observer(args, b).run()
if __name__ == '__main__':

View File

@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Read-only shadow stop signal from complete API slow/error log windows.
No customer requests, bodies, task IDs, log lines or credentials are persisted.
This measures HTTP slow/error tails, not total throughput, full p95, or async completion.
Run only with a reviewed hash-bound identity/policy file; it never mutates Kubernetes.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import time
OPS = ('save', 'slice', 'submit', 'review', 'acceptance')
TRACE = re.compile(r'\brequest_trace id=([a-f0-9]{32}) kind=(slow|failed) method=(PUT|POST) path=(/api/\S+) status=(\d{3}) elapsed_ms=(\d+) observations=')
def operations(method, route):
prefix = '/api/plugins/annotation-qa/tasks/{id}/'
legacy = '/api/annotation-tasks/{id}/'
if method == 'PUT' and (route == prefix + 'drafts' or route == '/api/human-items/{id}/review-draft'):
return ('save',)
if method != 'POST': return ()
if route in [prefix + s for s in ('slice-confirmations', 'slice-edit-applications', 'slice-edit-restorations', 'slice-dispositions', 'slice-reset', 'slice-revisions')] + [legacy + 'slice-revisions']:
return ('slice',)
if route in (prefix + 'combined-submissions', legacy + 'combined-submissions', prefix + 'annotation-confirmations', prefix + 'revisions', legacy + 'revisions', '/api/human-items/{id}/submissions'):
return ('submit',)
if route in (prefix + 'reviews', legacy + 'reviews'):
return ('review',)
# Shared human decision route does not identify role/stage. Both categories
# receive the same event, enforcing the stricter of their reviewed budgets.
if route == '/api/human-items/{id}/decisions': return ('review', 'acceptance')
if route == '/api/delivery-packages/{id}/decisions': return ('acceptance',)
return ()
def summarize(logs, limits, start, end, max_lines, received_at=None):
if set(limits) != set(OPS) or any(type(v) is not int or not 1000 <= v <= 20000 for v in limits.values()):
raise ValueError('invalid fixed HTTP tail limits')
received_at = received_at or dt.datetime.now(dt.timezone.utc)
result = {op: dict(slowOrFailed=0, serverErrors=0, maxObservedMs=0, aboveLimit=0) for op in OPS}
seen = set()
for text in logs:
lines = text.splitlines()
if len(lines) >= max_lines: raise ValueError('log window truncated')
for line in lines:
# kubectl --timestamps prepends server-generated RFC3339Nano.
stamp, _, body = line.partition(' ')
try: at = dt.datetime.fromisoformat(stamp.replace('Z', '+00:00'))
except ValueError: raise ValueError('log clock unavailable') from None
if at > received_at + dt.timedelta(seconds=2): raise ValueError('log clock in future')
if at < start or at > end: continue
m = TRACE.search(body)
if not m:
if 'request_trace ' in body and 'kind=diagnostic' in body:
status = re.search(r' status=(\d{3})', body)
elapsed = re.search(r' elapsed_ms=(\d+)', body)
if not status or not elapsed or int(status[1]) >= 500 or int(elapsed[1]) > min(limits.values()):
raise ValueError('unclassified diagnostic request failure or latency')
if 'request_trace ' in body and ('kind=slow' in body or 'kind=failed' in body):
# Unknown trace formats cannot silently claim monitored coverage.
if 'method=PUT ' in body or 'method=POST ' in body: raise ValueError('write trace format changed')
if any(token in body.lower() for token in ('panic:', 'fatal error:', 'write conflict retry exhausted')):
raise ValueError('API fatal/storage failure')
continue
request_id, _, method, route, status, elapsed = m.groups()
if request_id in seen: continue
seen.add(request_id)
for op in operations(method, route):
row = result[op]; ms = int(elapsed)
row['slowOrFailed'] += 1; row['serverErrors'] += int(int(status) >= 500)
row['maxObservedMs'] = max(row['maxObservedMs'], ms)
row['aboveLimit'] += int(ms > limits[op])
return result
def call(k, *args):
p = subprocess.run(k + list(args), capture_output=True, text=True, timeout=20)
if p.returncode: raise RuntimeError('bounded read-only metrics unavailable')
return p.stdout
def collect(k, binding, seconds=30, max_lines=4000):
if not 10 <= seconds <= 30: raise ValueError('invalid observation window')
now = dt.datetime.now(dt.timezone.utc); start = now - dt.timedelta(seconds=seconds)
ns = binding['sourceNamespace']
namespace = json.loads(call(k, 'get', 'namespace', ns, '-o', 'json'))
if namespace['metadata']['uid'] != binding['sourceNamespaceUID']: raise ValueError('namespace replaced')
d = json.loads(call(k, '-n', ns, 'get', 'deployment', 'droboticflow-api', '-o', 'json'))
if d['metadata']['uid'] != binding['apiDeploymentUID']: raise ValueError('API replaced')
c = d['spec']['template']['spec']['containers'][0]
if c['image'] != binding['apiImage'] or d.get('status', {}).get('updatedReplicas') != binding['apiReplicas'] or d.get('status', {}).get('availableReplicas') != binding['apiReplicas']:
raise ValueError('API identity or coverage changed')
pods = json.loads(call(k, '-n', ns, 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
active = [p for p in pods if p['status'].get('phase') not in ('Failed', 'Succeeded')]
if {p['metadata']['uid'] for p in active} != set(binding['apiPodUIDs']): raise ValueError('API pod set changed')
logs = []
for p in active:
statuses = p['status'].get('containerStatuses', [])
if (p['metadata'].get('deletionTimestamp') or p['status'].get('phase') != 'Running'
or not statuses or not all(s.get('ready') and s.get('restartCount', 0) == binding['podRestarts'][p['metadata']['uid']] for s in statuses)):
raise ValueError('API pod health changed')
logs.append(call(k, '-n', ns, 'logs', p['metadata']['name'], '-c', c['name'], '--timestamps=true', '--since-time=' + start.isoformat(), '--tail=' + str(max_lines)))
rows = summarize(logs, binding['tailLimitsMs'], start, now, max_lines)
# Mark the start of collection, not publication, so slow reads cannot refresh
# evidence which is already stale. Normal requests below1s remain uncounted.
return dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance',
unexpectedErrors=any(r['serverErrors'] for r in rows.values()), latencyRegressed=any(r['aboveLimit'] for r in rows.values()),
windowSeconds=seconds, monitoredPods=len(active), metrics=rows, tailLimitsMs=binding['tailLimitsMs'],
metricKind='http-slow-error-tail', fullP95Available=False, normalRequestCountsAvailable=False,
asyncCompletionVerified=False, idleOrFastOnly=not any(r['slowOrFailed'] for r in rows.values()))
def atomic_write(path, value):
path = Path(path); temporary = path.with_name(path.name + '.tmp')
with open(temporary, 'w', opener=lambda name, flags: os.open(name, flags, 0o600)) as f:
json.dump(value, f, indent=2); f.write('\n'); f.flush(); os.fsync(f.fileno())
os.replace(temporary, path)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True); p.add_argument('--bindings-sha256', required=True)
p.add_argument('--output', required=True); p.add_argument('--once', action='store_true')
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
binding = json.loads(raw); k = ['kubectl', '--kubeconfig', args.kubeconfig]
while True:
try: value = collect(k, binding)
except Exception as error:
reasons = {'invalid fixed HTTP tail limits', 'log window truncated', 'log clock unavailable', 'log clock in future', 'write trace format changed', 'unclassified diagnostic request failure or latency', 'API fatal/storage failure', 'namespace replaced', 'API replaced', 'API identity or coverage changed', 'API pod set changed', 'API pod health changed'}
reason = str(error) if type(error) is ValueError and str(error) in reasons else 'read-only collection unavailable'
atomic_write(args.output, dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=True, latencyRegressed=True, available=False, failureReason=reason))
raise RuntimeError('work-health collection stopped; no raw logs persisted') from None
value['bindingSHA256'] = args.bindings_sha256
atomic_write(args.output, value)
if args.once: return
time.sleep(10)
if __name__ == '__main__':
try: main()
except KeyboardInterrupt: raise SystemExit(0)
except Exception: raise SystemExit('work-health unavailable; inspect private aggregate evidence')

View File

@ -56,6 +56,16 @@ class ObserverTests(unittest.TestCase):
feed['at'] = now.isoformat(); feed['coverage'] = 'readiness-only'
path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now)
def test_business_health_exact_binding_required(self):
now = dt.datetime.now(dt.timezone.utc)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / 'health.json'
feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a'*64, metricKind='http-slow-error-tail')
path.write_text(json.dumps(feed))
self.assertFalse(m.business_health(path, now, 'a'*64))
with self.assertRaises(ValueError): m.business_health(path, now, 'b'*64)
feed['metricKind'] = 'manual-constant'; path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now, 'a'*64)
def test_unknown_io_is_not_zero(self):
with self.assertRaises(ValueError): m.full_io_pressure('no evidence')
self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25)

View File

@ -0,0 +1,81 @@
import datetime as dt
import importlib.util
import json
from pathlib import Path
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('work_health', Path(__file__).with_name('query-shadow-work-health.py'))
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
class WorkHealthTests(unittest.TestCase):
def setUp(self):
self.end = dt.datetime.now(dt.timezone.utc); self.start = self.end - dt.timedelta(seconds=30)
self.limits = dict(save=5000,slice=10000,submit=15000,review=5000,acceptance=15000)
def line(self, route, ms=2000, status=200, method='POST', request_id='a'*32):
return (self.end - dt.timedelta(seconds=1)).isoformat() + ' 2026/11/14 request_trace id=' + request_id + ' kind=slow method=' + method + ' path=' + route + ' status=' + str(status) + ' elapsed_ms=' + str(ms) + ' observations={"sensitive":"must never survive"}'
def summarize(self, *logs):
return m.summarize(logs,self.limits,self.start,self.end,4000)
def test_five_flows_legacy_and_shared_decision(self):
routes=[('PUT','/api/plugins/annotation-qa/tasks/{id}/drafts',('save',)),('POST','/api/annotation-tasks/{id}/slice-revisions',('slice',)),('POST','/api/annotation-tasks/{id}/combined-submissions',('submit',)),('POST','/api/plugins/annotation-qa/tasks/{id}/reviews',('review',)),('POST','/api/human-items/{id}/decisions',('review','acceptance')),('POST','/api/delivery-packages/{id}/decisions',('acceptance',))]
for method,route,ops in routes:
self.assertEqual(m.operations(method,route),ops)
self.assertEqual(m.operations('GET','/api/human-items/{id}/decisions'),())
self.assertEqual(m.operations('POST','/api/unrecognized/private-id'),())
def test_real_errors_dedup_and_limits_without_payload(self):
row=self.line('/api/human-items/{id}/decisions',6000,500)
value=self.summarize(row,row)
self.assertEqual(value['review'],dict(slowOrFailed=1,serverErrors=1,maxObservedMs=6000,aboveLimit=1))
self.assertEqual(value['acceptance']['aboveLimit'],0)
for secret in ['sensitive','must never survive','{id}','a'*32]: self.assertNotIn(secret,json.dumps(value))
def test_idle_is_not_fabricated_success_count(self):
rows=self.summarize('')
self.assertTrue(all(r['slowOrFailed']==0 for r in rows.values()))
self.assertTrue(all('successCount' not in r for r in rows.values()))
def test_missing_malformed_truncated_and_future_evidence(self):
for text in ['no-time log',self.line('/api/human-items/{id}/decisions').replace('elapsed_ms=2000','elapsed_ms=unknown')]:
with self.assertRaises(ValueError): self.summarize(text)
with self.assertRaises(ValueError): m.summarize(['x\nx'],self.limits,self.start,self.end,2)
with self.assertRaises(ValueError): self.summarize((self.end+dt.timedelta(seconds=3)).isoformat()+' log')
with self.assertRaises(ValueError): self.summarize(self.line('/api/human-items/{id}/decisions',status=500).replace('kind=slow method=POST path=/api/human-items/{id}/decisions','kind=diagnostic'))
def test_old_window_and_unrelated_reads_not_customer_write_tail(self):
old=self.line('/api/human-items/{id}/decisions').split(' ',1)[1]
rows=self.summarize((self.start-dt.timedelta(seconds=1)).isoformat()+' '+old,self.line('/api/dashboard/slow',20000,500,method='GET'))
self.assertTrue(all(r['slowOrFailed']==0 for r in rows.values()))
def test_requests_finishing_during_collection_do_not_look_like_clock_skew(self):
text = self.line('/api/human-items/{id}/decisions')
old_time = text.split(' ',1)[0]
text = text.replace(old_time,(self.end+dt.timedelta(seconds=3)).isoformat(),1)
rows=m.summarize([text],self.limits,self.start,self.end,4000,received_at=self.end+dt.timedelta(seconds=5))
self.assertEqual(rows['review']['slowOrFailed'],0)
def fixture(self):
b=dict(sourceNamespace='authority',sourceNamespaceUID='ns',apiDeploymentUID='api',apiImage='image@sha256:fixed',apiReplicas=1,apiPodUIDs=['pod'],podRestarts={'pod':0},tailLimitsMs=self.limits)
deployment=dict(metadata=dict(uid='api'),spec=dict(template=dict(spec=dict(containers=[dict(name='api',image=b['apiImage'])]))),status=dict(updatedReplicas=1,availableReplicas=1))
pod=dict(metadata=dict(uid='pod',name='api-1'),status=dict(phase='Running',containerStatuses=[dict(ready=True,restartCount=0)]))
return b,[json.dumps(dict(metadata=dict(uid='ns'))),json.dumps(deployment),json.dumps(dict(items=[pod])),'']
def test_collect_all_pods_and_no_false_p95(self):
b,replies=self.fixture()
with patch.object(m,'call',side_effect=replies) as called:
result=m.collect(['kubectl'],b)
self.assertFalse(result['fullP95Available']); self.assertFalse(result['asyncCompletionVerified'])
self.assertTrue(result['idleOrFastOnly']); self.assertEqual(result['monitoredPods'],1)
self.assertIn('--timestamps=true',called.call_args.args)
self.assertNotIn('pod',result)
def test_collect_replaced_identity_and_restart_fail_closed(self):
for which in ('namespace','image','pod','restart'):
b,replies=self.fixture()
if which=='namespace': b['sourceNamespaceUID']='other'
if which=='image': b['apiImage']='other'
if which=='pod': b['apiPodUIDs']=['other']
if which=='restart': b['podRestarts']={'pod':1}
with patch.object(m,'call',side_effect=replies):
with self.assertRaises(ValueError): m.collect(['kubectl'],b)
def test_collection_denied_cannot_be_a_healthy_empty_window(self):
b,_=self.fixture()
with patch.object(m,'call',side_effect=RuntimeError('denied')):
with self.assertRaises(RuntimeError): m.collect(['kubectl'],b)
def test_fixed_thresholds_cannot_be_raised_beyond_cap(self):
self.limits['submit']=20001
with self.assertRaises(ValueError): self.summarize('')
if __name__=='__main__':unittest.main()

View File

@ -1015,10 +1015,40 @@
],
"warnings": ["仅执行离线安全回归;节点写回归属、Pod重建、满盘及容量仍须已授权的独立合成实测,不能以该检查替代。"]
},
{
"name": "shadow-work-health",
"paths": [
"^scripts/local/(query-shadow-(observer|work-health)|test_query_shadow_(observer|work_health))\\.py$"
],
"frontend": [],
"go": {
"./internal/api": [
"^TestDiagnosticTrace"
]
},
"task": [
{
"argv": [
"python3",
"-m",
"unittest",
"discover",
"-s",
"scripts/local",
"-p",
"test_query_shadow*.py",
"-v"
],
"tests": true
}
],
"warnings": [
"HTTP slow/error tails only; no full p95 or async job completion claims. Production collectors are read-only and must bind current API identities."
]
},
{
"name": "independent-query-replica",
"paths": [
"^scripts/local/(query-shadow-observer|test_query_shadow_observer)\\.py$",
"^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$",
"^internal/api/query_replica.*\\.go$",
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$",