droboticflow/scripts/local/test_query_shadow_work_health.py

82 lines
6.0 KiB
Python

import datetime as dt
import importlib.util
import json
from pathlib import Path
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('work_health', Path(__file__).with_name('query-shadow-work-health.py'))
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
class WorkHealthTests(unittest.TestCase):
def setUp(self):
self.end = dt.datetime.now(dt.timezone.utc); self.start = self.end - dt.timedelta(seconds=30)
self.limits = dict(save=5000,slice=10000,submit=15000,review=5000,acceptance=15000)
def line(self, route, ms=2000, status=200, method='POST', request_id='a'*32):
return (self.end - dt.timedelta(seconds=1)).isoformat() + ' 2026/11/14 request_trace id=' + request_id + ' kind=slow method=' + method + ' path=' + route + ' status=' + str(status) + ' elapsed_ms=' + str(ms) + ' observations={"sensitive":"must never survive"}'
def summarize(self, *logs):
return m.summarize(logs,self.limits,self.start,self.end,4000)
def test_five_flows_legacy_and_shared_decision(self):
routes=[('PUT','/api/plugins/annotation-qa/tasks/{id}/drafts',('save',)),('POST','/api/annotation-tasks/{id}/slice-revisions',('slice',)),('POST','/api/annotation-tasks/{id}/combined-submissions',('submit',)),('POST','/api/plugins/annotation-qa/tasks/{id}/reviews',('review',)),('POST','/api/human-items/{id}/decisions',('review','acceptance')),('POST','/api/delivery-packages/{id}/decisions',('acceptance',))]
for method,route,ops in routes:
self.assertEqual(m.operations(method,route),ops)
self.assertEqual(m.operations('GET','/api/human-items/{id}/decisions'),())
self.assertEqual(m.operations('POST','/api/unrecognized/private-id'),())
def test_real_errors_dedup_and_limits_without_payload(self):
row=self.line('/api/human-items/{id}/decisions',6000,500)
value=self.summarize(row,row)
self.assertEqual(value['review'],dict(slowOrFailed=1,serverErrors=1,maxObservedMs=6000,aboveLimit=1))
self.assertEqual(value['acceptance']['aboveLimit'],0)
for secret in ['sensitive','must never survive','{id}','a'*32]: self.assertNotIn(secret,json.dumps(value))
def test_idle_is_not_fabricated_success_count(self):
rows=self.summarize('')
self.assertTrue(all(r['slowOrFailed']==0 for r in rows.values()))
self.assertTrue(all('successCount' not in r for r in rows.values()))
def test_missing_malformed_truncated_and_future_evidence(self):
for text in ['no-time log',self.line('/api/human-items/{id}/decisions').replace('elapsed_ms=2000','elapsed_ms=unknown')]:
with self.assertRaises(ValueError): self.summarize(text)
with self.assertRaises(ValueError): m.summarize(['x\nx'],self.limits,self.start,self.end,2)
with self.assertRaises(ValueError): self.summarize((self.end+dt.timedelta(seconds=3)).isoformat()+' log')
with self.assertRaises(ValueError): self.summarize(self.line('/api/human-items/{id}/decisions',status=500).replace('kind=slow method=POST path=/api/human-items/{id}/decisions','kind=diagnostic'))
def test_old_window_and_unrelated_reads_not_customer_write_tail(self):
old=self.line('/api/human-items/{id}/decisions').split(' ',1)[1]
rows=self.summarize((self.start-dt.timedelta(seconds=1)).isoformat()+' '+old,self.line('/api/dashboard/slow',20000,500,method='GET'))
self.assertTrue(all(r['slowOrFailed']==0 for r in rows.values()))
def test_requests_finishing_during_collection_do_not_look_like_clock_skew(self):
text = self.line('/api/human-items/{id}/decisions')
old_time = text.split(' ',1)[0]
text = text.replace(old_time,(self.end+dt.timedelta(seconds=3)).isoformat(),1)
rows=m.summarize([text],self.limits,self.start,self.end,4000,received_at=self.end+dt.timedelta(seconds=5))
self.assertEqual(rows['review']['slowOrFailed'],0)
def fixture(self):
b=dict(sourceNamespace='authority',sourceNamespaceUID='ns',apiDeploymentUID='api',apiImage='image@sha256:fixed',apiReplicas=1,apiPodUIDs=['pod'],podRestarts={'pod':0},tailLimitsMs=self.limits)
deployment=dict(metadata=dict(uid='api'),spec=dict(template=dict(spec=dict(containers=[dict(name='api',image=b['apiImage'])]))),status=dict(updatedReplicas=1,availableReplicas=1))
pod=dict(metadata=dict(uid='pod',name='api-1'),status=dict(phase='Running',containerStatuses=[dict(ready=True,restartCount=0)]))
return b,[json.dumps(dict(metadata=dict(uid='ns'))),json.dumps(deployment),json.dumps(dict(items=[pod])),'']
def test_collect_all_pods_and_no_false_p95(self):
b,replies=self.fixture()
with patch.object(m,'call',side_effect=replies) as called:
result=m.collect(['kubectl'],b)
self.assertFalse(result['fullP95Available']); self.assertFalse(result['asyncCompletionVerified'])
self.assertTrue(result['idleOrFastOnly']); self.assertEqual(result['monitoredPods'],1)
self.assertIn('--timestamps=true',called.call_args.args)
self.assertNotIn('pod',result)
def test_collect_replaced_identity_and_restart_fail_closed(self):
for which in ('namespace','image','pod','restart'):
b,replies=self.fixture()
if which=='namespace': b['sourceNamespaceUID']='other'
if which=='image': b['apiImage']='other'
if which=='pod': b['apiPodUIDs']=['other']
if which=='restart': b['podRestarts']={'pod':1}
with patch.object(m,'call',side_effect=replies):
with self.assertRaises(ValueError): m.collect(['kubectl'],b)
def test_collection_denied_cannot_be_a_healthy_empty_window(self):
b,_=self.fixture()
with patch.object(m,'call',side_effect=RuntimeError('denied')):
with self.assertRaises(RuntimeError): m.collect(['kubectl'],b)
def test_fixed_thresholds_cannot_be_raised_beyond_cap(self):
self.limits['submit']=20001
with self.assertRaises(ValueError): self.summarize('')
if __name__=='__main__':unittest.main()