droboticflow/scripts/local/query-lab-resource-stage.py
2026-11-13 22:31:44 -08:00

156 lines
9.2 KiB
Python

#!/usr/bin/env python3
"""Rebuild only the owned query25 lab Pod with an exact finite resource profile.
Original objects are saved before mutation. --restore uses that snapshot and
preserves the backing filesystem; neither path writes customer data.
"""
import argparse
import importlib.util
import hashlib
import json
from pathlib import Path
import subprocess
import urllib.request
import time
import query_lab_profiles as profiles
HERE = Path(__file__).resolve().parent
NS = 'droboticflow-query-lab'
NS_UID = '07d647bb-943c-472c-be0a-f6b1380d7105'
NODE = '10.189.16.4'
NODE_UID = 'ca3f254a-8452-4769-8929-6ec303905dbe'
def validate_saved(saved, pilot):
"""Rollback accepts only this lab's fixed storage and reviewed scripts."""
pod = saved['pod']
assert pod['metadata']['name'] == 'node4-query25' and pod['metadata']['namespace'] == NS
spec = pod['spec']
assert spec['nodeName'] == NODE and not any(spec.get(k, False) for k in ('hostNetwork', 'hostPID', 'hostIPC'))
assert len(spec['containers']) == len(spec['initContainers']) == 1
for c in spec['containers'] + spec['initContainers']:
assert c['image'] == pilot.IMAGE and not c.get('envFrom')
assert {e['name'] for e in c['env']} == {'POD_UID', 'NODE_NAME', 'HOME', 'MONGOSH_DISABLE_TELEMETRY'}
assert spec['containers'][0]['command'] == ['bash', '/scripts/query25.sh']
assert spec['initContainers'][0]['command'] == ['bash', '/scripts/init.sh']
volumes = {v['name']: v for v in spec['volumes']}
assert set(volumes) == {'lab', 'cg', 'scripts', 'tmp'}
assert volumes['lab']['hostPath']['path'] == '/var/lib/mongo-cpu-node4-query25-20261113'
assert volumes['cg']['hostPath']['path'] == '/sys/fs/cgroup'
assert volumes['scripts']['configMap']['name'] == 'node4-query25-scripts'
name = pod['metadata'].get('annotations', {}).get('query-lab/profile', 'baseline')
profiles.verify_resources(pod, name)
assert saved['quota']['spec']['hard'] == profiles.quota(name)
expected = pilot.manifests(True, name)[-2]['data']
actual = saved['config']['data']
prior_verifier = dict(expected)
prior_verifier['verify25.sh'] = expected['verify25.sh'].replace('Number(db.serverStatus().wiredTiger.cache["maximum bytes configured"])', 'db.serverStatus().wiredTiger.cache["maximum bytes configured"]')
legacy = name == 'baseline' and hashlib.sha256(json.dumps(actual, sort_keys=True).encode()).hexdigest() == 'e5eebd9dacae29e817d90909fa97dd31e648ef8eaf4aa81f6bcb31ac58ee8eed'
assert actual == expected or actual == prior_verifier or legacy, 'rollback scripts are not an approved profile'
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument('--kubeconfig', required=True)
ap.add_argument('--output', required=True)
ap.add_argument('--profile', choices=profiles.PROFILES, default='baseline')
ap.add_argument('--restore', help='original.json from the first stage; requires owned current UID')
ap.add_argument('--expected-pod-uid', required=True)
ap.add_argument('--execute', action='store_true')
args = ap.parse_args()
out = Path(args.output).resolve()
out.mkdir(parents=True, exist_ok=True)
assert not (out / 'original.json').exists(), 'never overwrite rollback evidence'
k = ['kubectl', '--kubeconfig', args.kubeconfig]
def call(*cmd, data=None, timeout=30):
p = subprocess.run(k + list(cmd), input=data, text=True, capture_output=True, timeout=timeout)
if p.returncode:
raise RuntimeError(p.stderr[:2000])
return p.stdout
def get(*cmd):
return json.loads(call(*cmd, '-o', 'json'))
assert get('get', 'namespace', NS)['metadata']['uid'] == NS_UID
node = get('get', 'node', NODE)
assert node['metadata']['uid'] == NODE_UID
assert args.restore or all(c['status'] == 'False' for c in node['status']['conditions'] if c['type'] in ('MemoryPressure', 'DiskPressure', 'PIDPressure'))
top = call('top', 'node', NODE, '--no-headers').split()
assert args.restore or (float(top[2].rstrip('%')) <= 25 and float(top[4].rstrip('%')) <= 60)
def prod_states():
return {p['metadata']['uid']: [s.get('restartCount', 0) for s in p['status'].get('containerStatuses', [])] for p in get('-n', 'droboticflow-fanyu', 'get', 'pods')['items']}
initial_production = prod_states()
def health():
with urllib.request.urlopen('http://120.48.180.80/api/ready', timeout=5) as r:
assert json.load(r)['ready'] is True
assert prod_states() == initial_production, 'production Pod/restart set changed'
if not args.restore:
health()
(out / 'production-before.json').write_text(json.dumps(initial_production))
pods = get('-n', NS, 'get', 'pods')['items']
assert len(pods) == 1 and pods[0]['metadata']['name'] == 'node4-query25', 'stop/clean owned fixture before rebuild'
pod = pods[0]
assert pod['metadata']['uid'] == args.expected_pod_uid
assert pod['spec']['nodeName'] == NODE
quota = get('-n', NS, 'get', 'resourcequota', 'smoke-budget')
config = get('-n', NS, 'get', 'configmap', 'node4-query25-scripts')
original = {'pod': pod, 'quota': quota, 'config': config}
(out / 'original.json').write_text(json.dumps(original, indent=2))
spec = importlib.util.spec_from_file_location('pilot', HERE / 'node4-query-pilot/pilot.py')
pilot = importlib.util.module_from_spec(spec)
spec.loader.exec_module(pilot)
objects = pilot.manifests(True, args.profile)
desired_pod = objects[-1]
desired_pod['spec']['activeDeadlineSeconds'] = 3600
desired_config, desired_hard = objects[-2]['data'], profiles.quota(args.profile)
if args.restore:
saved = json.loads(Path(args.restore).read_text())
validate_saved(saved, pilot)
desired_pod = {'apiVersion': 'v1', 'kind': 'Pod', 'metadata': {key: saved['pod']['metadata'][key] for key in ('name', 'namespace', 'labels')}, 'spec': saved['pod']['spec']}
if 'annotations' in saved['pod']['metadata']:
desired_pod['metadata']['annotations'] = saved['pod']['metadata']['annotations']
desired_config, desired_hard = saved['config']['data'], saved['quota']['spec']['hard']
(out / 'desired.json').write_text(json.dumps({'pod': desired_pod, 'data': desired_config, 'hard': desired_hard}, indent=2))
(out / 'rollback.txt').write_text('Stop and UID-clean only the owned source fixture/policies/bridge first. Then run query-lab-resource-stage.py --restore ' + str(out / 'original.json') + ' --expected-pod-uid <current lab UID> --execute with a NEW output directory. This restores exact scripts/quota/resources, remounts retained 25Gi data and preserves majority+j. No reformat, forced unmount, global cache drop, or disk resize.\n')
if not args.execute:
print('PLAN_ONLY', out)
return
delete = {'apiVersion': 'v1', 'kind': 'DeleteOptions', 'preconditions': {'uid': args.expected_pod_uid}}
call('delete', '--raw', f'/api/v1/namespaces/{NS}/pods/node4-query25', '-f', '-', data=json.dumps(delete))
call('-n', NS, 'wait', '--for=delete', 'pod/node4-query25', '--timeout=90s', timeout=95)
# Preconditions prevent overwriting another operator's config or quota.
for kind, old, field, value in [('configmap', config, 'data', desired_config), ('resourcequota', quota, 'spec/hard', desired_hard)]:
before = old['data'] if field == 'data' else old['spec']['hard']
fresh = get('-n', NS, 'get', kind, old['metadata']['name'])
assert fresh['metadata']['uid'] == old['metadata']['uid']
assert (fresh['data'] if field == 'data' else fresh['spec']['hard']) == before
old = fresh # Quota status resourceVersion changes as the old Pod exits.
patch = [{'op': 'test', 'path': '/metadata/resourceVersion', 'value': old['metadata']['resourceVersion']}, {'op': 'test', 'path': '/' + field, 'value': before}, {'op': 'replace', 'path': '/' + field, 'value': value}]
call('-n', NS, 'patch', kind, old['metadata']['name'], '--type=json', '-p', json.dumps(patch))
call('create', '-f', '-', data=json.dumps(desired_pod))
current = get('-n', NS, 'get', 'pod', 'node4-query25')
(out / 'created.json').write_text(json.dumps({'uid': current['metadata']['uid'], 'profile': desired_pod['metadata'].get('annotations', {}).get('query-lab/profile', 'baseline'), 'restored': bool(args.restore)}))
started = time.monotonic()
while True:
current = get('-n', NS, 'get', 'pod', 'node4-query25')
assert current['metadata']['uid'] == json.loads((out / 'created.json').read_text())['uid']
if not args.restore:
health()
if any(c['type'] == 'Ready' and c['status'] == 'True' for c in current['status'].get('conditions', [])):
break
assert current['status']['phase'] not in ('Failed', 'Succeeded'), 'lab startup failed'
assert time.monotonic() - started < 180, 'lab readiness deadline exceeded'
time.sleep(5)
verification = call('-n', NS, 'exec', 'node4-query25', '-c', 'probe', '--', 'bash', '/scripts/verify25.sh', timeout=90)
(out / 'verification.log').write_text(verification)
(out / 'after.json').write_text(json.dumps(get('-n', NS, 'get', 'pod', 'node4-query25'), indent=2))
print('VERIFIED', current['metadata']['uid'], verification.strip())
if __name__ == '__main__':
main()