forked from longtao.wu/droboticflow
156 lines
9.2 KiB
Python
156 lines
9.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Rebuild only the owned query25 lab Pod with an exact finite resource profile.
|
|
|
|
Original objects are saved before mutation. --restore uses that snapshot and
|
|
preserves the backing filesystem; neither path writes customer data.
|
|
"""
|
|
import argparse
|
|
import importlib.util
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import urllib.request
|
|
import time
|
|
import query_lab_profiles as profiles
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
NS = 'droboticflow-query-lab'
|
|
NS_UID = '07d647bb-943c-472c-be0a-f6b1380d7105'
|
|
NODE = '10.189.16.4'
|
|
NODE_UID = 'ca3f254a-8452-4769-8929-6ec303905dbe'
|
|
|
|
|
|
def validate_saved(saved, pilot):
|
|
"""Rollback accepts only this lab's fixed storage and reviewed scripts."""
|
|
pod = saved['pod']
|
|
assert pod['metadata']['name'] == 'node4-query25' and pod['metadata']['namespace'] == NS
|
|
spec = pod['spec']
|
|
assert spec['nodeName'] == NODE and not any(spec.get(k, False) for k in ('hostNetwork', 'hostPID', 'hostIPC'))
|
|
assert len(spec['containers']) == len(spec['initContainers']) == 1
|
|
for c in spec['containers'] + spec['initContainers']:
|
|
assert c['image'] == pilot.IMAGE and not c.get('envFrom')
|
|
assert {e['name'] for e in c['env']} == {'POD_UID', 'NODE_NAME', 'HOME', 'MONGOSH_DISABLE_TELEMETRY'}
|
|
assert spec['containers'][0]['command'] == ['bash', '/scripts/query25.sh']
|
|
assert spec['initContainers'][0]['command'] == ['bash', '/scripts/init.sh']
|
|
volumes = {v['name']: v for v in spec['volumes']}
|
|
assert set(volumes) == {'lab', 'cg', 'scripts', 'tmp'}
|
|
assert volumes['lab']['hostPath']['path'] == '/var/lib/mongo-cpu-node4-query25-20261113'
|
|
assert volumes['cg']['hostPath']['path'] == '/sys/fs/cgroup'
|
|
assert volumes['scripts']['configMap']['name'] == 'node4-query25-scripts'
|
|
name = pod['metadata'].get('annotations', {}).get('query-lab/profile', 'baseline')
|
|
profiles.verify_resources(pod, name)
|
|
assert saved['quota']['spec']['hard'] == profiles.quota(name)
|
|
expected = pilot.manifests(True, name)[-2]['data']
|
|
actual = saved['config']['data']
|
|
prior_verifier = dict(expected)
|
|
prior_verifier['verify25.sh'] = expected['verify25.sh'].replace('Number(db.serverStatus().wiredTiger.cache["maximum bytes configured"])', 'db.serverStatus().wiredTiger.cache["maximum bytes configured"]')
|
|
legacy = name == 'baseline' and hashlib.sha256(json.dumps(actual, sort_keys=True).encode()).hexdigest() == 'e5eebd9dacae29e817d90909fa97dd31e648ef8eaf4aa81f6bcb31ac58ee8eed'
|
|
assert actual == expected or actual == prior_verifier or legacy, 'rollback scripts are not an approved profile'
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument('--kubeconfig', required=True)
|
|
ap.add_argument('--output', required=True)
|
|
ap.add_argument('--profile', choices=profiles.PROFILES, default='baseline')
|
|
ap.add_argument('--restore', help='original.json from the first stage; requires owned current UID')
|
|
ap.add_argument('--expected-pod-uid', required=True)
|
|
ap.add_argument('--execute', action='store_true')
|
|
args = ap.parse_args()
|
|
out = Path(args.output).resolve()
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
assert not (out / 'original.json').exists(), 'never overwrite rollback evidence'
|
|
k = ['kubectl', '--kubeconfig', args.kubeconfig]
|
|
|
|
def call(*cmd, data=None, timeout=30):
|
|
p = subprocess.run(k + list(cmd), input=data, text=True, capture_output=True, timeout=timeout)
|
|
if p.returncode:
|
|
raise RuntimeError(p.stderr[:2000])
|
|
return p.stdout
|
|
|
|
def get(*cmd):
|
|
return json.loads(call(*cmd, '-o', 'json'))
|
|
|
|
assert get('get', 'namespace', NS)['metadata']['uid'] == NS_UID
|
|
node = get('get', 'node', NODE)
|
|
assert node['metadata']['uid'] == NODE_UID
|
|
assert args.restore or all(c['status'] == 'False' for c in node['status']['conditions'] if c['type'] in ('MemoryPressure', 'DiskPressure', 'PIDPressure'))
|
|
top = call('top', 'node', NODE, '--no-headers').split()
|
|
assert args.restore or (float(top[2].rstrip('%')) <= 25 and float(top[4].rstrip('%')) <= 60)
|
|
def prod_states():
|
|
return {p['metadata']['uid']: [s.get('restartCount', 0) for s in p['status'].get('containerStatuses', [])] for p in get('-n', 'droboticflow-fanyu', 'get', 'pods')['items']}
|
|
|
|
initial_production = prod_states()
|
|
|
|
def health():
|
|
with urllib.request.urlopen('http://120.48.180.80/api/ready', timeout=5) as r:
|
|
assert json.load(r)['ready'] is True
|
|
assert prod_states() == initial_production, 'production Pod/restart set changed'
|
|
|
|
if not args.restore:
|
|
health()
|
|
(out / 'production-before.json').write_text(json.dumps(initial_production))
|
|
pods = get('-n', NS, 'get', 'pods')['items']
|
|
assert len(pods) == 1 and pods[0]['metadata']['name'] == 'node4-query25', 'stop/clean owned fixture before rebuild'
|
|
pod = pods[0]
|
|
assert pod['metadata']['uid'] == args.expected_pod_uid
|
|
assert pod['spec']['nodeName'] == NODE
|
|
quota = get('-n', NS, 'get', 'resourcequota', 'smoke-budget')
|
|
config = get('-n', NS, 'get', 'configmap', 'node4-query25-scripts')
|
|
original = {'pod': pod, 'quota': quota, 'config': config}
|
|
(out / 'original.json').write_text(json.dumps(original, indent=2))
|
|
spec = importlib.util.spec_from_file_location('pilot', HERE / 'node4-query-pilot/pilot.py')
|
|
pilot = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(pilot)
|
|
objects = pilot.manifests(True, args.profile)
|
|
desired_pod = objects[-1]
|
|
desired_pod['spec']['activeDeadlineSeconds'] = 3600
|
|
desired_config, desired_hard = objects[-2]['data'], profiles.quota(args.profile)
|
|
if args.restore:
|
|
saved = json.loads(Path(args.restore).read_text())
|
|
validate_saved(saved, pilot)
|
|
desired_pod = {'apiVersion': 'v1', 'kind': 'Pod', 'metadata': {key: saved['pod']['metadata'][key] for key in ('name', 'namespace', 'labels')}, 'spec': saved['pod']['spec']}
|
|
if 'annotations' in saved['pod']['metadata']:
|
|
desired_pod['metadata']['annotations'] = saved['pod']['metadata']['annotations']
|
|
desired_config, desired_hard = saved['config']['data'], saved['quota']['spec']['hard']
|
|
(out / 'desired.json').write_text(json.dumps({'pod': desired_pod, 'data': desired_config, 'hard': desired_hard}, indent=2))
|
|
(out / 'rollback.txt').write_text('Stop and UID-clean only the owned source fixture/policies/bridge first. Then run query-lab-resource-stage.py --restore ' + str(out / 'original.json') + ' --expected-pod-uid <current lab UID> --execute with a NEW output directory. This restores exact scripts/quota/resources, remounts retained 25Gi data and preserves majority+j. No reformat, forced unmount, global cache drop, or disk resize.\n')
|
|
if not args.execute:
|
|
print('PLAN_ONLY', out)
|
|
return
|
|
delete = {'apiVersion': 'v1', 'kind': 'DeleteOptions', 'preconditions': {'uid': args.expected_pod_uid}}
|
|
call('delete', '--raw', f'/api/v1/namespaces/{NS}/pods/node4-query25', '-f', '-', data=json.dumps(delete))
|
|
call('-n', NS, 'wait', '--for=delete', 'pod/node4-query25', '--timeout=90s', timeout=95)
|
|
# Preconditions prevent overwriting another operator's config or quota.
|
|
for kind, old, field, value in [('configmap', config, 'data', desired_config), ('resourcequota', quota, 'spec/hard', desired_hard)]:
|
|
before = old['data'] if field == 'data' else old['spec']['hard']
|
|
fresh = get('-n', NS, 'get', kind, old['metadata']['name'])
|
|
assert fresh['metadata']['uid'] == old['metadata']['uid']
|
|
assert (fresh['data'] if field == 'data' else fresh['spec']['hard']) == before
|
|
old = fresh # Quota status resourceVersion changes as the old Pod exits.
|
|
patch = [{'op': 'test', 'path': '/metadata/resourceVersion', 'value': old['metadata']['resourceVersion']}, {'op': 'test', 'path': '/' + field, 'value': before}, {'op': 'replace', 'path': '/' + field, 'value': value}]
|
|
call('-n', NS, 'patch', kind, old['metadata']['name'], '--type=json', '-p', json.dumps(patch))
|
|
call('create', '-f', '-', data=json.dumps(desired_pod))
|
|
current = get('-n', NS, 'get', 'pod', 'node4-query25')
|
|
(out / 'created.json').write_text(json.dumps({'uid': current['metadata']['uid'], 'profile': desired_pod['metadata'].get('annotations', {}).get('query-lab/profile', 'baseline'), 'restored': bool(args.restore)}))
|
|
started = time.monotonic()
|
|
while True:
|
|
current = get('-n', NS, 'get', 'pod', 'node4-query25')
|
|
assert current['metadata']['uid'] == json.loads((out / 'created.json').read_text())['uid']
|
|
if not args.restore:
|
|
health()
|
|
if any(c['type'] == 'Ready' and c['status'] == 'True' for c in current['status'].get('conditions', [])):
|
|
break
|
|
assert current['status']['phase'] not in ('Failed', 'Succeeded'), 'lab startup failed'
|
|
assert time.monotonic() - started < 180, 'lab readiness deadline exceeded'
|
|
time.sleep(5)
|
|
verification = call('-n', NS, 'exec', 'node4-query25', '-c', 'probe', '--', 'bash', '/scripts/verify25.sh', timeout=90)
|
|
(out / 'verification.log').write_text(verification)
|
|
(out / 'after.json').write_text(json.dumps(get('-n', NS, 'get', 'pod', 'node4-query25'), indent=2))
|
|
print('VERIFIED', current['metadata']['uid'], verification.strip())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|