droboticflow/scripts/local/query_lab_profiles.py
d-robotics fc7786190f
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions
⚡ Publish query reports and coalesce task pool reads
2026-11-13 22:31:44 -08:00

74 lines
4.1 KiB
Python

"""Exact, bounded profiles for the synthetic node4 query25 experiment only."""
from copy import deepcopy
PROFILES = {
"baseline": {"cpu": "1", "memoryGi": 1, "cacheGi": "0.25", "rbps": 2097152, "wbps": 1048576, "riops": 64, "wiops": 32},
"compute": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 2097152, "wbps": 1048576, "riops": 64, "wiops": 32},
"balanced": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 4194304, "wbps": 2097152, "riops": 128, "wiops": 64},
"read-headroom": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 8388608, "wbps": 2097152, "riops": 256, "wiops": 64},
"query4-write4": {"cpu": "4", "memoryGi": 8, "cacheGi": "2", "rbps": 4194304, "wbps": 4194304, "riops": 128, "wiops": 128},
"query4-write8": {"cpu": "4", "memoryGi": 8, "cacheGi": "2", "rbps": 4194304, "wbps": 8388608, "riops": 128, "wiops": 256},
}
def profile(name):
return deepcopy(PROFILES[name])
def resources(name):
p = profile(name)
return {"cpu": p["cpu"], "memory": f'{p["memoryGi"]}Gi'}
def quota(name, fixture=False):
p = profile(name)
return {"pods": "2" if fixture else "1", "requests.cpu": str(int(p["cpu"]) + int(fixture)),
"limits.cpu": str(int(p["cpu"]) + 4 * int(fixture)),
"requests.memory": f'{p["memoryGi"] + 2 * int(fixture)}Gi',
"limits.memory": f'{p["memoryGi"] + 4 * int(fixture)}Gi',
"services": "0", "persistentvolumeclaims": "0"}
def render(objects, name):
"""Update one complete query25 manifest; never changes disk or durability."""
p = profile(name)
objects = deepcopy(objects)
config, pod = objects[-2:]
assert config["metadata"]["name"] == "node4-query25-scripts"
assert pod["metadata"]["name"] == "node4-query25"
original = 'rbps=2097152 wbps=1048576 riops=64 wiops=32'
io = ' '.join(f'{k}={p[k]}' for k in ('rbps', 'wbps', 'riops', 'wiops'))
for filename in ("init.sh", "guard.sh"):
assert original in config["data"][filename]
config["data"][filename] = config["data"][filename].replace(original, io)
config["data"]["guard.sh"] += f'\n[[ $(cat "$group/cpu.max") == "{int(p["cpu"]) * 100000} 100000" ]]\n[[ $(cat "$group/memory.max") == {p["memoryGi"] * 1024 ** 3} ]]\n'
config["data"]["query25.sh"] = config["data"]["query25.sh"].replace('--wiredTigerCacheSizeGB 0.25', '--wiredTigerCacheSizeGB ' + p['cacheGi'])
# Verify actual server settings, not only command text or admission requests.
config["data"]["verify25.sh"] = config["data"]["verify25.sh"].replace('const d=db.getSiblingDB', f'if(Number(db.serverStatus().wiredTiger.cache["maximum bytes configured"])!=={int(float(p["cacheGi"]) * 1024 ** 3)})throw Error("WT cache drift");const d=db.getSiblingDB')
config["data"]["profile.json"] = __import__('json').dumps({"name": name, **p}, sort_keys=True)
pod["metadata"].setdefault("annotations", {})["query-lab/profile"] = name
for container in pod["spec"]["containers"] + pod["spec"]["initContainers"]:
container["resources"] = {kind: resources(name) for kind in ("requests", "limits")}
objects[1]["spec"]["hard"] = quota(name)
return objects
def verify_resources(pod, name):
assert pod["metadata"].get("annotations", {}).get("query-lab/profile", "baseline") == name, "profile identity drift"
for c in pod["spec"]["containers"] + pod["spec"]["initContainers"]:
for kind in ("requests", "limits"):
actual = c["resources"][kind]
assert all(actual.get(k) == v for k, v in resources(name).items()), "resource drift"
assert set(actual) <= {"cpu", "memory", "cce.baidubce.com/ip"}
def verify_scripts(data, name):
import json
p = profile(name)
assert json.loads(data['profile.json']) == {'name': name, **p}, 'script profile drift'
io = ' '.join(f'{k}={p[k]}' for k in ('rbps', 'wbps', 'riops', 'wiops'))
for filename in ('init.sh', 'guard.sh'):
assert '252:0 ' + io in data[filename], 'backing IO drift'
assert '$loop_device ' + io in data[filename], 'loop IO drift'
assert '--wiredTigerCacheSizeGB ' + p['cacheGi'] + ' ' in data['query25.sh'], 'cache command drift'