droboticflow/scripts/test_query_replica.py
2026-11-14 22:07:22 -08:00

196 lines
9.6 KiB
Python

#!/usr/bin/env python3
"""Owned loopback-only replica lab; no company credentials or customer databases."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import re
import signal
import socket
import subprocess
import time
from lib.release_checks import check_environment, source_fingerprint, write_log
ROOT = Path(__file__).resolve().parents[1]
PRIVATE = ROOT / 'local-data/query-replica-local'
COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
CASES = ('cmd/query-replica/main_local_test.go',
'internal/store/query_replica_source_auth_local_test.go',
'internal/store/query_replica_safety_local_test.go',
'internal/store/query_replica_maintenance_flow_local_test.go',
'internal/store/query_replica_safety_test.go',
'internal/store/query_replica_diagnostic_test.go',
'internal/store/query_replica_projection_local_test.go',
'internal/store/query_replica_local_test.go',
'internal/store/query_sync_local_test.go',
'internal/store/query_replica_count_cache_test.go',
'internal/api/query_replica_local_test.go',
'cmd/query-cdc-probe/main_test.go',
'internal/store/query_replica_valkey_local_test.go',
'internal/store/workforce_report_cache_local_test.go',
'internal/store/workforce_facts_local_test.go',
'internal/store/workforce_publication_local_test.go',
'internal/store/query_replica_page_local_test.go',
'internal/store/supplier_authorization_batch_local_test.go',
'internal/store/query_cold_write_local_test.go',
'internal/store/query_periodic_local_test.go',
'internal/store/query_pool_buckets_local_test.go')
def owned_native():
for folder, port in [('source', 27119), ('read', 27129)]:
try:
pid = (PRIVATE / folder / 'mongod.pid').read_text().strip()
if not pid.isdecimal():
return False
command = subprocess.check_output(['ps', '-p', pid, '-o', 'command='], text=True, stderr=subprocess.DEVNULL)
if str(PRIVATE / folder) not in command or f'--port {port}' not in command:
return False
except (OSError, subprocess.SubprocessError):
return False
return True
def run(argv, env, timeout=660):
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, start_new_session=True)
try:
stdout, stderr = child.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
os.killpg(child.pid, signal.SIGTERM)
try:
stdout, stderr = child.communicate(timeout=5)
except subprocess.TimeoutExpired:
os.killpg(child.pid, signal.SIGKILL)
stdout, stderr = child.communicate()
return 2, stdout + '\n' + stderr + '\ncheck timed out'
return child.returncode, stdout + '\n' + stderr
def compose(env, *args):
# Refuse remote Docker contexts before performing any container mutation.
endpoint = subprocess.check_output(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'],
cwd=ROOT, env=env, stderr=subprocess.DEVNULL, text=True).strip()
if not endpoint.startswith(('unix://', 'npipe://')):
raise RuntimeError('only a local Docker socket is supported')
files = ['-f', str(COMPOSE)]
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
files += ['-f', str(ROOT / 'scripts/local/query-replica.query25-local.compose.yml')]
code, output = run(['docker', 'compose', *files, *args], env, 180)
if code:
proof = write_log(PRIVATE / 'logs', ['docker', 'compose', *args], output, 'failed')
raise RuntimeError('local dependency command failed; ' + proof['log'])
return output
def start(env):
# Existing unowned listeners must not be replaced or stopped.
busy = []
for port in (27119, 27129):
try:
with socket.create_connection(('127.0.0.1', port), timeout=.3):
busy.append(port)
except OSError:
pass
if busy:
running = compose(env, 'ps', '--status', 'running', '--services').splitlines()
required = {27119: 'source', 27129: 'query'}
if any(required[port] not in running for port in busy):
raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports')
if len(busy) == 2:
# Reuse isolated random databases, never recreate a shared fixture just
# because another authorized checkout owns its Compose metadata.
info = json.loads(subprocess.check_output(['docker', 'inspect', 'fanyu-query-replica-local-source-1',
'fanyu-query-replica-local-query-1'], cwd=ROOT, env=env, text=True))
resources = [{'name': x['Name'], 'cpu': x['HostConfig']['NanoCpus'],
'memory': x['HostConfig']['Memory']} for x in info]
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
query = next(x for x in resources if x['name'].endswith('query-1'))
if query['cpu'] != 1_000_000_000 or query['memory'] != 1024 ** 3:
raise RuntimeError('existing shared query fixture differs from1CPU/1Gi; do not recreate it')
env['QUERY_REPLICA_REUSED_RESOURCES'] = json.dumps(resources)
compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
"if (!db.hello().isWritablePrimary || db.hello().setName !== 'query-read-rs') quit(2)")
return
compose(env, 'up', '-d')
for _ in range(60):
try:
result = compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
"try {rs.status()} catch(e) {rs.initiate({_id:'query-read-rs',members:[{_id:0,host:'127.0.0.1:27129'}]})}; if (!db.hello().isWritablePrimary) quit(2)")
if result:
return
except RuntimeError:
time.sleep(1)
raise RuntimeError('query replica did not become primary')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--native', action='store_true')
parser.add_argument('--start', action='store_true')
parser.add_argument('--auto', action='store_true')
parser.add_argument('--down', action='store_true')
parser.add_argument('--race', action='store_true')
parser.add_argument('--scale', action='store_true')
parser.add_argument('--cache', action='store_true')
parser.add_argument('--bounded-query', action='store_true', help='functional tests with a 1CPU/1Gi query replica')
args = parser.parse_args()
PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700)
env = {k: v for k, v in check_environment().items() if not k.startswith('QUERY_')}
env['QUERY_REPLICA_LOCAL_TEST'] = '1'
if args.bounded_query:
env['QUERY_REPLICA_LOCAL_QUERY_BUDGET'] = 'query25'
before = source_fingerprint(ROOT)
with (PRIVATE / 'runner.lock').open('w') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if args.down:
compose(env, 'down')
return 0
native = args.native or (args.auto and owned_native())
if native:
if not owned_native():
raise RuntimeError('native mode requires this worktree\'s two owned Mongo processes')
elif args.start or args.auto:
start(env)
env['QUERY_REPLICA_DOCKER_LAB'] = '1'
else:
raise RuntimeError('select --native, --start or --auto explicitly')
if args.cache:
if 'valkey' not in compose(env, 'ps', '--status', 'running', '--services').splitlines():
compose(env, 'up', '-d', 'valkey')
env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1'
required = set()
for name in CASES:
required.update(re.findall(r'^func (Test\w+)\(', (ROOT / name).read_text(), re.M))
if not args.cache:
required = {name for name in required if not name.startswith('TestQueryReplicaSharedValkey')}
if args.scale:
required = {'TestQueryReplicaScaleLocal'}
env['QUERY_REPLICA_SCALE_LOCAL'] = '1'
regex = '^(' + '|'.join(sorted(required)) + ')$'
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
'-run', regex, '-count=1', '-timeout=10m', '-v']
if args.race:
argv.append('-race')
code, output = run(argv, env)
passed = set(re.findall(r'^--- PASS: (Test\w+)', output, re.M))
missing = sorted(required - passed)
changed = not before or source_fingerprint(ROOT) != before
skipped = bool(re.search(r'^--- SKIP:', output, re.M))
success = code == 0 and not missing and not changed and not skipped
proof = write_log(PRIVATE / 'logs', argv, output, 'passed' if success else 'pending_or_failed')
proof.update(missing=missing, sourceChanged=changed, skipped=skipped,
environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'),
productionCapacityAccepted=False, reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
print(json.dumps(proof, ensure_ascii=False))
return 0 if success else (code or 2)
if __name__ == '__main__':
try:
raise SystemExit(main())
except (RuntimeError, OSError, subprocess.SubprocessError) as error:
print(str(error))
raise SystemExit(2)