droboticflow/scripts/test_query_replica.py
d-robotics 412a12cfe5
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions
🛠️ Recover expired query snapshots and bound role-change reads
2026-11-16 08:21:41 -08:00

284 lines
15 KiB
Python

#!/usr/bin/env python3
"""Owned loopback-only replica lab; no company credentials or customer databases."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import re
import signal
import socket
import subprocess
import time
import uuid
import sys
from lib.release_checks import check_environment, source_fingerprint, write_log
ROOT = Path(__file__).resolve().parents[1]
PRIVATE = ROOT / 'local-data/query-replica-local'
COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
CASES = ('cmd/query-replica/main_local_test.go',
'cmd/query-replica/source_transport_test.go',
'internal/store/query_replica_source_auth_local_test.go',
'internal/store/query_replica_safety_local_test.go',
'internal/store/query_replica_maintenance_flow_local_test.go',
'internal/store/query_replica_safety_test.go',
'internal/store/query_replica_diagnostic_test.go',
'internal/store/query_replica_projection_local_test.go',
'internal/store/query_replica_local_test.go',
'internal/store/query_sync_local_test.go',
'internal/store/query_replica_count_cache_test.go',
'internal/api/query_replica_local_test.go',
'cmd/query-cdc-probe/main_test.go',
'internal/store/query_replica_valkey_local_test.go',
'internal/store/workforce_report_cache_local_test.go',
'internal/store/workforce_facts_local_test.go',
'internal/store/query_build_capture_local_test.go',
'internal/store/query_build_resume_local_test.go',
'internal/store/query_build_outcome_local_test.go',
'internal/store/workforce_publication_local_test.go',
'internal/store/query_replica_page_local_test.go',
'internal/store/supplier_authorization_batch_local_test.go',
'internal/store/query_cold_write_local_test.go',
'internal/store/query_periodic_local_test.go',
'internal/store/query_pool_buckets_local_test.go')
def owned_native():
for folder, port in [('source', 27119), ('read', 27129)]:
try:
pid = (PRIVATE / folder / 'mongod.pid').read_text().strip()
if not pid.isdecimal():
return False
command = subprocess.check_output(['ps', '-p', pid, '-o', 'command='], text=True, stderr=subprocess.DEVNULL)
if str(PRIVATE / folder) not in command or f'--port {port}' not in command:
return False
except (OSError, subprocess.SubprocessError):
return False
return True
def run(argv, env, timeout=660):
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, start_new_session=True)
try:
stdout, stderr = child.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
os.killpg(child.pid, signal.SIGTERM)
try:
stdout, stderr = child.communicate(timeout=5)
except subprocess.TimeoutExpired:
os.killpg(child.pid, signal.SIGKILL)
stdout, stderr = child.communicate()
return 2, stdout + '\n' + stderr + '\ncheck timed out'
return child.returncode, stdout + '\n' + stderr
def compose(env, *args):
# Refuse remote Docker contexts before performing any container mutation.
endpoint = subprocess.check_output(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'],
cwd=ROOT, env=env, stderr=subprocess.DEVNULL, text=True).strip()
if not endpoint.startswith(('unix://', 'npipe://')):
raise RuntimeError('only a local Docker socket is supported')
files = ['-f', str(COMPOSE)]
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
files += ['-f', str(ROOT / 'scripts/local/query-replica.query25-local.compose.yml')]
code, output = run(['docker', 'compose', *files, *args], env, 180)
if code:
proof = write_log(PRIVATE / 'logs', ['docker', 'compose', *args], output, 'failed')
raise RuntimeError('local dependency command failed; ' + proof['log'])
return output
def start(env):
# Existing unowned listeners must not be replaced or stopped.
busy = []
for port in (27119, 27129):
try:
with socket.create_connection(('127.0.0.1', port), timeout=.3):
busy.append(port)
except OSError:
pass
if busy:
running = compose(env, 'ps', '--status', 'running', '--services').splitlines()
required = {27119: 'source', 27129: 'query'}
if any(required[port] not in running for port in busy):
raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports')
if len(busy) == 2:
# Reuse isolated random databases, never recreate a shared fixture just
# because another authorized checkout owns its Compose metadata.
info = json.loads(subprocess.check_output(['docker', 'inspect', 'fanyu-query-replica-local-source-1',
'fanyu-query-replica-local-query-1'], cwd=ROOT, env=env, text=True))
resources = [{'name': x['Name'], 'cpu': x['HostConfig']['NanoCpus'],
'memory': x['HostConfig']['Memory']} for x in info]
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
query = next(x for x in resources if x['name'].endswith('query-1'))
if query['cpu'] != 1_000_000_000 or query['memory'] != 1024 ** 3:
raise RuntimeError('existing shared query fixture differs from1CPU/1Gi; do not recreate it')
env['QUERY_REPLICA_REUSED_RESOURCES'] = json.dumps(resources)
compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
"if (!db.hello().isWritablePrimary || db.hello().setName !== 'query-read-rs') quit(2)")
return
compose(env, 'up', '-d')
for _ in range(60):
try:
result = compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
"try {rs.status()} catch(e) {rs.initiate({_id:'query-read-rs',members:[{_id:0,host:'127.0.0.1:27129'}]})}; if (!db.hello().isWritablePrimary) quit(2)")
if result:
return
except RuntimeError:
time.sleep(1)
raise RuntimeError('query replica did not become primary')
def case_batches(required, bounded, native):
names = sorted(required)
if not names:
raise RuntimeError('no registered replica cases')
size = 20 if bounded and not native else len(names)
return [names[i:i + size] for i in range(0, len(names), size)]
def query_probe(env, script):
# Fixed loopback, no credentials, four-second process bound. This is not
# an arbitrary database command entry point or a production probe.
return run(['docker', 'exec', 'fanyu-query-replica-local-query-1',
'mongosh', '--quiet',
'mongodb://127.0.0.1:27129/?serverSelectionTimeoutMS=1000&connectTimeoutMS=1000&appName=droboticflow-query-fixture-probe&maxPoolSize=1&maxConnecting=1',
'--eval', script], env, 4)
def restart_query_fixture(env, probe, cursor):
name = 'fanyu-query-replica-local-query-1'
before = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0]
if (before['Name'] != '/' + name or before['HostConfig']['Memory'] != 1024 ** 3
or before['HostConfig']['NanoCpus'] != 1_000_000_000
or before['Config']['Labels'].get('com.docker.compose.project') != 'fanyu-query-replica-local'
or not any(m['Destination'] == '/data/db' for m in before['Mounts'])):
raise RuntimeError('refusing to restart a different query fixture')
# A direct client from another checkout must not be interrupted. Exclude only this named probe and the replica set internal client;
# wait briefly for completed test clients and in-progress handshakes to close.
for attempt in range(3):
code, _ = query_probe(env,
'sleep(200);'
'const clients=db.getSiblingDB("admin").aggregate(['
'{$currentOp:{allUsers:true,idleConnections:true}},'
'{$match:{client:{$exists:true},"clientMetadata.application.name":{$ne:"droboticflow-query-fixture-probe"},'
'"clientMetadata.driver.name":{$not:/^NetworkInterfaceTL/}}},'
'{$limit:1}]).toArray();if(clients.length) quit(3)')
if code == 0:
break
if attempt == 2:
raise RuntimeError('query restart admission failed: ' + ('other_clients' if code == 3 else 'probe_exit_' + str(code)))
time.sleep(.5)
owner = json.dumps(probe)
code, _ = query_probe(env, 'db.getSiblingDB(' + owner + ').checkpoint.updateOne({_id:"progress"},'
'{$set:{cursor:' + str(cursor) + ',fingerprint:"synthetic-majority-journal"}},'
'{upsert:true,writeConcern:{w:"majority",j:true}})')
if code:
raise RuntimeError('restart canary persistence failed')
compose(env, 'restart', '-t', '30', 'query')
after = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0]
if (after['Id'] != before['Id']
or sorted(after['Mounts'], key=lambda m: json.dumps(m, sort_keys=True))
!= sorted(before['Mounts'], key=lambda m: json.dumps(m, sort_keys=True))
or after['HostConfig']['Memory'] != before['HostConfig']['Memory']
or after['HostConfig']['NanoCpus'] != before['HostConfig']['NanoCpus']):
raise RuntimeError('query restart changed volume or resource identity')
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
code, _ = query_probe(env, 'if(!db.hello().isWritablePrimary||db.hello().setName!=="query-read-rs") quit(2);'
'const p=db.getSiblingDB(' + owner + ').checkpoint.findOne({_id:"progress"});'
'if(!p||p.cursor!==' + str(cursor) + '||p.fingerprint!=="synthetic-majority-journal") quit(4)')
if code == 0:
return {'container': before['Id'], 'cursor': cursor, 'majorityJournalRecovered': True,
'sameVolume': True, 'sameResources': True}
time.sleep(.5)
raise RuntimeError('query restart readiness or durable canary recovery failed')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--native', action='store_true')
parser.add_argument('--start', action='store_true')
parser.add_argument('--auto', action='store_true')
parser.add_argument('--down', action='store_true')
parser.add_argument('--race', action='store_true')
parser.add_argument('--scale', action='store_true')
parser.add_argument('--cache', action='store_true')
parser.add_argument('--bounded-query', action='store_true', help='functional tests with a 1CPU/1Gi query replica')
args = parser.parse_args()
PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700)
env = {k: v for k, v in check_environment().items() if not k.startswith('QUERY_')}
env['QUERY_REPLICA_LOCAL_TEST'] = '1'
if args.bounded_query:
env['QUERY_REPLICA_LOCAL_QUERY_BUDGET'] = 'query25'
before = source_fingerprint(ROOT)
with (PRIVATE / 'runner.lock').open('w') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if args.down:
compose(env, 'down')
return 0
native = args.native or (args.auto and owned_native())
if native:
if not owned_native():
raise RuntimeError('native mode requires this worktree\'s two owned Mongo processes')
elif args.start or args.auto:
start(env)
env['QUERY_REPLICA_DOCKER_LAB'] = '1'
else:
raise RuntimeError('select --native, --start or --auto explicitly')
if args.cache:
if 'valkey' not in compose(env, 'ps', '--status', 'running', '--services').splitlines():
compose(env, 'up', '-d', 'valkey')
env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1'
required = set()
for name in CASES:
required.update(re.findall(r'^func (Test\w+)\(', (ROOT / name).read_text(), re.M))
if not args.cache:
required = {name for name in required if not name.startswith('TestQueryReplicaSharedValkey')}
if args.scale:
required = {'TestQueryReplicaScaleLocal'}
env['QUERY_REPLICA_SCALE_LOCAL'] = '1'
batches = case_batches(required, args.bounded_query, native)
outputs, restart_proofs, commands = [], [], []
code = 0
probe = 'query_replica_restart_probe_' + uuid.uuid4().hex
for index, names in enumerate(batches):
if len(batches) > 1:
restart_proofs.append(restart_query_fixture(env, probe, index))
regex = '^(' + '|'.join(names) + ')$'
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
'-run', regex, '-count=1', '-timeout=10m', '-v']
if args.race:
argv.append('-race')
commands.append(argv)
code, output = run(argv, env)
outputs.append(output)
if code:
break
output = '\n'.join(outputs)
passed = set(re.findall(r'^--- PASS: (Test\w+)', output, re.M))
missing = sorted(required - passed)
changed = not before or source_fingerprint(ROOT) != before
skipped = bool(re.search(r'^--- SKIP:', output, re.M))
success = code == 0 and not missing and not changed and not skipped
proof = write_log(PRIVATE / 'logs', [sys.executable, str(Path(__file__).resolve()), *sys.argv[1:]], output, 'passed' if success else 'pending_or_failed')
proof.update(missing=missing, sourceChanged=changed, skipped=skipped,
environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'),
productionCapacityAccepted=False, boundedBatches=len(batches),
continuousCapacityAccepted=False, restartProofs=restart_proofs, requiredCount=len(required),
passedCount=len(passed), batchCommands=commands, completedBatches=len(outputs),
reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
print(json.dumps(proof, ensure_ascii=False))
return 0 if success else (code or 2)
if __name__ == '__main__':
try:
raise SystemExit(main())
except (RuntimeError, OSError, subprocess.SubprocessError) as error:
print(str(error))
raise SystemExit(2)