#!/usr/bin/env python3 """Owned loopback-only replica lab; no company credentials or customer databases.""" import argparse import fcntl import json import os from pathlib import Path import re import signal import socket import subprocess import time import uuid import sys from lib.release_checks import check_environment, source_fingerprint, write_log ROOT = Path(__file__).resolve().parents[1] PRIVATE = ROOT / 'local-data/query-replica-local' COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml' CASES = ('cmd/query-replica/main_local_test.go', 'cmd/query-replica/source_transport_test.go', 'internal/store/query_replica_source_auth_local_test.go', 'internal/store/query_replica_safety_local_test.go', 'internal/store/query_replica_maintenance_flow_local_test.go', 'internal/store/query_replica_safety_test.go', 'internal/store/query_replica_diagnostic_test.go', 'internal/store/query_replica_projection_local_test.go', 'internal/store/query_replica_local_test.go', 'internal/store/query_sync_local_test.go', 'internal/store/query_replica_count_cache_test.go', 'internal/api/query_replica_local_test.go', 'cmd/query-cdc-probe/main_test.go', 'internal/store/query_replica_valkey_local_test.go', 'internal/store/workforce_report_cache_local_test.go', 'internal/store/workforce_facts_local_test.go', 'internal/store/query_build_capture_local_test.go', 'internal/store/query_build_resume_local_test.go', 'internal/store/query_build_outcome_local_test.go', 'internal/store/workforce_publication_local_test.go', 'internal/store/query_replica_page_local_test.go', 'internal/store/supplier_authorization_batch_local_test.go', 'internal/store/query_cold_write_local_test.go', 'internal/store/query_periodic_local_test.go', 'internal/store/query_pool_buckets_local_test.go') def owned_native(): for folder, port in [('source', 27119), ('read', 27129)]: try: pid = (PRIVATE / folder / 'mongod.pid').read_text().strip() if not pid.isdecimal(): return False command = subprocess.check_output(['ps', '-p', pid, '-o', 'command='], text=True, stderr=subprocess.DEVNULL) if str(PRIVATE / folder) not in command or f'--port {port}' not in command: return False except (OSError, subprocess.SubprocessError): return False return True def run(argv, env, timeout=660): child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) try: stdout, stderr = child.communicate(timeout=timeout) except subprocess.TimeoutExpired: os.killpg(child.pid, signal.SIGTERM) try: stdout, stderr = child.communicate(timeout=5) except subprocess.TimeoutExpired: os.killpg(child.pid, signal.SIGKILL) stdout, stderr = child.communicate() return 2, stdout + '\n' + stderr + '\ncheck timed out' return child.returncode, stdout + '\n' + stderr def compose(env, *args): # Refuse remote Docker contexts before performing any container mutation. endpoint = subprocess.check_output(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'], cwd=ROOT, env=env, stderr=subprocess.DEVNULL, text=True).strip() if not endpoint.startswith(('unix://', 'npipe://')): raise RuntimeError('only a local Docker socket is supported') files = ['-f', str(COMPOSE)] if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25': files += ['-f', str(ROOT / 'scripts/local/query-replica.query25-local.compose.yml')] code, output = run(['docker', 'compose', *files, *args], env, 180) if code: proof = write_log(PRIVATE / 'logs', ['docker', 'compose', *args], output, 'failed') raise RuntimeError('local dependency command failed; ' + proof['log']) return output def start(env): # Existing unowned listeners must not be replaced or stopped. busy = [] for port in (27119, 27129): try: with socket.create_connection(('127.0.0.1', port), timeout=.3): busy.append(port) except OSError: pass if busy: running = compose(env, 'ps', '--status', 'running', '--services').splitlines() required = {27119: 'source', 27129: 'query'} if any(required[port] not in running for port in busy): raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports') if len(busy) == 2: # Reuse isolated random databases, never recreate a shared fixture just # because another authorized checkout owns its Compose metadata. info = json.loads(subprocess.check_output(['docker', 'inspect', 'fanyu-query-replica-local-source-1', 'fanyu-query-replica-local-query-1'], cwd=ROOT, env=env, text=True)) resources = [{'name': x['Name'], 'cpu': x['HostConfig']['NanoCpus'], 'memory': x['HostConfig']['Memory']} for x in info] if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25': query = next(x for x in resources if x['name'].endswith('query-1')) if query['cpu'] != 1_000_000_000 or query['memory'] != 1024 ** 3: raise RuntimeError('existing shared query fixture differs from1CPU/1Gi; do not recreate it') env['QUERY_REPLICA_REUSED_RESOURCES'] = json.dumps(resources) compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval', "if (!db.hello().isWritablePrimary || db.hello().setName !== 'query-read-rs') quit(2)") return compose(env, 'up', '-d') for _ in range(60): try: result = compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval', "try {rs.status()} catch(e) {rs.initiate({_id:'query-read-rs',members:[{_id:0,host:'127.0.0.1:27129'}]})}; if (!db.hello().isWritablePrimary) quit(2)") if result: return except RuntimeError: time.sleep(1) raise RuntimeError('query replica did not become primary') def case_batches(required, bounded, native): names = sorted(required) if not names: raise RuntimeError('no registered replica cases') size = 20 if bounded and not native else len(names) return [names[i:i + size] for i in range(0, len(names), size)] def query_probe(env, script): # Fixed loopback, no credentials, four-second process bound. This is not # an arbitrary database command entry point or a production probe. return run(['docker', 'exec', 'fanyu-query-replica-local-query-1', 'mongosh', '--quiet', 'mongodb://127.0.0.1:27129/?serverSelectionTimeoutMS=1000&connectTimeoutMS=1000&appName=droboticflow-query-fixture-probe&maxPoolSize=1&maxConnecting=1', '--eval', script], env, 4) def restart_query_fixture(env, probe, cursor): name = 'fanyu-query-replica-local-query-1' before = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0] if (before['Name'] != '/' + name or before['HostConfig']['Memory'] != 1024 ** 3 or before['HostConfig']['NanoCpus'] != 1_000_000_000 or before['Config']['Labels'].get('com.docker.compose.project') != 'fanyu-query-replica-local' or not any(m['Destination'] == '/data/db' for m in before['Mounts'])): raise RuntimeError('refusing to restart a different query fixture') # A direct client from another checkout must not be interrupted. Exclude only this named probe and the replica set internal client; # wait briefly for completed test clients and in-progress handshakes to close. for attempt in range(3): code, _ = query_probe(env, 'sleep(200);' 'const clients=db.getSiblingDB("admin").aggregate([' '{$currentOp:{allUsers:true,idleConnections:true}},' '{$match:{client:{$exists:true},"clientMetadata.application.name":{$ne:"droboticflow-query-fixture-probe"},' '"clientMetadata.driver.name":{$not:/^NetworkInterfaceTL/}}},' '{$limit:1}]).toArray();if(clients.length) quit(3)') if code == 0: break if attempt == 2: raise RuntimeError('query restart admission failed: ' + ('other_clients' if code == 3 else 'probe_exit_' + str(code))) time.sleep(.5) owner = json.dumps(probe) code, _ = query_probe(env, 'db.getSiblingDB(' + owner + ').checkpoint.updateOne({_id:"progress"},' '{$set:{cursor:' + str(cursor) + ',fingerprint:"synthetic-majority-journal"}},' '{upsert:true,writeConcern:{w:"majority",j:true}})') if code: raise RuntimeError('restart canary persistence failed') compose(env, 'restart', '-t', '30', 'query') after = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0] if (after['Id'] != before['Id'] or sorted(after['Mounts'], key=lambda m: json.dumps(m, sort_keys=True)) != sorted(before['Mounts'], key=lambda m: json.dumps(m, sort_keys=True)) or after['HostConfig']['Memory'] != before['HostConfig']['Memory'] or after['HostConfig']['NanoCpus'] != before['HostConfig']['NanoCpus']): raise RuntimeError('query restart changed volume or resource identity') deadline = time.monotonic() + 30 while time.monotonic() < deadline: code, _ = query_probe(env, 'if(!db.hello().isWritablePrimary||db.hello().setName!=="query-read-rs") quit(2);' 'const p=db.getSiblingDB(' + owner + ').checkpoint.findOne({_id:"progress"});' 'if(!p||p.cursor!==' + str(cursor) + '||p.fingerprint!=="synthetic-majority-journal") quit(4)') if code == 0: return {'container': before['Id'], 'cursor': cursor, 'majorityJournalRecovered': True, 'sameVolume': True, 'sameResources': True} time.sleep(.5) raise RuntimeError('query restart readiness or durable canary recovery failed') def main(): parser = argparse.ArgumentParser() parser.add_argument('--native', action='store_true') parser.add_argument('--start', action='store_true') parser.add_argument('--auto', action='store_true') parser.add_argument('--down', action='store_true') parser.add_argument('--race', action='store_true') parser.add_argument('--scale', action='store_true') parser.add_argument('--cache', action='store_true') parser.add_argument('--bounded-query', action='store_true', help='functional tests with a 1CPU/1Gi query replica') args = parser.parse_args() PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700) env = {k: v for k, v in check_environment().items() if not k.startswith('QUERY_')} env['QUERY_REPLICA_LOCAL_TEST'] = '1' if args.bounded_query: env['QUERY_REPLICA_LOCAL_QUERY_BUDGET'] = 'query25' before = source_fingerprint(ROOT) with (PRIVATE / 'runner.lock').open('w') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if args.down: compose(env, 'down') return 0 native = args.native or (args.auto and owned_native()) if native: if not owned_native(): raise RuntimeError('native mode requires this worktree\'s two owned Mongo processes') elif args.start or args.auto: start(env) env['QUERY_REPLICA_DOCKER_LAB'] = '1' else: raise RuntimeError('select --native, --start or --auto explicitly') if args.cache: if 'valkey' not in compose(env, 'ps', '--status', 'running', '--services').splitlines(): compose(env, 'up', '-d', 'valkey') env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1' required = set() for name in CASES: required.update(re.findall(r'^func (Test\w+)\(', (ROOT / name).read_text(), re.M)) if not args.cache: required = {name for name in required if not name.startswith('TestQueryReplicaSharedValkey')} if args.scale: required = {'TestQueryReplicaScaleLocal'} env['QUERY_REPLICA_SCALE_LOCAL'] = '1' batches = case_batches(required, args.bounded_query, native) outputs, restart_proofs, commands = [], [], [] code = 0 probe = 'query_replica_restart_probe_' + uuid.uuid4().hex for index, names in enumerate(batches): if len(batches) > 1: restart_proofs.append(restart_query_fixture(env, probe, index)) regex = '^(' + '|'.join(names) + ')$' argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica', '-run', regex, '-count=1', '-timeout=10m', '-v'] if args.race: argv.append('-race') commands.append(argv) code, output = run(argv, env) outputs.append(output) if code: break output = '\n'.join(outputs) passed = set(re.findall(r'^--- PASS: (Test\w+)', output, re.M)) missing = sorted(required - passed) changed = not before or source_fingerprint(ROOT) != before skipped = bool(re.search(r'^--- SKIP:', output, re.M)) success = code == 0 and not missing and not changed and not skipped proof = write_log(PRIVATE / 'logs', [sys.executable, str(Path(__file__).resolve()), *sys.argv[1:]], output, 'passed' if success else 'pending_or_failed') proof.update(missing=missing, sourceChanged=changed, skipped=skipped, environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'), productionCapacityAccepted=False, boundedBatches=len(batches), continuousCapacityAccepted=False, restartProofs=restart_proofs, requiredCount=len(required), passedCount=len(passed), batchCommands=commands, completedBatches=len(outputs), reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]'))) print(json.dumps(proof, ensure_ascii=False)) return 0 if success else (code or 2) if __name__ == '__main__': try: raise SystemExit(main()) except (RuntimeError, OSError, subprocess.SubprocessError) as error: print(str(error)) raise SystemExit(2)