173 lines
7.8 KiB
Python
173 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Owned loopback-only replica lab; no company credentials or customer databases."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import time
|
|
from lib.release_checks import check_environment, source_fingerprint, write_log
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PRIVATE = ROOT / 'local-data/query-replica-local'
|
|
COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
|
|
CASES = ('internal/store/query_replica_local_test.go',
|
|
'internal/store/query_sync_local_test.go',
|
|
'internal/store/query_replica_count_cache_test.go',
|
|
'internal/api/query_replica_local_test.go',
|
|
'cmd/query-cdc-probe/main_test.go',
|
|
'internal/store/query_replica_valkey_local_test.go',
|
|
'internal/store/workforce_report_cache_local_test.go',
|
|
'internal/store/workforce_facts_local_test.go',
|
|
'internal/store/workforce_publication_local_test.go',
|
|
'internal/store/query_replica_page_local_test.go',
|
|
'internal/store/supplier_authorization_batch_local_test.go',
|
|
'internal/store/query_cold_write_local_test.go',
|
|
'internal/store/query_periodic_local_test.go',
|
|
'internal/store/query_pool_buckets_local_test.go')
|
|
|
|
|
|
def owned_native():
|
|
for folder, port in [('source', 27119), ('read', 27129)]:
|
|
try:
|
|
pid = (PRIVATE / folder / 'mongod.pid').read_text().strip()
|
|
if not pid.isdecimal():
|
|
return False
|
|
command = subprocess.check_output(['ps', '-p', pid, '-o', 'command='], text=True, stderr=subprocess.DEVNULL)
|
|
if str(PRIVATE / folder) not in command or f'--port {port}' not in command:
|
|
return False
|
|
except (OSError, subprocess.SubprocessError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def run(argv, env, timeout=660):
|
|
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, text=True, start_new_session=True)
|
|
try:
|
|
stdout, stderr = child.communicate(timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(child.pid, signal.SIGTERM)
|
|
try:
|
|
stdout, stderr = child.communicate(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(child.pid, signal.SIGKILL)
|
|
stdout, stderr = child.communicate()
|
|
return 2, stdout + '\n' + stderr + '\ncheck timed out'
|
|
return child.returncode, stdout + '\n' + stderr
|
|
|
|
|
|
def compose(env, *args):
|
|
# Refuse remote Docker contexts before performing any container mutation.
|
|
endpoint = subprocess.check_output(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'],
|
|
cwd=ROOT, env=env, stderr=subprocess.DEVNULL, text=True).strip()
|
|
if not endpoint.startswith(('unix://', 'npipe://')):
|
|
raise RuntimeError('only a local Docker socket is supported')
|
|
files = ['-f', str(COMPOSE)]
|
|
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
|
|
files += ['-f', str(ROOT / 'scripts/local/query-replica.query25-local.compose.yml')]
|
|
code, output = run(['docker', 'compose', *files, *args], env, 180)
|
|
if code:
|
|
proof = write_log(PRIVATE / 'logs', ['docker', 'compose', *args], output, 'failed')
|
|
raise RuntimeError('local dependency command failed; ' + proof['log'])
|
|
return output
|
|
|
|
|
|
def start(env):
|
|
# Existing unowned listeners must not be replaced or stopped.
|
|
busy = []
|
|
for port in (27119, 27129):
|
|
try:
|
|
with socket.create_connection(('127.0.0.1', port), timeout=.3):
|
|
busy.append(port)
|
|
except OSError:
|
|
pass
|
|
if busy:
|
|
running = compose(env, 'ps', '--status', 'running', '--services').splitlines()
|
|
required = {27119: 'source', 27129: 'query'}
|
|
if any(required[port] not in running for port in busy):
|
|
raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports')
|
|
compose(env, 'up', '-d')
|
|
for _ in range(60):
|
|
try:
|
|
result = compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
|
|
"try {rs.status()} catch(e) {rs.initiate({_id:'query-read-rs',members:[{_id:0,host:'127.0.0.1:27129'}]})}; if (!db.hello().isWritablePrimary) quit(2)")
|
|
if result:
|
|
return
|
|
except RuntimeError:
|
|
time.sleep(1)
|
|
raise RuntimeError('query replica did not become primary')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--native', action='store_true')
|
|
parser.add_argument('--start', action='store_true')
|
|
parser.add_argument('--auto', action='store_true')
|
|
parser.add_argument('--down', action='store_true')
|
|
parser.add_argument('--race', action='store_true')
|
|
parser.add_argument('--scale', action='store_true')
|
|
parser.add_argument('--cache', action='store_true')
|
|
parser.add_argument('--bounded-query', action='store_true', help='functional tests with a 1CPU/1Gi query replica')
|
|
args = parser.parse_args()
|
|
PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
env = {k: v for k, v in check_environment().items() if not k.startswith('QUERY_')}
|
|
env['QUERY_REPLICA_LOCAL_TEST'] = '1'
|
|
if args.bounded_query:
|
|
env['QUERY_REPLICA_LOCAL_QUERY_BUDGET'] = 'query25'
|
|
before = source_fingerprint(ROOT)
|
|
with (PRIVATE / 'runner.lock').open('w') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if args.down:
|
|
compose(env, 'down')
|
|
return 0
|
|
native = args.native or (args.auto and owned_native())
|
|
if native:
|
|
if not owned_native():
|
|
raise RuntimeError('native mode requires this worktree\'s two owned Mongo processes')
|
|
elif args.start or args.auto:
|
|
start(env)
|
|
env['QUERY_REPLICA_DOCKER_LAB'] = '1'
|
|
else:
|
|
raise RuntimeError('select --native, --start or --auto explicitly')
|
|
if args.cache:
|
|
compose(env, 'up', '-d', 'valkey')
|
|
env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1'
|
|
required = set()
|
|
for name in CASES:
|
|
required.update(re.findall(r'^func (Test\w+)\(', (ROOT / name).read_text(), re.M))
|
|
if not args.cache:
|
|
required = {name for name in required if not name.startswith('TestQueryReplicaSharedValkey')}
|
|
if args.scale:
|
|
required = {'TestQueryReplicaScaleLocal'}
|
|
env['QUERY_REPLICA_SCALE_LOCAL'] = '1'
|
|
regex = '^(' + '|'.join(sorted(required)) + ')$'
|
|
argv = ['go', 'test', './internal/store', './internal/api', './cmd/query-cdc-probe',
|
|
'-run', regex, '-count=1', '-timeout=10m', '-v']
|
|
if args.race:
|
|
argv.append('-race')
|
|
code, output = run(argv, env)
|
|
passed = set(re.findall(r'^--- PASS: (Test\w+)', output, re.M))
|
|
missing = sorted(required - passed)
|
|
changed = not before or source_fingerprint(ROOT) != before
|
|
skipped = bool(re.search(r'^--- SKIP:', output, re.M))
|
|
success = code == 0 and not missing and not changed and not skipped
|
|
proof = write_log(PRIVATE / 'logs', argv, output, 'passed' if success else 'pending_or_failed')
|
|
proof.update(missing=missing, sourceChanged=changed, skipped=skipped,
|
|
environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'),
|
|
productionCapacityAccepted=False)
|
|
print(json.dumps(proof, ensure_ascii=False))
|
|
return 0 if success else (code or 2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
raise SystemExit(main())
|
|
except (RuntimeError, OSError, subprocess.SubprocessError) as error:
|
|
print(str(error))
|
|
raise SystemExit(2)
|