284 lines
15 KiB
Python
284 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Owned loopback-only replica lab; no company credentials or customer databases."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
import sys
|
|
from lib.release_checks import check_environment, source_fingerprint, write_log
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PRIVATE = ROOT / 'local-data/query-replica-local'
|
|
COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
|
|
CASES = ('cmd/query-replica/main_local_test.go',
|
|
'cmd/query-replica/source_transport_test.go',
|
|
'internal/store/query_replica_source_auth_local_test.go',
|
|
'internal/store/query_replica_safety_local_test.go',
|
|
'internal/store/query_replica_maintenance_flow_local_test.go',
|
|
'internal/store/query_replica_safety_test.go',
|
|
'internal/store/query_replica_diagnostic_test.go',
|
|
'internal/store/query_replica_projection_local_test.go',
|
|
'internal/store/query_replica_local_test.go',
|
|
'internal/store/query_sync_local_test.go',
|
|
'internal/store/query_replica_count_cache_test.go',
|
|
'internal/api/query_replica_local_test.go',
|
|
'cmd/query-cdc-probe/main_test.go',
|
|
'internal/store/query_replica_valkey_local_test.go',
|
|
'internal/store/workforce_report_cache_local_test.go',
|
|
'internal/store/workforce_facts_local_test.go',
|
|
'internal/store/query_build_capture_local_test.go',
|
|
'internal/store/query_build_resume_local_test.go',
|
|
'internal/store/query_build_outcome_local_test.go',
|
|
'internal/store/workforce_publication_local_test.go',
|
|
'internal/store/query_replica_page_local_test.go',
|
|
'internal/store/supplier_authorization_batch_local_test.go',
|
|
'internal/store/query_cold_write_local_test.go',
|
|
'internal/store/query_periodic_local_test.go',
|
|
'internal/store/query_pool_buckets_local_test.go')
|
|
|
|
|
|
def owned_native():
|
|
for folder, port in [('source', 27119), ('read', 27129)]:
|
|
try:
|
|
pid = (PRIVATE / folder / 'mongod.pid').read_text().strip()
|
|
if not pid.isdecimal():
|
|
return False
|
|
command = subprocess.check_output(['ps', '-p', pid, '-o', 'command='], text=True, stderr=subprocess.DEVNULL)
|
|
if str(PRIVATE / folder) not in command or f'--port {port}' not in command:
|
|
return False
|
|
except (OSError, subprocess.SubprocessError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def run(argv, env, timeout=660):
|
|
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, text=True, start_new_session=True)
|
|
try:
|
|
stdout, stderr = child.communicate(timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(child.pid, signal.SIGTERM)
|
|
try:
|
|
stdout, stderr = child.communicate(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(child.pid, signal.SIGKILL)
|
|
stdout, stderr = child.communicate()
|
|
return 2, stdout + '\n' + stderr + '\ncheck timed out'
|
|
return child.returncode, stdout + '\n' + stderr
|
|
|
|
|
|
def compose(env, *args):
|
|
# Refuse remote Docker contexts before performing any container mutation.
|
|
endpoint = subprocess.check_output(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'],
|
|
cwd=ROOT, env=env, stderr=subprocess.DEVNULL, text=True).strip()
|
|
if not endpoint.startswith(('unix://', 'npipe://')):
|
|
raise RuntimeError('only a local Docker socket is supported')
|
|
files = ['-f', str(COMPOSE)]
|
|
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
|
|
files += ['-f', str(ROOT / 'scripts/local/query-replica.query25-local.compose.yml')]
|
|
code, output = run(['docker', 'compose', *files, *args], env, 180)
|
|
if code:
|
|
proof = write_log(PRIVATE / 'logs', ['docker', 'compose', *args], output, 'failed')
|
|
raise RuntimeError('local dependency command failed; ' + proof['log'])
|
|
return output
|
|
|
|
|
|
def start(env):
|
|
# Existing unowned listeners must not be replaced or stopped.
|
|
busy = []
|
|
for port in (27119, 27129):
|
|
try:
|
|
with socket.create_connection(('127.0.0.1', port), timeout=.3):
|
|
busy.append(port)
|
|
except OSError:
|
|
pass
|
|
if busy:
|
|
running = compose(env, 'ps', '--status', 'running', '--services').splitlines()
|
|
required = {27119: 'source', 27129: 'query'}
|
|
if any(required[port] not in running for port in busy):
|
|
raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports')
|
|
if len(busy) == 2:
|
|
# Reuse isolated random databases, never recreate a shared fixture just
|
|
# because another authorized checkout owns its Compose metadata.
|
|
info = json.loads(subprocess.check_output(['docker', 'inspect', 'fanyu-query-replica-local-source-1',
|
|
'fanyu-query-replica-local-query-1'], cwd=ROOT, env=env, text=True))
|
|
resources = [{'name': x['Name'], 'cpu': x['HostConfig']['NanoCpus'],
|
|
'memory': x['HostConfig']['Memory']} for x in info]
|
|
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
|
|
query = next(x for x in resources if x['name'].endswith('query-1'))
|
|
if query['cpu'] != 1_000_000_000 or query['memory'] != 1024 ** 3:
|
|
raise RuntimeError('existing shared query fixture differs from1CPU/1Gi; do not recreate it')
|
|
env['QUERY_REPLICA_REUSED_RESOURCES'] = json.dumps(resources)
|
|
compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
|
|
"if (!db.hello().isWritablePrimary || db.hello().setName !== 'query-read-rs') quit(2)")
|
|
return
|
|
compose(env, 'up', '-d')
|
|
for _ in range(60):
|
|
try:
|
|
result = compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
|
|
"try {rs.status()} catch(e) {rs.initiate({_id:'query-read-rs',members:[{_id:0,host:'127.0.0.1:27129'}]})}; if (!db.hello().isWritablePrimary) quit(2)")
|
|
if result:
|
|
return
|
|
except RuntimeError:
|
|
time.sleep(1)
|
|
raise RuntimeError('query replica did not become primary')
|
|
|
|
|
|
def case_batches(required, bounded, native):
|
|
names = sorted(required)
|
|
if not names:
|
|
raise RuntimeError('no registered replica cases')
|
|
size = 20 if bounded and not native else len(names)
|
|
return [names[i:i + size] for i in range(0, len(names), size)]
|
|
|
|
|
|
def query_probe(env, script):
|
|
# Fixed loopback, no credentials, four-second process bound. This is not
|
|
# an arbitrary database command entry point or a production probe.
|
|
return run(['docker', 'exec', 'fanyu-query-replica-local-query-1',
|
|
'mongosh', '--quiet',
|
|
'mongodb://127.0.0.1:27129/?serverSelectionTimeoutMS=1000&connectTimeoutMS=1000&appName=droboticflow-query-fixture-probe&maxPoolSize=1&maxConnecting=1',
|
|
'--eval', script], env, 4)
|
|
|
|
|
|
def restart_query_fixture(env, probe, cursor):
|
|
name = 'fanyu-query-replica-local-query-1'
|
|
before = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0]
|
|
if (before['Name'] != '/' + name or before['HostConfig']['Memory'] != 1024 ** 3
|
|
or before['HostConfig']['NanoCpus'] != 1_000_000_000
|
|
or before['Config']['Labels'].get('com.docker.compose.project') != 'fanyu-query-replica-local'
|
|
or not any(m['Destination'] == '/data/db' for m in before['Mounts'])):
|
|
raise RuntimeError('refusing to restart a different query fixture')
|
|
# A direct client from another checkout must not be interrupted. Exclude only this named probe and the replica set internal client;
|
|
# wait briefly for completed test clients and in-progress handshakes to close.
|
|
for attempt in range(3):
|
|
code, _ = query_probe(env,
|
|
'sleep(200);'
|
|
'const clients=db.getSiblingDB("admin").aggregate(['
|
|
'{$currentOp:{allUsers:true,idleConnections:true}},'
|
|
'{$match:{client:{$exists:true},"clientMetadata.application.name":{$ne:"droboticflow-query-fixture-probe"},'
|
|
'"clientMetadata.driver.name":{$not:/^NetworkInterfaceTL/}}},'
|
|
'{$limit:1}]).toArray();if(clients.length) quit(3)')
|
|
if code == 0:
|
|
break
|
|
if attempt == 2:
|
|
raise RuntimeError('query restart admission failed: ' + ('other_clients' if code == 3 else 'probe_exit_' + str(code)))
|
|
time.sleep(.5)
|
|
owner = json.dumps(probe)
|
|
code, _ = query_probe(env, 'db.getSiblingDB(' + owner + ').checkpoint.updateOne({_id:"progress"},'
|
|
'{$set:{cursor:' + str(cursor) + ',fingerprint:"synthetic-majority-journal"}},'
|
|
'{upsert:true,writeConcern:{w:"majority",j:true}})')
|
|
if code:
|
|
raise RuntimeError('restart canary persistence failed')
|
|
compose(env, 'restart', '-t', '30', 'query')
|
|
after = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0]
|
|
if (after['Id'] != before['Id']
|
|
or sorted(after['Mounts'], key=lambda m: json.dumps(m, sort_keys=True))
|
|
!= sorted(before['Mounts'], key=lambda m: json.dumps(m, sort_keys=True))
|
|
or after['HostConfig']['Memory'] != before['HostConfig']['Memory']
|
|
or after['HostConfig']['NanoCpus'] != before['HostConfig']['NanoCpus']):
|
|
raise RuntimeError('query restart changed volume or resource identity')
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
code, _ = query_probe(env, 'if(!db.hello().isWritablePrimary||db.hello().setName!=="query-read-rs") quit(2);'
|
|
'const p=db.getSiblingDB(' + owner + ').checkpoint.findOne({_id:"progress"});'
|
|
'if(!p||p.cursor!==' + str(cursor) + '||p.fingerprint!=="synthetic-majority-journal") quit(4)')
|
|
if code == 0:
|
|
return {'container': before['Id'], 'cursor': cursor, 'majorityJournalRecovered': True,
|
|
'sameVolume': True, 'sameResources': True}
|
|
time.sleep(.5)
|
|
raise RuntimeError('query restart readiness or durable canary recovery failed')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--native', action='store_true')
|
|
parser.add_argument('--start', action='store_true')
|
|
parser.add_argument('--auto', action='store_true')
|
|
parser.add_argument('--down', action='store_true')
|
|
parser.add_argument('--race', action='store_true')
|
|
parser.add_argument('--scale', action='store_true')
|
|
parser.add_argument('--cache', action='store_true')
|
|
parser.add_argument('--bounded-query', action='store_true', help='functional tests with a 1CPU/1Gi query replica')
|
|
args = parser.parse_args()
|
|
PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
env = {k: v for k, v in check_environment().items() if not k.startswith('QUERY_')}
|
|
env['QUERY_REPLICA_LOCAL_TEST'] = '1'
|
|
if args.bounded_query:
|
|
env['QUERY_REPLICA_LOCAL_QUERY_BUDGET'] = 'query25'
|
|
before = source_fingerprint(ROOT)
|
|
with (PRIVATE / 'runner.lock').open('w') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if args.down:
|
|
compose(env, 'down')
|
|
return 0
|
|
native = args.native or (args.auto and owned_native())
|
|
if native:
|
|
if not owned_native():
|
|
raise RuntimeError('native mode requires this worktree\'s two owned Mongo processes')
|
|
elif args.start or args.auto:
|
|
start(env)
|
|
env['QUERY_REPLICA_DOCKER_LAB'] = '1'
|
|
else:
|
|
raise RuntimeError('select --native, --start or --auto explicitly')
|
|
if args.cache:
|
|
if 'valkey' not in compose(env, 'ps', '--status', 'running', '--services').splitlines():
|
|
compose(env, 'up', '-d', 'valkey')
|
|
env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1'
|
|
required = set()
|
|
for name in CASES:
|
|
required.update(re.findall(r'^func (Test\w+)\(', (ROOT / name).read_text(), re.M))
|
|
if not args.cache:
|
|
required = {name for name in required if not name.startswith('TestQueryReplicaSharedValkey')}
|
|
if args.scale:
|
|
required = {'TestQueryReplicaScaleLocal'}
|
|
env['QUERY_REPLICA_SCALE_LOCAL'] = '1'
|
|
batches = case_batches(required, args.bounded_query, native)
|
|
outputs, restart_proofs, commands = [], [], []
|
|
code = 0
|
|
probe = 'query_replica_restart_probe_' + uuid.uuid4().hex
|
|
for index, names in enumerate(batches):
|
|
if len(batches) > 1:
|
|
restart_proofs.append(restart_query_fixture(env, probe, index))
|
|
regex = '^(' + '|'.join(names) + ')$'
|
|
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
|
|
'-run', regex, '-count=1', '-timeout=10m', '-v']
|
|
if args.race:
|
|
argv.append('-race')
|
|
commands.append(argv)
|
|
code, output = run(argv, env)
|
|
outputs.append(output)
|
|
if code:
|
|
break
|
|
output = '\n'.join(outputs)
|
|
|
|
passed = set(re.findall(r'^--- PASS: (Test\w+)', output, re.M))
|
|
missing = sorted(required - passed)
|
|
changed = not before or source_fingerprint(ROOT) != before
|
|
skipped = bool(re.search(r'^--- SKIP:', output, re.M))
|
|
success = code == 0 and not missing and not changed and not skipped
|
|
proof = write_log(PRIVATE / 'logs', [sys.executable, str(Path(__file__).resolve()), *sys.argv[1:]], output, 'passed' if success else 'pending_or_failed')
|
|
proof.update(missing=missing, sourceChanged=changed, skipped=skipped,
|
|
environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'),
|
|
productionCapacityAccepted=False, boundedBatches=len(batches),
|
|
continuousCapacityAccepted=False, restartProofs=restart_proofs, requiredCount=len(required),
|
|
passedCount=len(passed), batchCommands=commands, completedBatches=len(outputs),
|
|
reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
|
|
print(json.dumps(proof, ensure_ascii=False))
|
|
return 0 if success else (code or 2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
raise SystemExit(main())
|
|
except (RuntimeError, OSError, subprocess.SubprocessError) as error:
|
|
print(str(error))
|
|
raise SystemExit(2)
|