forked from longtao.wu/droboticflow
Containers started on the internal bridge get no external nameservers, so a later egress attach could not resolve bj.bcebos.com and the BOS preflight failed with a client error before any object was written. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
476 lines
27 KiB
Python
476 lines
27 KiB
Python
#!/usr/bin/env -S python3 -I
|
|
"""Run the published Linux client against a disposable current API, Mongo and bounded real BOS fixtures."""
|
|
import argparse
|
|
import base64
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
import urllib.parse
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PACKAGE = ROOT / 'output/drosync-1.1.6-linux-x86_64/drosync-1.1.6-linux-x86_64.tar.gz'
|
|
PACKAGE_SHA256 = 'c73bf0014f677985a4889249bed0db3961af4adbe0511f39611fd5672e40a08f'
|
|
TOOLS_IMAGE = 'drosync-portable-test-tools:20261031'
|
|
MONGO_IMAGE = 'mongo:8.0.17'
|
|
LABEL = 'drosync.acceptance.run'
|
|
# Containers start on the internal bridge, so Docker's embedded resolver needs explicit upstreams for BOS.
|
|
DNS_ARGS = ['--dns', '223.5.5.5', '--dns', '114.114.114.114']
|
|
NO_PROXY_ENV = [part for name in ['HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'] for part in ['-e', name + '=']]
|
|
ALLOWED_MEMBERS = {
|
|
'README-systemd-portable.md', 'drosync', 'drosync-daemon', 'drosync-portable.service',
|
|
'drosync.build.json', 'environment.example', 'install-systemd-portable.sh',
|
|
'uninstall-systemd-portable.sh',
|
|
}
|
|
BOOTSTRAP = r'''
|
|
import hashlib,json,os,subprocess,sys,tarfile
|
|
from pathlib import Path,PurePosixPath
|
|
run=sys.argv[1]
|
|
assert len(run)==24 and all(c in '0123456789abcdef' for c in run)
|
|
assert os.geteuid()==0 and Path('/proc/1/comm').read_text().strip()=='systemd'
|
|
subprocess.run(['systemctl','stop','drosync.service'],capture_output=True,timeout=30)
|
|
archive=Path('/fixtures/bundle.tar.gz')
|
|
assert hashlib.sha256(archive.read_bytes()).hexdigest()==sys.argv[2]
|
|
stage=Path('/tmp/installed-bootstrap-'+run);stage.mkdir(mode=0o700)
|
|
allowed={'README-systemd-portable.md','drosync','drosync-daemon','drosync-portable.service','drosync.build.json','environment.example','install-systemd-portable.sh','uninstall-systemd-portable.sh'}
|
|
seen=set()
|
|
with tarfile.open(archive,'r:gz') as package:
|
|
members=package.getmembers()
|
|
assert len(members)<=16 and sum(m.size for m in members)<=64*1024*1024
|
|
for member in members:
|
|
parts=PurePosixPath(member.name).parts
|
|
assert parts and parts[0]=='drosync-1.1.6' and '..' not in parts and not member.name.startswith('/')
|
|
if member.isdir():
|
|
assert len(parts)==1
|
|
continue
|
|
assert member.isfile() and len(parts)==2 and parts[1] in allowed and parts[1] not in seen
|
|
seen.add(parts[1]);target=stage/parts[1]
|
|
with package.extractfile(member) as source,target.open('xb') as output:
|
|
while block:=source.read(1024*1024):output.write(block)
|
|
target.chmod(0o755 if parts[1] in {'drosync','drosync-daemon','install-systemd-portable.sh','uninstall-systemd-portable.sh'} else 0o644)
|
|
assert seen==allowed
|
|
assert json.loads((stage/'drosync.build.json').read_text())['version']=='1.1.6'
|
|
state='/home/ubt/installed-state-'+run
|
|
assert not Path(state).exists()
|
|
Path('/etc/drosync').mkdir(exist_ok=True)
|
|
Path('/etc/drosync/environment').write_text('DROSYNC_DATA_DIR='+state+'\nDROSYNC_SECRET_FILE='+state+'/device-token\nDROSYNC_LOG_PATH='+state+'/daemon.log\nDROSYNC_FILE_CONCURRENCY=2\nDROSYNC_PUT_CONCURRENCY=1\nDROSYNC_PART_CONCURRENCY=2\nDROSYNC_MAPPING_CONCURRENCY=1\nDROSYNC_HASH_CONCURRENCY=1\n')
|
|
subprocess.run(['bash',str(stage/'install-systemd-portable.sh'),'--enable'],check=True,timeout=60,cwd='/tmp')
|
|
'''
|
|
|
|
|
|
def safe(text, values=()):
|
|
if isinstance(text, bytes):
|
|
text = text.decode(errors='replace')
|
|
for value in values:
|
|
if value:
|
|
text = text.replace(value, '[REDACTED]')
|
|
text = re.sub(r'(?i)Bearer\s+[^\s"\'<>]+', 'Bearer [REDACTED]', text)
|
|
text = re.sub(r'bce-auth-v1/[^\s"\'<>]+', '[REDACTED-BCE-SIGNATURE]', text)
|
|
text = re.sub(r'eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+', '[REDACTED-TOKEN]', text)
|
|
def redact_url(match):
|
|
try:
|
|
url = urllib.parse.urlsplit(match.group())
|
|
host = url.netloc.rsplit('@', 1)[-1]
|
|
return urllib.parse.urlunsplit((url.scheme, host, url.path, '', ''))
|
|
except ValueError:
|
|
return '[REDACTED-URL]'
|
|
return re.sub(r'https?://[^\s"\'<>]+', redact_url, text)
|
|
|
|
|
|
def validate_package(package, expected=PACKAGE_SHA256):
|
|
if package.is_symlink() or not package.is_file() or package.stat().st_size > 64 * 1024 * 1024:
|
|
raise RuntimeError('release must be the bounded regular package file')
|
|
digest = hashlib.sha256(package.read_bytes()).hexdigest()
|
|
if digest != expected:
|
|
raise RuntimeError('published package SHA256 mismatch')
|
|
seen = set()
|
|
with tarfile.open(package, 'r:gz') as archive:
|
|
members = archive.getmembers()
|
|
if len(members) > 16 or sum(m.size for m in members) > 64 * 1024 * 1024:
|
|
raise RuntimeError('release archive exceeds its expected shape')
|
|
for member in members:
|
|
parts = PurePosixPath(member.name).parts
|
|
if not parts or parts[0] != 'drosync-1.1.6' or '..' in parts or member.name.startswith('/'):
|
|
raise RuntimeError('unsafe archive path')
|
|
if member.isdir() and len(parts) == 1:
|
|
continue
|
|
if not member.isfile() or len(parts) != 2 or parts[1] not in ALLOWED_MEMBERS or parts[1] in seen:
|
|
raise RuntimeError('unexpected, duplicate or linked archive member')
|
|
seen.add(parts[1])
|
|
if seen != ALLOWED_MEMBERS:
|
|
raise RuntimeError('release archive is incomplete')
|
|
return digest
|
|
|
|
|
|
class AcceptanceRun:
|
|
def __init__(self, output):
|
|
self.run_id = secrets.token_hex(12)
|
|
parent = output.parent.resolve()
|
|
if parent != (ROOT / 'output').resolve() or not output.name.startswith('drosync-installed-') or output.is_symlink():
|
|
raise RuntimeError('use a new output/drosync-installed-* directory')
|
|
self.out = parent / output.name
|
|
self.out.mkdir(mode=0o700, exist_ok=False)
|
|
self.fixtures = self.out / 'fixtures'
|
|
self.fixtures.mkdir(mode=0o755)
|
|
self.evidence = self.out / 'evidence'
|
|
self.evidence.mkdir(mode=0o700)
|
|
self.main = 'drosync-installed-client-' + self.run_id
|
|
self.mongo = 'drosync-installed-mongo-' + self.run_id
|
|
self.network = 'drosync-installed-net-' + self.run_id
|
|
self.egress = 'drosync-installed-egress-' + self.run_id
|
|
self.cleanup_container = 'drosync-installed-cleanup-' + self.run_id
|
|
self.owned_containers = {}
|
|
self.owned_networks = {}
|
|
self.requested_containers = set()
|
|
self.requested_networks = set()
|
|
self.private_values = []
|
|
self.config = None
|
|
self.deadline = time.monotonic() + 30 * 60
|
|
self.record = {
|
|
'status': 'PREPARING', 'runId': self.run_id, 'startedAtUTC': self.now(),
|
|
'bucket': 'testxyz', 'prefix': 'drosync-installed-acceptance/' + self.run_id + '/',
|
|
'endpoint': 'https://bj.bcebos.com', 'tests': [], 'cleanup': [],
|
|
'productionServicesModified': False, 'fieldClientUpgraded': False,
|
|
'bucketACLChanged': False, 'newReleaseBuilt': False,
|
|
'credentialRedactionTriggered': False,
|
|
}
|
|
self.save()
|
|
|
|
@staticmethod
|
|
def now():
|
|
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
|
|
def save(self):
|
|
path = self.out / 'run.json'
|
|
path.write_text(json.dumps(self.record, indent=2) + '\n')
|
|
path.chmod(0o600)
|
|
|
|
def call(self, args, timeout=60, data=None, log=None, required=True, cleanup=False, env=None):
|
|
remaining = self.deadline - time.monotonic() - (0 if cleanup else 180)
|
|
if remaining <= 0 or timeout <= 0:
|
|
raise RuntimeError('acceptance work deadline exceeded')
|
|
try:
|
|
result = subprocess.run(args, cwd=ROOT, input=data, capture_output=True,
|
|
timeout=min(timeout, remaining), env=env)
|
|
except subprocess.TimeoutExpired as error:
|
|
result = subprocess.CompletedProcess(args, 124, error.stdout or b'', error.stderr or b'')
|
|
raw = result.stdout + result.stderr
|
|
if any(value.encode() in raw for value in self.private_values if value):
|
|
self.record['credentialRedactionTriggered'] = True
|
|
if log:
|
|
destination = self.out / log
|
|
destination.write_text(safe(raw, self.private_values))
|
|
destination.chmod(0o600)
|
|
if required and result.returncode:
|
|
raise RuntimeError('command failed; inspect redacted ' + (log or 'run evidence'))
|
|
return result
|
|
|
|
def docker(self, *args, **kwargs):
|
|
return self.call(['docker', '--context', 'colima', *args], **kwargs)
|
|
|
|
def inspect(self, kind, name, cleanup=False):
|
|
return json.loads(self.docker(kind, 'inspect', name, cleanup=cleanup).stdout)[0]
|
|
|
|
def preflight(self):
|
|
package_sha = validate_package(PACKAGE)
|
|
host_free = shutil.disk_usage(ROOT).free
|
|
disk = self.call(['colima', 'ssh', '--', 'df', '-Pk', '/var/lib/docker']).stdout.decode()
|
|
vm_free = int(disk.strip().splitlines()[-1].split()[3]) * 1024
|
|
if host_free < 8 << 30 or vm_free < 4 << 30:
|
|
raise RuntimeError('insufficient headroom for bounded installed acceptance')
|
|
images = {}
|
|
for image in [TOOLS_IMAGE, MONGO_IMAGE]:
|
|
info = self.inspect('image', image)
|
|
if image == TOOLS_IMAGE and info['Architecture'] != 'amd64':
|
|
raise RuntimeError('systemd fixture must be Linux amd64')
|
|
images[image] = {'id': info['Id'], 'architecture': info['Architecture'], 'repoDigests': info.get('RepoDigests', [])}
|
|
self.record['preflight'] = {'packageSHA256': package_sha, 'hostFreeBytes': host_free, 'vmFreeBytes': vm_free, 'images': images}
|
|
self.record['sourceCommit'] = self.call(['git', 'rev-parse', 'HEAD']).stdout.decode().strip()
|
|
self.record['sourceDirty'] = bool(self.call(['git', 'diff', '--name-only']).stdout.strip())
|
|
self.save()
|
|
|
|
def prepare(self):
|
|
source_paths = list((ROOT / 'internal').rglob('*.go')) + [ROOT / name for name in ['go.mod', 'go.sum', 'scripts/drosync-installed-storage-acceptance.py', 'scripts/drosync-ubuntu22-installed-acceptance.py']]
|
|
self.record['sourceSHA256'] = {str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(source_paths)}
|
|
env = dict(os.environ, CGO_ENABLED='0', GOOS='linux', GOARCH='amd64')
|
|
self.call(['go', 'test', '-c', '-o', str(self.fixtures / 'api.test'), './internal/api'], timeout=180, log='build.txt', env=env)
|
|
shutil.copyfile(PACKAGE, self.fixtures / 'bundle.tar.gz')
|
|
shutil.copyfile(ROOT / 'scripts/drosync-ubuntu22-installed-acceptance.py', self.fixtures / 'drosync-ubuntu22-installed-acceptance.py')
|
|
shutil.copyfile(ROOT / 'doc/drosync-1.1.5-linux-x86_64-evidence/start-systemd-private-remount.sh', self.fixtures / 'start-systemd.sh')
|
|
(self.fixtures / 'bootstrap.py').write_text(BOOTSTRAP)
|
|
self.record['fixtureSHA256'] = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in self.fixtures.iterdir()}
|
|
self.save()
|
|
for network, internal in [(self.network, True), (self.egress, False)]:
|
|
self.reserve_name('network', network)
|
|
args = ['network', 'create', '--label', LABEL + '=' + self.run_id]
|
|
if internal:
|
|
args.append('--internal')
|
|
result = self.docker(*args, network)
|
|
self.owned_networks[network] = result.stdout.decode().strip()
|
|
self.record['networks'] = self.owned_networks.copy()
|
|
self.save()
|
|
mongo_args = ['run', '-d', '--name', self.mongo, '--label', LABEL + '=' + self.run_id,
|
|
'--network', self.network, '--network-alias', self.mongo,
|
|
'--cpus', '1', '--memory', '2g', '--pids-limit', '128', '--ulimit', 'nofile=64000:64000',
|
|
'--log-opt', 'max-size=2m', '--log-opt', 'max-file=2',
|
|
'--tmpfs', '/data/db:rw,size=512m', '--tmpfs', '/data/configdb:rw,size=16m',
|
|
MONGO_IMAGE, '--bind_ip_all', '--wiredTigerCacheSizeGB', '0.25', '--setParameter', 'diagnosticDataCollectionEnabled=false']
|
|
self.start_container(self.mongo, mongo_args)
|
|
main_args = ['run', '-d', '--name', self.main, '--label', LABEL + '=' + self.run_id,
|
|
'--platform', 'linux/amd64', '--network', self.network,
|
|
'--hostname', 'drosync-installed-test', '--add-host', 'drosync-installed-test:127.0.0.1',
|
|
'--cgroupns=private', '--cpus', '2', '--memory', '2g', '--cap-add', 'SYS_ADMIN',
|
|
'--security-opt', 'apparmor=unconfined', '--tmpfs', '/run:rw,size=64m', '--tmpfs', '/run/lock:rw,size=8m',
|
|
'--stop-signal', 'SIGRTMIN+3', '--log-opt', 'max-size=2m', '--log-opt', 'max-file=2',
|
|
*DNS_ARGS, *NO_PROXY_ENV,
|
|
'-v', str(self.fixtures) + ':/fixtures:ro', '-v', str(self.evidence) + ':/evidence',
|
|
'--entrypoint', '/bin/bash', TOOLS_IMAGE, '/fixtures/start-systemd.sh']
|
|
self.start_container(self.main, main_args)
|
|
until = time.monotonic() + 60
|
|
while True:
|
|
state = self.docker('exec', self.main, 'systemctl', 'is-system-running', '--wait', required=False, timeout=10)
|
|
if state.returncode == 0 and state.stdout.strip() == b'running':
|
|
break
|
|
if time.monotonic() >= until:
|
|
raise RuntimeError('isolated systemd did not become ready')
|
|
time.sleep(0.25)
|
|
ready = self.docker('exec', self.mongo, 'mongosh', '--quiet', '--eval', 'print(JSON.stringify({version:db.version(),ping:db.adminCommand({ping:1}).ok}))')
|
|
mongo_state = json.loads(ready.stdout)
|
|
if mongo_state != {'version': '8.0.17', 'ping': 1}:
|
|
raise RuntimeError('unexpected isolated Mongo version/readiness')
|
|
self.record['mongoStart'] = mongo_state
|
|
self.docker('exec', self.main, 'python3', '-I', '/fixtures/bootstrap.py', self.run_id, PACKAGE_SHA256, timeout=120, log='bootstrap.txt')
|
|
until = time.monotonic() + 30
|
|
while True:
|
|
version = self.docker('exec', self.main, 'runuser', '-u', 'ubt', '--', '/opt/drosync/drosync', '--json', 'version', required=False, timeout=10)
|
|
try:
|
|
value = json.loads(version.stdout)
|
|
if version.returncode == 0 and value['version'] == value['daemon']['version'] == '1.1.6' and value['daemon']['installationFormat'] == 'portable':
|
|
self.record['installedVersion'] = value
|
|
break
|
|
except (ValueError, KeyError, TypeError):
|
|
pass
|
|
if time.monotonic() >= until:
|
|
raise RuntimeError('installed CLI/daemon identity did not converge')
|
|
time.sleep(0.25)
|
|
self.docker('exec', self.main, '/opt/drosync/drosync', '--json', 'update', 'pause', log='update-pause.txt')
|
|
self.docker('network', 'connect', self.egress, self.main)
|
|
self.record['networkIsolation'] = {'control': '127.0.0.1 inside the installed container', 'mongo': 'dedicated internal bridge without host ports', 'storage': 'dedicated egress bridge attached only after installing the fresh state namespace'}
|
|
self.save()
|
|
|
|
def reserve_name(self, kind, name, cleanup=False):
|
|
listing = ['container', 'ls', '-a'] if kind == 'container' else ['network', 'ls']
|
|
if self.docker(*listing, '--filter', 'name=^' + name + '$', '--format', '{{.ID}}', cleanup=cleanup).stdout.strip():
|
|
raise RuntimeError('planned resource name already exists; nothing replaced')
|
|
requested = self.requested_containers if kind == 'container' else self.requested_networks
|
|
requested.add(name)
|
|
self.record['requestedContainers'] = sorted(self.requested_containers)
|
|
self.record['requestedNetworks'] = sorted(self.requested_networks)
|
|
self.save()
|
|
|
|
def adopt_requested(self):
|
|
failures = []
|
|
for kind, requested, owned in [('container', self.requested_containers, self.owned_containers), ('network', self.requested_networks, self.owned_networks)]:
|
|
for name in requested - owned.keys():
|
|
try:
|
|
listing = ['container', 'ls', '-a'] if kind == 'container' else ['network', 'ls']
|
|
exists = self.docker(*listing, '--filter', 'name=^' + name + '$', '--format', '{{.ID}}', cleanup=True).stdout.strip()
|
|
if not exists:
|
|
continue
|
|
info = self.inspect(kind, name, cleanup=True)
|
|
labels = info['Config'].get('Labels', {}) if kind == 'container' else info.get('Labels', {})
|
|
if labels.get(LABEL) != self.run_id:
|
|
raise RuntimeError('unexpected resource owner')
|
|
owned[name] = info['Id']
|
|
except Exception:
|
|
failures.append('resource-ownership-unconfirmed:' + name)
|
|
return failures
|
|
|
|
def start_container(self, name, args):
|
|
self.reserve_name('container', name)
|
|
result = self.docker(*args)
|
|
self.owned_containers[name] = result.stdout.decode().strip()
|
|
self.record['containers'] = self.owned_containers.copy()
|
|
self.save()
|
|
|
|
def load_credentials(self):
|
|
result = self.call(['kubectl', '--kubeconfig', str(Path.home() / '.config/droboticflow/kubeconfigs/poc.config'),
|
|
'--request-timeout=20s', '-n', 'droboticflow', 'get', 'secret', 'droboticflow-bos-workspace', '-o', 'json'], timeout=30)
|
|
secret = json.loads(result.stdout)['data']
|
|
credentials = {key: base64.b64decode(secret[key], validate=True).decode('ascii') for key in ['accessKeyId', 'secretAccessKey']}
|
|
if any(not value or len(value) > 512 or not value.isprintable() for value in credentials.values()):
|
|
raise RuntimeError('invalid credential encoding; values suppressed')
|
|
self.private_values = list(credentials.values())
|
|
self.config = dict(credentials, bucket='testxyz', endpoint='https://bj.bcebos.com',
|
|
prefix=self.record['prefix'], runId=self.run_id, outputDir='/evidence',
|
|
mongoURI='mongodb://' + self.mongo + ':27017/?directConnection=true')
|
|
secret.clear()
|
|
credentials.clear()
|
|
|
|
def run_test(self, name, timeout):
|
|
data = bytearray(json.dumps(self.config).encode())
|
|
try:
|
|
result = self.docker('exec', '-i', '-e', 'DROSYNC_INSTALLED_STORAGE_ACCEPTANCE=1', self.main,
|
|
'/fixtures/api.test', '-test.run', '^' + name + '$', '-test.v', '-test.timeout', '23m',
|
|
data=data, timeout=timeout, required=False, log=name + '.txt')
|
|
finally:
|
|
data[:] = b'\0' * len(data)
|
|
entry = {'test': name, 'exitCode': result.returncode, 'finishedAtUTC': self.now()}
|
|
self.record['tests'].append(entry)
|
|
self.save()
|
|
if result.returncode or ('--- PASS: ' + name + ' (').encode() not in result.stdout:
|
|
raise RuntimeError('real acceptance test failed or did not execute; inspect redacted evidence')
|
|
cleanup = json.loads((self.evidence / 'bos-cleanup.json').read_text())
|
|
if cleanup.get('status') != 'PASS':
|
|
raise RuntimeError('test did not confirm exact BOS cleanup')
|
|
if name == 'TestDirectUploadRealBOS':
|
|
shutil.copyfile(self.evidence / 'bos-cleanup.json', self.evidence / 'direct-cleanup.json')
|
|
else:
|
|
for file in ['installed-result.json', 'installed-cleanup.json']:
|
|
if json.loads((self.evidence / file).read_text()).get('status') != 'PASS':
|
|
raise RuntimeError('installed test result/cleanup is incomplete')
|
|
|
|
def stop_owned(self, name):
|
|
info = self.inspect('container', name, cleanup=True)
|
|
if info['Id'] != self.owned_containers[name] or info['Config'].get('Labels', {}).get(LABEL) != self.run_id:
|
|
raise RuntimeError('container identity differs; preserved rather than stopped')
|
|
self.docker('stop', '--time', '10', name, cleanup=True, timeout=25)
|
|
|
|
def recover_cloud(self):
|
|
if self.config is None or not (self.evidence / 'bos-ledger.json').exists():
|
|
return
|
|
cleanup_path = self.evidence / 'bos-cleanup.json'
|
|
if self.record['status'] == 'PASS' and cleanup_path.exists() and json.loads(cleanup_path.read_text()).get('status') == 'PASS':
|
|
return
|
|
self.reserve_name('container', self.cleanup_container, cleanup=True)
|
|
args = ['run', '--rm', '-i', '--name', self.cleanup_container, '--label', LABEL + '=' + self.run_id,
|
|
'--platform', 'linux/amd64', '--network', self.egress, '--cpus', '1', '--memory', '512m',
|
|
'--read-only', '--tmpfs', '/tmp:rw,size=16m', *DNS_ARGS, *NO_PROXY_ENV, '-v', str(self.fixtures) + ':/fixtures:ro',
|
|
'-v', str(self.evidence) + ':/evidence', '-e', 'DROSYNC_INSTALLED_STORAGE_ACCEPTANCE=1',
|
|
'-e', 'DROSYNC_INSTALLED_CLEANUP_ONLY=1', '--entrypoint', '/fixtures/api.test', TOOLS_IMAGE,
|
|
'-test.run', '^TestBOSAcceptanceCleanupOnly$', '-test.v', '-test.timeout', '130s']
|
|
payload = bytearray(json.dumps(self.config).encode())
|
|
try:
|
|
result = self.docker(*args, data=payload, timeout=145, required=False, log='recovery-cleanup.txt', cleanup=True)
|
|
finally:
|
|
payload[:] = b'\0' * len(payload)
|
|
if result.returncode or b'--- PASS: TestBOSAcceptanceCleanupOnly (' not in result.stdout or not cleanup_path.exists() or json.loads(cleanup_path.read_text()).get('status') != 'PASS':
|
|
raise RuntimeError('BOS cleanup could not be confirmed; exact ledger retained')
|
|
self.record['cleanup'].append({'step': 'independent-ledger-recovery', 'status': 'PASS'})
|
|
|
|
def cleanup(self):
|
|
failures = self.adopt_requested()
|
|
writers_stopped = self.main not in self.owned_containers
|
|
if not writers_stopped:
|
|
try:
|
|
self.stop_owned(self.main)
|
|
writers_stopped = True
|
|
except Exception:
|
|
failures.append('installed-container-stop-unconfirmed')
|
|
if writers_stopped:
|
|
try:
|
|
self.recover_cloud()
|
|
except Exception:
|
|
failures.append('exact-BOS-cleanup-unconfirmed')
|
|
if self.mongo in self.owned_containers:
|
|
try:
|
|
result = self.docker('exec', self.mongo, 'mongosh', '--quiet', '--eval', 'const r=db.adminCommand({listDatabases:1,nameOnly:true});if(r.ok!==1)quit(2);print(JSON.stringify(r.databases.map(d=>d.name)));', cleanup=True)
|
|
names = json.loads(result.stdout)
|
|
self.record['databasesBeforeContainerRemoval'] = names
|
|
if set(names) - {'admin', 'config', 'local'}:
|
|
failures.append('test-database-remained-until-container-removal')
|
|
except Exception:
|
|
failures.append('final-Mongo-read-unconfirmed')
|
|
failures.extend(self.adopt_requested())
|
|
for name in list(self.owned_containers):
|
|
try:
|
|
self.stop_owned(name)
|
|
self.docker('rm', name, cleanup=True)
|
|
self.record['cleanup'].append({'container': name, 'removed': True})
|
|
except Exception:
|
|
failures.append('container-removal-unconfirmed:' + name)
|
|
for name, identity in self.owned_networks.items():
|
|
try:
|
|
info = self.inspect('network', name, cleanup=True)
|
|
if info['Id'] != identity or info.get('Labels', {}).get(LABEL) != self.run_id:
|
|
raise RuntimeError('network identity differs')
|
|
self.docker('network', 'rm', name, cleanup=True)
|
|
self.record['cleanup'].append({'network': name, 'removed': True})
|
|
except Exception:
|
|
failures.append('network-removal-unconfirmed:' + name)
|
|
changed_sources = [name for name, digest in self.record.get('sourceSHA256', {}).items() if not (ROOT / name).is_file() or hashlib.sha256((ROOT / name).read_bytes()).hexdigest() != digest]
|
|
self.record['sourceChangedDuringRun'] = changed_sources
|
|
if changed_sources:
|
|
failures.append('source-changed-during-run')
|
|
self.record['cleanupFailures'] = failures
|
|
self.record['cleanupVerified'] = not failures
|
|
if failures or self.record['credentialRedactionTriggered']:
|
|
self.record['status'] = 'FAIL'
|
|
self.record['finishedAtUTC'] = self.now()
|
|
self.save()
|
|
if self.config:
|
|
self.config.clear()
|
|
self.private_values.clear()
|
|
|
|
def execute(self, preflight_only=False):
|
|
try:
|
|
self.preflight()
|
|
if preflight_only:
|
|
self.record['status'] = 'PREFLIGHT_PASS'
|
|
return
|
|
self.prepare()
|
|
self.load_credentials()
|
|
work_deadline = min(time.monotonic() + 20 * 60, self.deadline - 180)
|
|
self.record['status'] = 'RUNNING'
|
|
self.save()
|
|
self.run_test('TestDirectUploadRealBOS', min(240, work_deadline - time.monotonic()))
|
|
self.run_test('TestUbuntu22InstalledRealUpload', work_deadline - time.monotonic())
|
|
self.record['status'] = 'PASS'
|
|
except BaseException as error:
|
|
self.record['status'] = 'FAIL'
|
|
self.record['failureType'] = type(error).__name__
|
|
self.record['failure'] = safe(str(error), self.private_values) if isinstance(error, RuntimeError) else 'acceptance did not finish; raw exception suppressed'
|
|
finally:
|
|
previous = {number: signal.getsignal(number) for number in (signal.SIGINT, signal.SIGTERM)}
|
|
def defer_interrupt(_number, _frame):
|
|
self.record['interruptedDuringCleanup'] = True
|
|
self.record['status'] = 'FAIL'
|
|
try:
|
|
for number in previous:
|
|
signal.signal(number, defer_interrupt)
|
|
self.cleanup()
|
|
finally:
|
|
for number, handler in previous.items():
|
|
signal.signal(number, handler)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output', type=Path, required=True, help='new output/drosync-installed-* directory')
|
|
parser.add_argument('--preflight', action='store_true', help='verify cached assets and capacity without credentials, builds or containers')
|
|
args = parser.parse_args()
|
|
def interrupted(_number, _frame):
|
|
raise KeyboardInterrupt()
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
run = AcceptanceRun(args.output)
|
|
run.execute(args.preflight)
|
|
print(json.dumps(run.record, indent=2))
|
|
return 0 if run.record['status'] in ['PASS', 'PREFLIGHT_PASS'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
raise SystemExit(main())
|
|
except Exception:
|
|
print('Installed storage acceptance could not start; sensitive details suppressed.', file=sys.stderr)
|
|
raise SystemExit(1) from None
|