forked from longtao.wu/droboticflow
The old shipped v1.0.0 used a 300-second whole-response deadline. Keep active transfers alive, validate ranged retries, filter BOS folder markers, and preserve accurate failure/progress reporting. Verify the Linux package with interrupted 4 GiB transfers, a 307-second stream, and the real 7.47 GiB checkpoint with matching SHA256 receipts.
143 lines
7.2 KiB
Python
143 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only POC/BOS acceptance. Never persist passwords, cookies or signed URLs."""
|
|
import argparse
|
|
import getpass
|
|
import hashlib
|
|
import http.cookiejar
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--api', required=True)
|
|
parser.add_argument('--email', required=True)
|
|
parser.add_argument('--prefix', required=True)
|
|
parser.add_argument('--connection')
|
|
parser.add_argument('--binary', type=Path)
|
|
parser.add_argument('--output', type=Path)
|
|
parser.add_argument('--report', type=Path, required=True)
|
|
parser.add_argument('--inspect-empty-parents', action='store_true')
|
|
args = parser.parse_args()
|
|
if args.binary and not args.output:
|
|
parser.error('--binary requires --output')
|
|
password = os.environ.get('DROBOTICFLOW_PASSWORD') or getpass.getpass('POC password: ')
|
|
api = args.api.rstrip('/')
|
|
opener = urllib.request.build_opener(
|
|
urllib.request.ProxyHandler({}),
|
|
urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()),
|
|
)
|
|
|
|
def request_json(path, payload=None):
|
|
data = None if payload is None else json.dumps(payload).encode()
|
|
req = urllib.request.Request(api + path, data=data, headers={'Content-Type': 'application/json'})
|
|
with opener.open(req, timeout=30) as response:
|
|
return json.load(response)
|
|
|
|
request_json('/api/auth/login', {'email': args.email, 'password': password})
|
|
print('Login: HTTP 200', flush=True)
|
|
connections = request_json('/api/storage/connections')['items']
|
|
print('Connections:', json.dumps([
|
|
{key: c.get(key) for key in ('id', 'name', 'bucket', 'rootPrefix')}
|
|
for c in connections
|
|
], ensure_ascii=False), flush=True)
|
|
connection = next((c for c in connections if c['id'] == args.connection), None) if args.connection else connections[0]
|
|
if not connection:
|
|
raise RuntimeError('Requested storage connection is unavailable')
|
|
query = urllib.parse.urlencode({'connectionId': connection['id'], 'bucket': connection['bucket'], 'prefix': args.prefix})
|
|
manifest = request_json('/api/object-storage/directory/manifest?' + query)
|
|
report = {'api': api, 'login': 200, 'connection': connection['id'], 'bucket': manifest['bucket'],
|
|
'prefix': args.prefix,
|
|
'manifestFiles': manifest['totalFiles'], 'manifestBytes': manifest['totalBytes'], 'files': []}
|
|
if not manifest['files'] and args.inspect_empty_parents:
|
|
report['parentListings'] = []
|
|
parent = args.prefix.rstrip('/')
|
|
for _ in range(3):
|
|
parent = parent.rsplit('/', 1)[0]
|
|
query = urllib.parse.urlencode({'connectionId': connection['id'], 'bucket': connection['bucket'],
|
|
'prefix': parent, 'limit': 20})
|
|
listing = request_json('/api/object-storage/objects?' + query)
|
|
summary = {'prefix': parent, 'items': [
|
|
{key: item.get(key) for key in ('key', 'kind', 'sizeBytes')}
|
|
for item in listing.get('items', [])
|
|
]}
|
|
print('Parent listing:', json.dumps(summary, ensure_ascii=False), flush=True)
|
|
report['parentListings'].append(summary)
|
|
bos_opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
for item in manifest['files']:
|
|
entry = {'path': item['relativePath'], 'size': item['sizeBytes']}
|
|
req = urllib.request.Request(item['downloadUrl'], headers={'Range': 'bytes=0-0'})
|
|
try:
|
|
with bos_opener.open(req, timeout=30) as response:
|
|
response.read(1)
|
|
entry.update(status=response.status, contentRange=response.headers.get('Content-Range'),
|
|
etag=response.headers.get('ETag', '').strip('"'))
|
|
except urllib.error.HTTPError as exc:
|
|
entry['status'] = exc.code
|
|
print(json.dumps(entry, ensure_ascii=False), flush=True)
|
|
report['files'].append(entry)
|
|
if args.binary and manifest['files']:
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
args.report.parent.mkdir(parents=True, exist_ok=True)
|
|
log = args.report.with_suffix('.download.log')
|
|
started = time.monotonic()
|
|
env = {**os.environ, 'DROBOTICFLOW_PASSWORD': password,
|
|
'NO_PROXY': '*', 'no_proxy': '*'}
|
|
command = [str(args.binary.resolve()), '--api', api, '--email', args.email,
|
|
'--connection', connection['id'], '--bucket', connection['bucket'],
|
|
'--prefix', args.prefix, '--output', str(args.output.resolve()), '--threads', '4']
|
|
with log.open('w') as output:
|
|
result = subprocess.run(command, env=env, stdout=output, stderr=subprocess.STDOUT)
|
|
report['downloadExit'] = result.returncode
|
|
report['elapsedSeconds'] = round(time.monotonic() - started, 2)
|
|
for entry in report['files']:
|
|
if entry['path'].endswith('/'):
|
|
continue
|
|
path = args.output / entry['path']
|
|
entry['localSize'] = path.stat().st_size if path.is_file() else None
|
|
if path.is_file():
|
|
digest = hashlib.sha256()
|
|
md5 = hashlib.md5()
|
|
with path.open('rb') as source:
|
|
for block in iter(lambda: source.read(8 * 1024 * 1024), b''):
|
|
digest.update(block)
|
|
md5.update(block)
|
|
entry['sha256'] = digest.hexdigest()
|
|
entry['md5'] = md5.hexdigest()
|
|
entry['etagMatchesMD5'] = md5.hexdigest() == entry.get('etag')
|
|
receipt_path = args.output / 'CHECKPOINT_COMPLETE.json'
|
|
if receipt_path.is_file():
|
|
receipt = json.loads(receipt_path.read_text())
|
|
downloaded = {entry['path']: entry for entry in report['files']}
|
|
checks = []
|
|
for expected in receipt.get('files', []):
|
|
actual = downloaded.get(expected['path'], {})
|
|
checks.append({'path': expected['path'], 'matches':
|
|
actual.get('localSize') == expected.get('bytes') and
|
|
actual.get('sha256') == expected.get('sha256')})
|
|
report['checkpointReceiptChecks'] = checks
|
|
print('Download exit:', result.returncode, 'seconds:', report['elapsedSeconds'], flush=True)
|
|
args.report.parent.mkdir(parents=True, exist_ok=True)
|
|
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
|
print('Sanitized report:', args.report, flush=True)
|
|
if args.binary and (not manifest['files'] or report['downloadExit'] or
|
|
any(not check['matches'] for check in report.get('checkpointReceiptChecks', [])) or any(
|
|
e.get('localSize') != e['size'] for e in report['files'] if not e['path'].endswith('/')
|
|
)):
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except urllib.error.HTTPError as error:
|
|
raise SystemExit(f'HTTP request failed: {error.code} (URL and response omitted)') from None
|
|
except urllib.error.URLError as error:
|
|
raise SystemExit(f'Network request failed: {type(error.reason).__name__} (URL omitted)') from None
|