droboticflow/scripts/drobotic-pull-live-acceptance.py
eust-w 8780cd598c 🐛 Fix large model downloads and package drobotic-pull 1.0.1
The old shipped v1.0.0 used a 300-second whole-response deadline. Keep active transfers alive, validate ranged retries, filter BOS folder markers, and preserve accurate failure/progress reporting. Verify the Linux package with interrupted 4 GiB transfers, a 307-second stream, and the real 7.47 GiB checkpoint with matching SHA256 receipts.
2026-10-22 14:19:48 +08:00

143 lines
7.2 KiB
Python

#!/usr/bin/env python3
"""Read-only POC/BOS acceptance. Never persist passwords, cookies or signed URLs."""
import argparse
import getpass
import hashlib
import http.cookiejar
import json
import os
from pathlib import Path
import subprocess
import time
import urllib.error
import urllib.parse
import urllib.request
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--api', required=True)
parser.add_argument('--email', required=True)
parser.add_argument('--prefix', required=True)
parser.add_argument('--connection')
parser.add_argument('--binary', type=Path)
parser.add_argument('--output', type=Path)
parser.add_argument('--report', type=Path, required=True)
parser.add_argument('--inspect-empty-parents', action='store_true')
args = parser.parse_args()
if args.binary and not args.output:
parser.error('--binary requires --output')
password = os.environ.get('DROBOTICFLOW_PASSWORD') or getpass.getpass('POC password: ')
api = args.api.rstrip('/')
opener = urllib.request.build_opener(
urllib.request.ProxyHandler({}),
urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()),
)
def request_json(path, payload=None):
data = None if payload is None else json.dumps(payload).encode()
req = urllib.request.Request(api + path, data=data, headers={'Content-Type': 'application/json'})
with opener.open(req, timeout=30) as response:
return json.load(response)
request_json('/api/auth/login', {'email': args.email, 'password': password})
print('Login: HTTP 200', flush=True)
connections = request_json('/api/storage/connections')['items']
print('Connections:', json.dumps([
{key: c.get(key) for key in ('id', 'name', 'bucket', 'rootPrefix')}
for c in connections
], ensure_ascii=False), flush=True)
connection = next((c for c in connections if c['id'] == args.connection), None) if args.connection else connections[0]
if not connection:
raise RuntimeError('Requested storage connection is unavailable')
query = urllib.parse.urlencode({'connectionId': connection['id'], 'bucket': connection['bucket'], 'prefix': args.prefix})
manifest = request_json('/api/object-storage/directory/manifest?' + query)
report = {'api': api, 'login': 200, 'connection': connection['id'], 'bucket': manifest['bucket'],
'prefix': args.prefix,
'manifestFiles': manifest['totalFiles'], 'manifestBytes': manifest['totalBytes'], 'files': []}
if not manifest['files'] and args.inspect_empty_parents:
report['parentListings'] = []
parent = args.prefix.rstrip('/')
for _ in range(3):
parent = parent.rsplit('/', 1)[0]
query = urllib.parse.urlencode({'connectionId': connection['id'], 'bucket': connection['bucket'],
'prefix': parent, 'limit': 20})
listing = request_json('/api/object-storage/objects?' + query)
summary = {'prefix': parent, 'items': [
{key: item.get(key) for key in ('key', 'kind', 'sizeBytes')}
for item in listing.get('items', [])
]}
print('Parent listing:', json.dumps(summary, ensure_ascii=False), flush=True)
report['parentListings'].append(summary)
bos_opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
for item in manifest['files']:
entry = {'path': item['relativePath'], 'size': item['sizeBytes']}
req = urllib.request.Request(item['downloadUrl'], headers={'Range': 'bytes=0-0'})
try:
with bos_opener.open(req, timeout=30) as response:
response.read(1)
entry.update(status=response.status, contentRange=response.headers.get('Content-Range'),
etag=response.headers.get('ETag', '').strip('"'))
except urllib.error.HTTPError as exc:
entry['status'] = exc.code
print(json.dumps(entry, ensure_ascii=False), flush=True)
report['files'].append(entry)
if args.binary and manifest['files']:
args.output.mkdir(parents=True, exist_ok=True)
args.report.parent.mkdir(parents=True, exist_ok=True)
log = args.report.with_suffix('.download.log')
started = time.monotonic()
env = {**os.environ, 'DROBOTICFLOW_PASSWORD': password,
'NO_PROXY': '*', 'no_proxy': '*'}
command = [str(args.binary.resolve()), '--api', api, '--email', args.email,
'--connection', connection['id'], '--bucket', connection['bucket'],
'--prefix', args.prefix, '--output', str(args.output.resolve()), '--threads', '4']
with log.open('w') as output:
result = subprocess.run(command, env=env, stdout=output, stderr=subprocess.STDOUT)
report['downloadExit'] = result.returncode
report['elapsedSeconds'] = round(time.monotonic() - started, 2)
for entry in report['files']:
if entry['path'].endswith('/'):
continue
path = args.output / entry['path']
entry['localSize'] = path.stat().st_size if path.is_file() else None
if path.is_file():
digest = hashlib.sha256()
md5 = hashlib.md5()
with path.open('rb') as source:
for block in iter(lambda: source.read(8 * 1024 * 1024), b''):
digest.update(block)
md5.update(block)
entry['sha256'] = digest.hexdigest()
entry['md5'] = md5.hexdigest()
entry['etagMatchesMD5'] = md5.hexdigest() == entry.get('etag')
receipt_path = args.output / 'CHECKPOINT_COMPLETE.json'
if receipt_path.is_file():
receipt = json.loads(receipt_path.read_text())
downloaded = {entry['path']: entry for entry in report['files']}
checks = []
for expected in receipt.get('files', []):
actual = downloaded.get(expected['path'], {})
checks.append({'path': expected['path'], 'matches':
actual.get('localSize') == expected.get('bytes') and
actual.get('sha256') == expected.get('sha256')})
report['checkpointReceiptChecks'] = checks
print('Download exit:', result.returncode, 'seconds:', report['elapsedSeconds'], flush=True)
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
print('Sanitized report:', args.report, flush=True)
if args.binary and (not manifest['files'] or report['downloadExit'] or
any(not check['matches'] for check in report.get('checkpointReceiptChecks', [])) or any(
e.get('localSize') != e['size'] for e in report['files'] if not e['path'].endswith('/')
)):
raise SystemExit(1)
if __name__ == '__main__':
try:
main()
except urllib.error.HTTPError as error:
raise SystemExit(f'HTTP request failed: {error.code} (URL and response omitted)') from None
except urllib.error.URLError as error:
raise SystemExit(f'Network request failed: {type(error.reason).__name__} (URL omitted)') from None