forked from longtao.wu/droboticflow
69 lines
3.0 KiB
Bash
Executable File
69 lines
3.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# 从公司 BOS 桶下载一个完整 Capture 到本地(同事已放行)。
|
||
#
|
||
# 用法: scripts/local/download-capture.sh [Capture目录名]
|
||
# Capture 目录名 = FYLX_B 下的一级目录,默认 FYLX_B01_AZER764007R_19700101_001838
|
||
# 下载到 ~/Desktop/fanyu/droboticflow/local-data/captures/<目录名>/,逐文件 SHA256 记录在下载日志
|
||
#
|
||
# 说明: 纯下载(只读 BOS)。AK/SK 使用桌面 source.md 里的 POC BOS 凭据。
|
||
|
||
set -euo pipefail
|
||
|
||
CAPTURE="${1:-FYLX_B01_AZER764007R_19700101_001838}"
|
||
PREFIX="FYLX_B/$CAPTURE/"
|
||
DEST_ROOT="$HOME/Desktop/fanyu/droboticflow/local-data/captures/$CAPTURE"
|
||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
|
||
# BOS 凭据:从仓库外读取(绝不硬编码进 git)。默认仓库内不追踪文件:
|
||
# ~/Desktop/fanyu/droboticflow/local-data/bos-credentials.csv 内容为 "Access,Secret" 一行
|
||
# 也可用环境变量 BOS_AK / BOS_SK 覆盖。
|
||
CRED_CSV="${BOS_CRED_CSV:-$HOME/Desktop/fanyu/droboticflow/local-data/bos-credentials.csv}"
|
||
if [ -n "${BOS_AK:-}" ] && [ -n "${BOS_SK:-}" ]; then
|
||
AK="$BOS_AK"; SK="$BOS_SK"
|
||
elif [ -f "$CRED_CSV" ]; then
|
||
AK=$(sed -n '2p' "$CRED_CSV" | cut -d',' -f1 | tr -d '\r"')
|
||
SK=$(sed -n '2p' "$CRED_CSV" | cut -d',' -f2 | tr -d '\r"')
|
||
else
|
||
echo "错误: 找不到 BOS 凭据。请设置 BOS_AK/BOS_SK 环境变量,或确保 $CRED_CSV 存在" >&2
|
||
exit 1
|
||
fi
|
||
[ -n "$AK" ] && [ -n "$SK" ] || { echo "错误: 凭据解析为空"; exit 1; }
|
||
|
||
mkdir -p "$DEST_ROOT"
|
||
echo "下载 $PREFIX → $DEST_ROOT"
|
||
|
||
"$REPO_ROOT/.venv/bin/python" - <<EOF
|
||
import ssl, os, hashlib, time
|
||
ssl._create_default_https_context = ssl._create_unverified_context # 公司网络代理证书不在Python CA链里
|
||
from baidubce.services.bos.bos_client import BosClient
|
||
from baidubce.bce_client_configuration import BceClientConfiguration
|
||
from baidubce.auth.bce_credentials import BceCredentials
|
||
|
||
cfg = BceClientConfiguration(BceCredentials("$AK", "$SK"), endpoint="https://bj.bcebos.com")
|
||
client = BosClient(cfg)
|
||
prefix, dest_root = "$PREFIX", "$DEST_ROOT"
|
||
r = client.list_objects("drobodata-fanyu", prefix=prefix, max_keys=100)
|
||
total_mb = sum(o.size for o in r.contents)/1024/1024
|
||
print(f"对象数: {len(r.contents)}, 总计 {total_mb:.0f} MB")
|
||
for obj in r.contents:
|
||
rel = obj.key[len(prefix):]
|
||
dest = os.path.join(dest_root, rel)
|
||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||
if os.path.exists(dest) and os.path.getsize(dest) == obj.size:
|
||
print(f"SKIP {rel}"); continue
|
||
t0 = time.time()
|
||
resp = client.get_object("drobodata-fanyu", obj.key)
|
||
h = hashlib.sha256()
|
||
with open(dest, "wb") as f:
|
||
while True:
|
||
chunk = resp.data.read(1024*256)
|
||
if not chunk: break
|
||
f.write(chunk); h.update(chunk)
|
||
print(f"OK {rel} {obj.size/1024/1024:.1f}MB {obj.size/1024/(time.time()-t0):.0f}KB/s sha256={h.hexdigest()[:16]}")
|
||
print("ALL DONE")
|
||
EOF
|
||
|
||
echo
|
||
echo "校验: find $DEST_ROOT -type f | wc -l (应与下载对象数一致)"
|
||
echo "下一步: scripts/local/wire-local-capture.sh $CAPTURE"
|