droboticflow/scripts/local/download-capture.sh

69 lines
3.0 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env bash
# 从公司 BOS 桶下载一个完整 Capture 到本地(同事已放行)。
#
# 用法: scripts/local/download-capture.sh [Capture目录名]
# Capture 目录名 = FYLX_B 下的一级目录,默认 FYLX_B01_AZER764007R_19700101_001838
# 下载到 ~/Desktop/fanyu/droboticflow/local-data/captures/<目录名>/,逐文件 SHA256 记录在下载日志
#
# 说明: 纯下载(只读 BOS)。AK/SK 使用桌面 source.md 里的 POC BOS 凭据。
set -euo pipefail
CAPTURE="${1:-FYLX_B01_AZER764007R_19700101_001838}"
PREFIX="FYLX_B/$CAPTURE/"
DEST_ROOT="$HOME/Desktop/fanyu/droboticflow/local-data/captures/$CAPTURE"
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# BOS 凭据:从仓库外读取(绝不硬编码进 git)。默认仓库内不追踪文件:
# ~/Desktop/fanyu/droboticflow/local-data/bos-credentials.csv 内容为 "Access,Secret" 一行
# 也可用环境变量 BOS_AK / BOS_SK 覆盖。
CRED_CSV="${BOS_CRED_CSV:-$HOME/Desktop/fanyu/droboticflow/local-data/bos-credentials.csv}"
if [ -n "${BOS_AK:-}" ] && [ -n "${BOS_SK:-}" ]; then
AK="$BOS_AK"; SK="$BOS_SK"
elif [ -f "$CRED_CSV" ]; then
AK=$(sed -n '2p' "$CRED_CSV" | cut -d',' -f1 | tr -d '\r"')
SK=$(sed -n '2p' "$CRED_CSV" | cut -d',' -f2 | tr -d '\r"')
else
echo "错误: 找不到 BOS 凭据。请设置 BOS_AK/BOS_SK 环境变量,或确保 $CRED_CSV 存在" >&2
exit 1
fi
[ -n "$AK" ] && [ -n "$SK" ] || { echo "错误: 凭据解析为空"; exit 1; }
mkdir -p "$DEST_ROOT"
echo "下载 $PREFIX → $DEST_ROOT"
"$REPO_ROOT/.venv/bin/python" - <<EOF
import ssl, os, hashlib, time
ssl._create_default_https_context = ssl._create_unverified_context # 公司网络代理证书不在Python CA链里
from baidubce.services.bos.bos_client import BosClient
from baidubce.bce_client_configuration import BceClientConfiguration
from baidubce.auth.bce_credentials import BceCredentials
cfg = BceClientConfiguration(BceCredentials("$AK", "$SK"), endpoint="https://bj.bcebos.com")
client = BosClient(cfg)
prefix, dest_root = "$PREFIX", "$DEST_ROOT"
r = client.list_objects("drobodata-fanyu", prefix=prefix, max_keys=100)
total_mb = sum(o.size for o in r.contents)/1024/1024
print(f"对象数: {len(r.contents)}, 总计 {total_mb:.0f} MB")
for obj in r.contents:
rel = obj.key[len(prefix):]
dest = os.path.join(dest_root, rel)
os.makedirs(os.path.dirname(dest), exist_ok=True)
if os.path.exists(dest) and os.path.getsize(dest) == obj.size:
print(f"SKIP {rel}"); continue
t0 = time.time()
resp = client.get_object("drobodata-fanyu", obj.key)
h = hashlib.sha256()
with open(dest, "wb") as f:
while True:
chunk = resp.data.read(1024*256)
if not chunk: break
f.write(chunk); h.update(chunk)
print(f"OK {rel} {obj.size/1024/1024:.1f}MB {obj.size/1024/(time.time()-t0):.0f}KB/s sha256={h.hexdigest()[:16]}")
print("ALL DONE")
EOF
echo
echo "校验: find $DEST_ROOT -type f | wc -l (应与下载对象数一致)"
echo "下一步: scripts/local/wire-local-capture.sh $CAPTURE"