2026-10-09 16:41:13 +08:00

94 lines
3.7 KiB
Python

"""bos_fetch: 从 BOS 前缀下载数据集到产物目录 (平台数据落地算子)。
超参/环境:
BOS_URI 必填, bos://bucket/prefix
BOS_ENDPOINT 默认 bj.bcebos.com
BOS_ACCESS_KEY_ID 凭据 (或平台注入的 BOS_ACCESS_KEY_ID / BOS_AK)
BOS_SECRET_ACCESS_KEY
OUTPUT_DIR 平台注入
产物: $OUTPUT_DIR/dataset/<相对路径>, 清单写 $OUTPUT_DIR/.artifacts.json (portId=dataset)。
"""
import json
import os
import sys
from baidubce import protocol as bce_protocol
from baidubce.auth.bce_credentials import BceCredentials
from baidubce.bce_client_configuration import BceClientConfiguration
from baidubce.services.bos.bos_client import BosClient
sys.path.insert(0, "/app")
from ws_common import report_utils # noqa: E402
def main() -> None:
print("[bos_fetch] env keys:", sorted(k for k in os.environ if k.startswith("BOS_")), flush=True)
uri = os.environ.get("BOS_URI", "").strip()
if not uri.startswith("bos://"):
sys.exit(f"[bos_fetch] BOS_URI must be bos://bucket/prefix, got {uri!r}")
rest = uri[len("bos://") :]
bucket, _, prefix = rest.partition("/")
prefix = prefix.strip("/")
# bce-sdk 的 endpoint 只接受 region 形式 (如 bj); 域名形式会报 Invalid endpoint
endpoint = os.environ.get("BOS_ENDPOINT", "bj.bcebos.com").strip() or "bj.bcebos.com"
ak = os.environ.get("BOS_ACCESS_KEY_ID") or os.environ.get("BOS_AK", "")
sk = os.environ.get("BOS_SECRET_ACCESS_KEY") or os.environ.get("BOS_SK", "")
if not ak or not sk:
sys.exit("[bos_fetch] missing BOS credentials (BOS_ACCESS_KEY_ID/BOS_SECRET_ACCESS_KEY)")
output_dir = os.environ.get("OUTPUT_DIR", "").strip()
if not output_dir:
sys.exit("[bos_fetch] OUTPUT_DIR not set")
out = os.path.join(output_dir, "dataset")
# bce-sdk: protocol 须为协议对象 (str 会报 'str' object has no attribute 'name')
client = BosClient(BceClientConfiguration(BceCredentials(ak, sk), endpoint=endpoint, protocol=bce_protocol.HTTPS))
# 目录前缀精确化: BOS prefix 是裸字符串前缀, "robotwin_hammer_head" 会误匹配
# 兄弟目录 "robotwin_hammer_head_front/_wrists"; 追加 "/" 后只匹配该目录内对象。
# 目录列举为空时回退裸前缀 (单文件 URI 语义)。
effective = f"{prefix}/" if prefix and not prefix.endswith("/") else prefix
def list_all(pfx):
marker = ""
keys = []
while True:
resp = client.list_objects(bucket, prefix=pfx if pfx else None, marker=marker, max_keys=1000)
for obj in resp.contents:
if not obj.key.endswith("/"):
keys.append((obj.key, obj.size))
if getattr(resp, "is_truncated", False):
marker = resp.next_marker
else:
return keys
keys = list_all(effective) if effective else list_all(None)
if not keys and effective != prefix:
keys = list_all(prefix)
if not keys:
sys.exit(f"[bos_fetch] no objects under bos://{bucket}/{prefix}")
total = 0
for i, (key, size) in enumerate(keys, 1):
rel = key[len(effective):].lstrip("/") if effective else key
if not rel:
rel = key.rsplit("/", 1)[-1]
dst = os.path.join(out, rel)
os.makedirs(os.path.dirname(dst), exist_ok=True)
client.get_object_to_file(bucket, key, dst)
total += size
if i % 5 == 0 or i == len(keys):
report_utils.emit_progress("bos_fetch", files=i, bytes=total)
report_utils.write_artifacts(output_dir, [
{"name": "dataset", "type": "directory", "portId": "dataset", "path": "dataset"},
])
print(f"[bos_fetch] done: {len(keys)} files, {total} bytes -> {out}")
if __name__ == "__main__":
main()