94 lines
3.7 KiB
Python
94 lines
3.7 KiB
Python
"""bos_fetch: 从 BOS 前缀下载数据集到产物目录 (平台数据落地算子)。
|
|
|
|
超参/环境:
|
|
BOS_URI 必填, bos://bucket/prefix
|
|
BOS_ENDPOINT 默认 bj.bcebos.com
|
|
BOS_ACCESS_KEY_ID 凭据 (或平台注入的 BOS_ACCESS_KEY_ID / BOS_AK)
|
|
BOS_SECRET_ACCESS_KEY
|
|
OUTPUT_DIR 平台注入
|
|
|
|
产物: $OUTPUT_DIR/dataset/<相对路径>, 清单写 $OUTPUT_DIR/.artifacts.json (portId=dataset)。
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
from baidubce import protocol as bce_protocol
|
|
from baidubce.auth.bce_credentials import BceCredentials
|
|
from baidubce.bce_client_configuration import BceClientConfiguration
|
|
from baidubce.services.bos.bos_client import BosClient
|
|
|
|
sys.path.insert(0, "/app")
|
|
from ws_common import report_utils # noqa: E402
|
|
|
|
|
|
def main() -> None:
|
|
print("[bos_fetch] env keys:", sorted(k for k in os.environ if k.startswith("BOS_")), flush=True)
|
|
uri = os.environ.get("BOS_URI", "").strip()
|
|
if not uri.startswith("bos://"):
|
|
sys.exit(f"[bos_fetch] BOS_URI must be bos://bucket/prefix, got {uri!r}")
|
|
rest = uri[len("bos://") :]
|
|
bucket, _, prefix = rest.partition("/")
|
|
prefix = prefix.strip("/")
|
|
# bce-sdk 的 endpoint 只接受 region 形式 (如 bj); 域名形式会报 Invalid endpoint
|
|
endpoint = os.environ.get("BOS_ENDPOINT", "bj.bcebos.com").strip() or "bj.bcebos.com"
|
|
|
|
ak = os.environ.get("BOS_ACCESS_KEY_ID") or os.environ.get("BOS_AK", "")
|
|
sk = os.environ.get("BOS_SECRET_ACCESS_KEY") or os.environ.get("BOS_SK", "")
|
|
if not ak or not sk:
|
|
sys.exit("[bos_fetch] missing BOS credentials (BOS_ACCESS_KEY_ID/BOS_SECRET_ACCESS_KEY)")
|
|
|
|
output_dir = os.environ.get("OUTPUT_DIR", "").strip()
|
|
if not output_dir:
|
|
sys.exit("[bos_fetch] OUTPUT_DIR not set")
|
|
out = os.path.join(output_dir, "dataset")
|
|
|
|
# bce-sdk: protocol 须为协议对象 (str 会报 'str' object has no attribute 'name')
|
|
client = BosClient(BceClientConfiguration(BceCredentials(ak, sk), endpoint=endpoint, protocol=bce_protocol.HTTPS))
|
|
|
|
# 目录前缀精确化: BOS prefix 是裸字符串前缀, "robotwin_hammer_head" 会误匹配
|
|
# 兄弟目录 "robotwin_hammer_head_front/_wrists"; 追加 "/" 后只匹配该目录内对象。
|
|
# 目录列举为空时回退裸前缀 (单文件 URI 语义)。
|
|
effective = f"{prefix}/" if prefix and not prefix.endswith("/") else prefix
|
|
|
|
def list_all(pfx):
|
|
marker = ""
|
|
keys = []
|
|
while True:
|
|
resp = client.list_objects(bucket, prefix=pfx if pfx else None, marker=marker, max_keys=1000)
|
|
for obj in resp.contents:
|
|
if not obj.key.endswith("/"):
|
|
keys.append((obj.key, obj.size))
|
|
if getattr(resp, "is_truncated", False):
|
|
marker = resp.next_marker
|
|
else:
|
|
return keys
|
|
|
|
keys = list_all(effective) if effective else list_all(None)
|
|
if not keys and effective != prefix:
|
|
keys = list_all(prefix)
|
|
|
|
if not keys:
|
|
sys.exit(f"[bos_fetch] no objects under bos://{bucket}/{prefix}")
|
|
|
|
total = 0
|
|
for i, (key, size) in enumerate(keys, 1):
|
|
rel = key[len(effective):].lstrip("/") if effective else key
|
|
if not rel:
|
|
rel = key.rsplit("/", 1)[-1]
|
|
dst = os.path.join(out, rel)
|
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
client.get_object_to_file(bucket, key, dst)
|
|
total += size
|
|
if i % 5 == 0 or i == len(keys):
|
|
report_utils.emit_progress("bos_fetch", files=i, bytes=total)
|
|
report_utils.write_artifacts(output_dir, [
|
|
{"name": "dataset", "type": "directory", "portId": "dataset", "path": "dataset"},
|
|
])
|
|
print(f"[bos_fetch] done: {len(keys)} files, {total} bytes -> {out}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|