"""bos_fetch: 从 BOS 前缀下载数据集到产物目录 (平台数据落地算子)。 超参/环境: BOS_URI 必填, bos://bucket/prefix BOS_ENDPOINT 默认 bj.bcebos.com BOS_ACCESS_KEY_ID 凭据 (或平台注入的 BOS_ACCESS_KEY_ID / BOS_AK) BOS_SECRET_ACCESS_KEY OUTPUT_DIR 平台注入 产物: $OUTPUT_DIR/dataset/<相对路径>, 清单写 $OUTPUT_DIR/.artifacts.json (portId=dataset)。 """ import json import os import sys from baidubce import protocol as bce_protocol from baidubce.auth.bce_credentials import BceCredentials from baidubce.bce_client_configuration import BceClientConfiguration from baidubce.services.bos.bos_client import BosClient sys.path.insert(0, "/app") from ws_common import report_utils # noqa: E402 def main() -> None: print("[bos_fetch] env keys:", sorted(k for k in os.environ if k.startswith("BOS_")), flush=True) uri = os.environ.get("BOS_URI", "").strip() if not uri.startswith("bos://"): sys.exit(f"[bos_fetch] BOS_URI must be bos://bucket/prefix, got {uri!r}") rest = uri[len("bos://") :] bucket, _, prefix = rest.partition("/") prefix = prefix.strip("/") # bce-sdk 的 endpoint 只接受 region 形式 (如 bj); 域名形式会报 Invalid endpoint endpoint = os.environ.get("BOS_ENDPOINT", "bj.bcebos.com").strip() or "bj.bcebos.com" ak = os.environ.get("BOS_ACCESS_KEY_ID") or os.environ.get("BOS_AK", "") sk = os.environ.get("BOS_SECRET_ACCESS_KEY") or os.environ.get("BOS_SK", "") if not ak or not sk: sys.exit("[bos_fetch] missing BOS credentials (BOS_ACCESS_KEY_ID/BOS_SECRET_ACCESS_KEY)") output_dir = os.environ.get("OUTPUT_DIR", "").strip() if not output_dir: sys.exit("[bos_fetch] OUTPUT_DIR not set") out = os.path.join(output_dir, "dataset") # bce-sdk: protocol 须为协议对象 (str 会报 'str' object has no attribute 'name') client = BosClient(BceClientConfiguration(BceCredentials(ak, sk), endpoint=endpoint, protocol=bce_protocol.HTTPS)) # 目录前缀精确化: BOS prefix 是裸字符串前缀, "robotwin_hammer_head" 会误匹配 # 兄弟目录 "robotwin_hammer_head_front/_wrists"; 追加 "/" 后只匹配该目录内对象。 # 目录列举为空时回退裸前缀 (单文件 URI 语义)。 effective = f"{prefix}/" if prefix and not prefix.endswith("/") else prefix def list_all(pfx): marker = "" keys = [] while True: resp = client.list_objects(bucket, prefix=pfx if pfx else None, marker=marker, max_keys=1000) for obj in resp.contents: if not obj.key.endswith("/"): keys.append((obj.key, obj.size)) if getattr(resp, "is_truncated", False): marker = resp.next_marker else: return keys keys = list_all(effective) if effective else list_all(None) if not keys and effective != prefix: keys = list_all(prefix) if not keys: sys.exit(f"[bos_fetch] no objects under bos://{bucket}/{prefix}") total = 0 for i, (key, size) in enumerate(keys, 1): rel = key[len(effective):].lstrip("/") if effective else key if not rel: rel = key.rsplit("/", 1)[-1] dst = os.path.join(out, rel) os.makedirs(os.path.dirname(dst), exist_ok=True) client.get_object_to_file(bucket, key, dst) total += size if i % 5 == 0 or i == len(keys): report_utils.emit_progress("bos_fetch", files=i, bytes=total) report_utils.write_artifacts(output_dir, [ {"name": "dataset", "type": "directory", "portId": "dataset", "path": "dataset"}, ]) print(f"[bos_fetch] done: {len(keys)} files, {total} bytes -> {out}") if __name__ == "__main__": main()