jev-training-data/split_model.py

29 lines
983 B
Python

#!/usr/bin/env python3
"""split_model.py — 把训练产物切分到 git 待推送目录 (大文件切18MB分片)
用法: python3 split_model.py <out目录> <目标目录>
"""
import os, sys, shutil
out, dest = sys.argv[1], sys.argv[2]
os.makedirs(dest, exist_ok=True)
n = 0
for root, _, files in os.walk(out):
for fn in sorted(files):
p = os.path.join(root, fn)
rel = os.path.relpath(p, out).replace("/", "__")
sz = os.path.getsize(p)
if sz <= 18 * 1024 * 1024:
shutil.copy(p, os.path.join(dest, rel))
n += 1
else:
with open(p, "rb") as f:
i = 0
while True:
b = f.read(18 * 1024 * 1024)
if not b:
break
open(os.path.join(dest, f"{rel}.part{i:03d}"), "wb").write(b)
i += 1
n += 1
print(f"split_model: {n} 个文件就绪 -> {dest}", flush=True)