jev-training-data/push_model.py

65 lines
2.7 KiB
Python

#!/usr/bin/env python3
"""push_model.py — 训练产物(模型/指标)回传 gitea; 大文件切 18MB 分片
用法: python3 push_model.py <out目录> (env: GITEA_USER/GITEA_TOKEN)
"""
import os, sys, base64, json, urllib.request, urllib.error
REPO = "dingbin.wang/jev-training-data"
PART = 18 * 1024 * 1024
def api(method, path, body=None):
url = "https://gitea.d-robotics.cc/api/v1" + path
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(url, data=data, method=method,
headers={"Content-Type": "application/json"})
auth = base64.b64encode(f"{os.environ['GITEA_USER']}:{os.environ['GITEA_TOKEN']}".encode()).decode()
req.add_header("Authorization", "Basic " + auth)
try:
with urllib.request.urlopen(req, timeout=120) as r:
return r.status, json.loads(r.read() or b"{}")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read() or b"{}")
def put_file(path, content: bytes, msg):
status, cur = api("GET", f"/repos/{REPO}/contents/{path}")
body = {"message": msg, "content": base64.b64encode(content).decode(), "branch": "main"}
if status == 200 and isinstance(cur, dict) and cur.get("sha"):
if cur.get("size") == len(content):
return "unchanged"
body["sha"] = cur["sha"]
s, r = api("PUT", f"/repos/{REPO}/contents/{path}", body)
if s not in (200, 201):
raise RuntimeError(f"PUT {path} -> {s} {json.dumps(r)[:200]}")
return s
def push_large(local, remote):
size = os.path.getsize(local)
n = (size + PART - 1) // PART
manifest = {"file": remote, "size": size, "parts": n, "part_size": PART}
with open(local, "rb") as f:
for i in range(n):
chunk = f.read(PART)
put_file(f"{remote}.part{i:03d}", chunk, f"model part {i+1}/{n}")
print(f" {remote}.part{i:03d} {len(chunk)//1024//1024}MB ok", flush=True)
put_file(f"{remote}.manifest.json", json.dumps(manifest).encode(), f"manifest {remote}")
if __name__ == "__main__":
out = sys.argv[1]
# 1. 小文件直传
for root, _, files in os.walk(out):
for fn in files:
p = os.path.join(root, fn)
rel = os.path.relpath(p, out)
size = os.path.getsize(p)
if size > PART:
continue # 大文件第2步
put_file(f"model/{rel}", open(p, "rb").read(), f"artifact {rel}")
print(f" model/{rel} {size//1024}KB ok", flush=True)
# 2. 大文件分片
for root, _, files in os.walk(out):
for fn in files:
p = os.path.join(root, fn)
if os.path.getsize(p) > PART:
push_large(p, f"model_parts/{fn}")
print("push_model done", flush=True)