#!/usr/bin/env python3 """push_model.py — 训练产物(模型/指标)回传 gitea; 大文件切 18MB 分片 用法: python3 push_model.py (env: GITEA_USER/GITEA_TOKEN) """ import os, sys, base64, json, urllib.request, urllib.error REPO = "dingbin.wang/jev-training-data" PART = 18 * 1024 * 1024 def api(method, path, body=None): url = "https://gitea.d-robotics.cc/api/v1" + path data = json.dumps(body).encode() if body is not None else None req = urllib.request.Request(url, data=data, method=method, headers={"Content-Type": "application/json"}) auth = base64.b64encode(f"{os.environ['GITEA_USER']}:{os.environ['GITEA_TOKEN']}".encode()).decode() req.add_header("Authorization", "Basic " + auth) try: with urllib.request.urlopen(req, timeout=120) as r: return r.status, json.loads(r.read() or b"{}") except urllib.error.HTTPError as e: return e.code, json.loads(e.read() or b"{}") def put_file(path, content: bytes, msg): status, cur = api("GET", f"/repos/{REPO}/contents/{path}") body = {"message": msg, "content": base64.b64encode(content).decode(), "branch": "main"} if status == 200 and isinstance(cur, dict) and cur.get("sha"): if cur.get("size") == len(content): return "unchanged" body["sha"] = cur["sha"] s, r = api("PUT", f"/repos/{REPO}/contents/{path}", body) if s not in (200, 201): raise RuntimeError(f"PUT {path} -> {s} {json.dumps(r)[:200]}") return s def push_large(local, remote): size = os.path.getsize(local) n = (size + PART - 1) // PART manifest = {"file": remote, "size": size, "parts": n, "part_size": PART} with open(local, "rb") as f: for i in range(n): chunk = f.read(PART) put_file(f"{remote}.part{i:03d}", chunk, f"model part {i+1}/{n}") print(f" {remote}.part{i:03d} {len(chunk)//1024//1024}MB ok", flush=True) put_file(f"{remote}.manifest.json", json.dumps(manifest).encode(), f"manifest {remote}") if __name__ == "__main__": out = sys.argv[1] # 1. 小文件直传 for root, _, files in os.walk(out): for fn in files: p = os.path.join(root, fn) rel = os.path.relpath(p, out) size = os.path.getsize(p) if size > PART: continue # 大文件第2步 put_file(f"model/{rel}", open(p, "rb").read(), f"artifact {rel}") print(f" model/{rel} {size//1024}KB ok", flush=True) # 2. 大文件分片 for root, _, files in os.walk(out): for fn in files: p = os.path.join(root, fn) if os.path.getsize(p) > PART: push_large(p, f"model_parts/{fn}") print("push_model done", flush=True)