65 lines
2.7 KiB
Python
65 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""push_model.py — 训练产物(模型/指标)回传 gitea; 大文件切 18MB 分片
|
|
用法: python3 push_model.py <out目录> (env: GITEA_USER/GITEA_TOKEN)
|
|
"""
|
|
import os, sys, base64, json, urllib.request, urllib.error
|
|
|
|
REPO = "dingbin.wang/jev-training-data"
|
|
PART = 18 * 1024 * 1024
|
|
|
|
def api(method, path, body=None):
|
|
url = "https://gitea.d-robotics.cc/api/v1" + path
|
|
data = json.dumps(body).encode() if body is not None else None
|
|
req = urllib.request.Request(url, data=data, method=method,
|
|
headers={"Content-Type": "application/json"})
|
|
auth = base64.b64encode(f"{os.environ['GITEA_USER']}:{os.environ['GITEA_TOKEN']}".encode()).decode()
|
|
req.add_header("Authorization", "Basic " + auth)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=120) as r:
|
|
return r.status, json.loads(r.read() or b"{}")
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, json.loads(e.read() or b"{}")
|
|
|
|
def put_file(path, content: bytes, msg):
|
|
status, cur = api("GET", f"/repos/{REPO}/contents/{path}")
|
|
body = {"message": msg, "content": base64.b64encode(content).decode(), "branch": "main"}
|
|
if status == 200 and isinstance(cur, dict) and cur.get("sha"):
|
|
if cur.get("size") == len(content):
|
|
return "unchanged"
|
|
body["sha"] = cur["sha"]
|
|
s, r = api("PUT", f"/repos/{REPO}/contents/{path}", body)
|
|
if s not in (200, 201):
|
|
raise RuntimeError(f"PUT {path} -> {s} {json.dumps(r)[:200]}")
|
|
return s
|
|
|
|
def push_large(local, remote):
|
|
size = os.path.getsize(local)
|
|
n = (size + PART - 1) // PART
|
|
manifest = {"file": remote, "size": size, "parts": n, "part_size": PART}
|
|
with open(local, "rb") as f:
|
|
for i in range(n):
|
|
chunk = f.read(PART)
|
|
put_file(f"{remote}.part{i:03d}", chunk, f"model part {i+1}/{n}")
|
|
print(f" {remote}.part{i:03d} {len(chunk)//1024//1024}MB ok", flush=True)
|
|
put_file(f"{remote}.manifest.json", json.dumps(manifest).encode(), f"manifest {remote}")
|
|
|
|
if __name__ == "__main__":
|
|
out = sys.argv[1]
|
|
# 1. 小文件直传
|
|
for root, _, files in os.walk(out):
|
|
for fn in files:
|
|
p = os.path.join(root, fn)
|
|
rel = os.path.relpath(p, out)
|
|
size = os.path.getsize(p)
|
|
if size > PART:
|
|
continue # 大文件第2步
|
|
put_file(f"model/{rel}", open(p, "rb").read(), f"artifact {rel}")
|
|
print(f" model/{rel} {size//1024}KB ok", flush=True)
|
|
# 2. 大文件分片
|
|
for root, _, files in os.walk(out):
|
|
for fn in files:
|
|
p = os.path.join(root, fn)
|
|
if os.path.getsize(p) > PART:
|
|
push_large(p, f"model_parts/{fn}")
|
|
print("push_model done", flush=True)
|