53 lines
2.1 KiB
Python
53 lines
2.1 KiB
Python
#!/usr/bin/env python3
|
|
"""train_laya.py — Laya 官方 finetune() 单卡封装 + 内置 eval_v2 评分
|
|
用法: python3 train_laya.py <模型快照目录> <输出目录>
|
|
"""
|
|
import json, os, sys
|
|
|
|
data_path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/jd/laya_v05_train_2400.jsonl"
|
|
model_dir = sys.argv[2] if len(sys.argv) > 2 else None
|
|
output_dir = sys.argv[3] if len(sys.argv) > 3 else "/tmp/laya_out"
|
|
eval_path = "/tmp/jd/eval_v2.jsonl"
|
|
|
|
from laya.train import TrainConfig, finetune
|
|
|
|
cfg = TrainConfig(
|
|
freeze_encoder=True, # 只训决策头: 从构造上杜绝 encoder 灾难性遗忘
|
|
epochs=2,
|
|
micro_batch=8,
|
|
grad_accum=8,
|
|
amp=True,
|
|
log_every=50,
|
|
)
|
|
summary = finetune(data_path, model_dir, output_dir, cfg)
|
|
print("[laya-ft] finetune summary:", json.dumps(summary, ensure_ascii=False)[:400], flush=True)
|
|
|
|
# ---- eval_v2 评分 ----
|
|
import laya
|
|
agent = laya.Agent(output_dir, device="cuda")
|
|
rows = [json.loads(l) for l in open(eval_path, encoding="utf-8")]
|
|
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
|
|
probs, labels = [], []
|
|
for i, r in enumerate(rows):
|
|
state = f"用户问题: {r['query']}\n\n文档内容:\n{r['doc_text'][:3000]}"
|
|
try:
|
|
res = agent.predict(state, {"relevant": {"type": "noul", "instructions": INSTR}})
|
|
p = float(res["answers"]["relevant"]["noul"])
|
|
except Exception:
|
|
p = 0.5
|
|
probs.append(p)
|
|
labels.append(r["hard_label"])
|
|
if (i + 1) % 25 == 0:
|
|
print(f"[eval] {i+1}/{len(rows)}", flush=True)
|
|
|
|
import bisect
|
|
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
|
|
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
|
|
auc = sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg)) if pos and neg else 0.0
|
|
low = sum(1 for p in pos if p < 0.5)
|
|
fp = sum(1 for p in neg if p >= 0.7)
|
|
print(f"[laya-ft] eval_v2: AUC={auc:.4f} pos<0.5={low}/22 neg>=0.7={fp}/85", flush=True)
|
|
with open("/dev/termination-log", "w") as f:
|
|
f.write(f"LAYA-FT AUC={auc:.4f} pos<0.5={low}/22 neg>=0.7={fp}/85")
|
|
print("LAYA-FT-DONE", flush=True)
|