train_all r42: 3-epoch + en_pos oversample 1.5x + en-priority crosspair + window label smoothing 0.85
This commit is contained in:
parent
6d461e72f7
commit
2eefc9c4c2
57
train_all.py
57
train_all.py
@ -50,20 +50,30 @@ def bisect_auc(probs, labels):
|
||||
def _lang(s):
|
||||
return "zh" if any("一" <= c <= "鿿" for c in s) else "en"
|
||||
|
||||
def augment_crosspair_negs(rows, win_tokens, max_new=2500):
|
||||
def augment_crosspair_negs(rows, win_tokens, max_new=3200):
|
||||
"""长文档 × 随机同语言其他 query → 长负例。
|
||||
训练负例几乎都是短文档而 eval_v2 负例全是长文档——模型没学过
|
||||
『长而无关 → 每窗都低分』。跨配对直接补齐 长度×标签 的失衡象限。"""
|
||||
『长而无关 → 每窗都低分』。跨配对直接补齐 长度×标签 的失衡象限。
|
||||
canary_en 诊断: 同域 hard negative(同产品不同主题)是英文难点 → en 候选优先填满 50%。"""
|
||||
rng = random.Random(20261106)
|
||||
nch_thr = {"zh": win_tokens, "en": win_tokens * 3} # 字符≈token 粗估
|
||||
cand = [i for i, r in enumerate(rows)
|
||||
if len(r.get("doc_text") or "") > nch_thr[_lang(r.get("doc_text") or "")]]
|
||||
rng.shuffle(cand)
|
||||
cand_en = [i for i in cand if _lang(rows[i].get("doc_text") or "") == "en"]
|
||||
cand_zh = [i for i in cand if _lang(rows[i].get("doc_text") or "") == "zh"]
|
||||
# en 优先: en 填一半, 剩余 zh (en 不足时 zh 补齐)
|
||||
n_en = min(len(cand_en), max_new // 2)
|
||||
n_zh = min(len(cand_zh), max_new - n_en)
|
||||
picked = cand_en[:n_en] + cand_zh[:n_zh]
|
||||
if n_en + n_zh < max_new:
|
||||
rest = cand_en[n_en:] + cand_zh[n_zh:]
|
||||
picked += rest[:max_new - n_en - n_zh]
|
||||
pools = {}
|
||||
for i, r in enumerate(rows):
|
||||
pools.setdefault(_lang(r["query"]), []).append(i)
|
||||
added = []
|
||||
for i in cand[:max_new]:
|
||||
for i in picked:
|
||||
L = _lang(rows[i].get("doc_text") or "")
|
||||
pool = pools[L]
|
||||
j = rng.choice(pool)
|
||||
@ -75,6 +85,19 @@ def augment_crosspair_negs(rows, win_tokens, max_new=2500):
|
||||
"hard_label": 0, "soft_label": 0.0})
|
||||
return added
|
||||
|
||||
def oversample_en_pos(rows, factor=1.5):
|
||||
"""canary_en 弱判别: en 正例语义学习欠充分 → 过采样 ×factor (粗放但便宜)"""
|
||||
en_pos = [r for r in rows
|
||||
if r.get("hard_label") == 1 and _lang(r["query"]) == "en"]
|
||||
extra_n = int(len(en_pos) * (factor - 1))
|
||||
rng = random.Random(42)
|
||||
extra = []
|
||||
for r in rng.sample(en_pos, min(extra_n, len(en_pos))):
|
||||
d = dict(r)
|
||||
d["pair_id"] = f"os-{d.get('pair_id', '?')}"
|
||||
extra.append(d)
|
||||
return extra
|
||||
|
||||
def phase_data():
|
||||
"""v05n tar.gz 分片重组 → DATA_DIR/train.jsonl"""
|
||||
import glob, shutil, tarfile
|
||||
@ -237,20 +260,22 @@ def run(args):
|
||||
self.q = {"t": "noul", "ins": INSTR, "crit": None}
|
||||
self.rng = random.Random(20261105)
|
||||
def _doc_window(self, text):
|
||||
"""返回 (window_text, aug_flag)。aug_p 概率对长文档随机取窗(答案可能不在窗内)"""
|
||||
if not self.train_aug:
|
||||
return text
|
||||
return text, False
|
||||
ids = tok(text[:12000], add_special_tokens=False)["input_ids"]
|
||||
if len(ids) <= self.win_tokens:
|
||||
return text
|
||||
return text, False
|
||||
if self.rng.random() < self.aug_p:
|
||||
s = self.rng.randrange(0, len(ids) - self.win_tokens + 1)
|
||||
return tok.decode(ids[s:s + self.win_tokens])
|
||||
return text
|
||||
return tok.decode(ids[s:s + self.win_tokens]), True
|
||||
return text, False
|
||||
def __len__(self):
|
||||
return len(self.rows)
|
||||
def __getitem__(self, i):
|
||||
r = self.rows[i]
|
||||
state = f"用户问题: {r['query']}\n\n文档内容:\n{self._doc_window(r['doc_text'])}"
|
||||
doc_win, aug = self._doc_window(r["doc_text"])
|
||||
state = f"用户问题: {r['query']}\n\n文档内容:\n{doc_win}"
|
||||
ids, markers = build_sequence(tok, state, self.q, self.max_len, self.head_len)
|
||||
pad = self.max_len - len(ids)
|
||||
mm = [True] * len(markers) + [False] * (8 - len(markers))
|
||||
@ -258,12 +283,18 @@ def run(args):
|
||||
if soft is None:
|
||||
soft = r["hard_label"]
|
||||
soft_clean = float(soft) if soft in (0, 1) else -1.0
|
||||
# 窗口标签平滑: 随机窗口的正例, 答案可能不在窗内 → BCE 目标 0.85
|
||||
label = float(r["hard_label"])
|
||||
if aug and label == 1.0:
|
||||
label = 0.85
|
||||
if soft_clean < 0:
|
||||
soft_clean = 0.85
|
||||
return {"input_ids": torch.tensor(ids + [0] * pad),
|
||||
"attention_mask": torch.tensor([1] * len(ids) + [0] * pad),
|
||||
"marker_pos": torch.tensor(markers + [0] * (8 - len(markers))),
|
||||
"marker_mask": torch.tensor(mm),
|
||||
"qtype": torch.tensor(2),
|
||||
"label": torch.tensor(float(r["hard_label"])),
|
||||
"label": torch.tensor(label),
|
||||
"soft": torch.tensor(float(soft)),
|
||||
"soft_clean": torch.tensor(soft_clean)}
|
||||
|
||||
@ -303,9 +334,12 @@ def run(args):
|
||||
train, calib = train[:64], calib[:16]
|
||||
log(" TINY mode: train[:64] calib[:16]")
|
||||
else:
|
||||
xp = augment_crosspair_negs(train, args.win_tokens, max_new=2500)
|
||||
xp = augment_crosspair_negs(train, args.win_tokens, max_new=3200)
|
||||
train = train + xp
|
||||
log(f"crosspair long negs: +{len(xp)} (total {len(train)})")
|
||||
os_rows = oversample_en_pos(train, factor=args.en_boost)
|
||||
train = train + os_rows
|
||||
log(f"crosspair long negs: +{len(xp)}, en_pos oversample: +{len(os_rows)} "
|
||||
f"(total {len(train)})")
|
||||
log(f"train={len(train)} calib={len(calib)}")
|
||||
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
@ -569,4 +603,5 @@ if __name__ == "__main__":
|
||||
ap.add_argument("--head_len", type=int, default=192, help="question 头部预算")
|
||||
ap.add_argument("--win_tokens", type=int, default=800, help="随机窗口 token 数")
|
||||
ap.add_argument("--aug_p", type=float, default=0.4, help="随机窗口采样概率")
|
||||
ap.add_argument("--en_boost", type=float, default=1.5, help="en 正例过采样倍数 (1=关)")
|
||||
run(ap.parse_args())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user