136 lines
6.4 KiB
Python
136 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
"""07_build_splits.py — 合并三层(RDK自建+公开+注入) → train/calibration/eval/injection
|
|
用法: python3 07_build_splits.py [--tag v0.1]
|
|
输出: data/jev_dataset_<tag>/{train,calibration,eval,injection}.jsonl + manifest.json
|
|
|
|
规则:
|
|
- RDK 91题(qid 对)全部进 eval, 不参与训练(防止真题泄漏到训练集)
|
|
- 标签: judge p>=0.8→1, <=0.2→0, 其间丢弃进 review; soft_label=judge p
|
|
- hint 交叉核对: hint 与 judge 方向相反的记入 quality_report
|
|
- 公开集: t2reranking 每 query 正例下采样至 1 条; 全局正负比钳到 ≤1.5:1(优先丢正例)
|
|
- 注入集: 仅进 injection 切分, 期望分数保持低位, 不参与普通训练
|
|
"""
|
|
import json, os, sys, random, hashlib, argparse, shutil
|
|
from collections import Counter, defaultdict
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
D = lambda n: os.path.join(HERE, "..", "data", n)
|
|
random.seed(20261014)
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--tag", default="v0.1")
|
|
ap.add_argument("--pos-neg-ratio", type=float, default=1.5)
|
|
args = ap.parse_args()
|
|
|
|
corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in open(D("corpus.jsonl"), encoding="utf-8")}
|
|
labels = {}
|
|
for l in open(D("labels.jsonl"), encoding="utf-8"):
|
|
r = json.loads(l)
|
|
labels[r["pair_id"]] = r
|
|
pairs = [json.loads(l) for l in open(D("pairs_candidate.jsonl"), encoding="utf-8")]
|
|
|
|
qids_eval = set()
|
|
for l in open(D("eval_anchors.json"), encoding="utf-8"):
|
|
pass
|
|
for q in json.load(open(D("eval_anchors.json"), encoding="utf-8"))["questions"]:
|
|
qids_eval.add(q["qid"])
|
|
|
|
out, quality = [], []
|
|
n_drop = 0
|
|
for p in pairs:
|
|
lab = labels.get(hashlib.sha256((p["query"] + "\x00" + p["doc_id"]).encode()).hexdigest()[:16])
|
|
if not lab:
|
|
n_drop += 1
|
|
continue
|
|
pj = lab["judge1"]["p"] if lab.get("judge1") else 0.0
|
|
if pj >= 0.8:
|
|
hard = 1
|
|
elif pj <= 0.2:
|
|
hard = 0
|
|
else:
|
|
n_drop += 1
|
|
continue
|
|
hint = p.get("label_hint")
|
|
conflicted = hint is not None and (hint >= 0.9) != (hard == 1)
|
|
if conflicted:
|
|
quality.append({"pair_key": p["pair_key"], "query": p["query"], "doc_id": p["doc_id"],
|
|
"source": p["source"], "hint": hint, "judge_p": pj})
|
|
if p.get("qid"): # eval 锚定对方向矛盾 → 可疑锚定/坏题, 整对剔除不进任何切分
|
|
n_drop += 1
|
|
continue
|
|
split = "eval" if p.get("qid") in qids_eval else "train"
|
|
out.append({"pair_id": p["pair_key"], "query": p["query"], "query_type": p["source"],
|
|
"doc_id": p["doc_id"], "doc_text": corpus[p["doc_id"]]["text"],
|
|
"doc_truncated": corpus[p["doc_id"]]["truncated"],
|
|
"hard_label": hard, "grade": lab["judge1"]["grade"],
|
|
"soft_label": pj, "judge_scores": [pj],
|
|
"difficulty": p["source"].replace("synthetic_", "").replace("eval_", ""),
|
|
"mutation": None, "split": split,
|
|
"provenance": {"source": "rdk_" + p["source"], "qid": p.get("qid")}})
|
|
|
|
# 公开集
|
|
pub = [json.loads(l) for l in open(D("public_pairs.jsonl"), encoding="utf-8")]
|
|
seen_qpos = set()
|
|
pub_bal = []
|
|
for r in pub:
|
|
if r["provenance"]["dataset"] == "t2reranking" and r["hard_label"] == 1:
|
|
k = r["query"]
|
|
if k in seen_qpos:
|
|
continue
|
|
seen_qpos.add(k)
|
|
pub_bal.append(r)
|
|
n_pub_pos = sum(1 for r in pub_bal if r["hard_label"] == 1)
|
|
n_pub_neg = len(pub_bal) - n_pub_pos
|
|
if n_pub_pos > n_pub_neg * args.pos_neg_ratio:
|
|
pos_idx = [i for i, r in enumerate(pub_bal) if r["hard_label"] == 1]
|
|
random.shuffle(pos_idx)
|
|
drop = set(pos_idx[:n_pub_pos - int(n_pub_neg * args.pos_neg_ratio)])
|
|
pub_bal = [r for i, r in enumerate(pub_bal) if i not in drop]
|
|
for r in pub_bal:
|
|
to_calib = random.random() < 0.15 # 校准集按同比例抽正负例, 覆盖软标签全分布
|
|
out.append({**r, "split": "calibration" if to_calib else "train"})
|
|
|
|
# 注入集
|
|
inj_docs = {}
|
|
if os.path.exists(D("injection_pairs.jsonl")):
|
|
base_p = {(r["query"], r["doc_id"]): r for r in out}
|
|
for l in open(D("injection_pairs.jsonl"), encoding="utf-8"):
|
|
r = json.loads(l)
|
|
k = hashlib.sha256((r["query"] + "\x00" + r["doc_id"]).encode()).hexdigest()[:16]
|
|
base = labels.get(k)
|
|
out.append({"pair_id": "inj_" + k, "query": r["query"], "query_type": "injection",
|
|
"doc_id": r["doc_id"], "doc_text": r["doc_text_mutated"],
|
|
"doc_truncated": len(r["doc_text_mutated"]) > 6000,
|
|
"hard_label": 0, "grade": "none", "soft_label": None,
|
|
"judge_scores": [], "difficulty": "injection_" + r["mutation"],
|
|
"mutation": r["mutation"], "split": "injection",
|
|
"provenance": {"source": "rdk_injection", "base_pair": k,
|
|
"base_judge_p": (base or {}).get("judge1", {}).get("p")}})
|
|
|
|
# 落盘
|
|
outdir = D(f"jev_dataset_{args.tag}")
|
|
if os.path.exists(outdir):
|
|
shutil.rmtree(outdir)
|
|
os.makedirs(outdir)
|
|
by_split = defaultdict(list)
|
|
for r in out:
|
|
by_split[r["split"]].append(r)
|
|
manifest = {"tag": args.tag, "seed": 20261014, "splits": {}, "drops_unlabeled_or_midband": n_drop}
|
|
for sp in ["train", "calibration", "eval", "injection"]:
|
|
rows = by_split.get(sp, [])
|
|
with open(os.path.join(outdir, sp + ".jsonl"), "w", encoding="utf-8") as f:
|
|
for r in rows:
|
|
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
manifest["splits"][sp] = {"count": len(rows),
|
|
"pos": sum(1 for r in rows if r["hard_label"] == 1),
|
|
"datasets": dict(Counter(r["provenance"]["source"] for r in rows))}
|
|
manifest["quality_review"] = {"count": len(quality), "samples": quality[:20]}
|
|
json.dump(manifest, open(os.path.join(outdir, "manifest.json"), "w", encoding="utf-8"),
|
|
ensure_ascii=False, indent=1)
|
|
print(json.dumps(manifest, ensure_ascii=False, indent=1)[:2000])
|
|
print(f"→ {os.path.normpath(outdir)}")
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|