208 lines
9.5 KiB
Python
208 lines
9.5 KiB
Python
#!/usr/bin/env python3
|
|
"""08_import_public.py — 公开数据集 → jev 统一训练样本
|
|
用法:
|
|
python3 08_import_public.py ragbench # techqa + emanual(句子支撑标注→文档级分级)
|
|
python3 08_import_public.py t2reranking
|
|
python3 08_import_public.py t2retrieval
|
|
python3 08_import_public.py duretrieval
|
|
输出: data/public_pairs.jsonl (追加式, 按 pair_id 去重)
|
|
schema: 见 README.md; provenance 记录数据集/切分/原始ID/许可
|
|
"""
|
|
import json, os, sys, hashlib, random, re
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
RAW = os.path.join(HERE, "..", "data", "raw")
|
|
OUT = os.path.join(HERE, "..", "data", "public_pairs.jsonl")
|
|
random.seed(20261014)
|
|
MAX_CHARS = 6000
|
|
|
|
LICENSES = {
|
|
"ragbench": "CC BY 4.0 (Galileo AI RAGBench)",
|
|
"t2reranking": "T2Ranking (CC BY-SA 4.0, THUIR)",
|
|
"t2retrieval": "T2Ranking (CC BY-SA 4.0, THUIR)",
|
|
"duretrieval": "DuReader-Retrieval (Apache-2.0, Baidu)",
|
|
"qasper": "CC BY 4.0 (AllenAI)",
|
|
}
|
|
|
|
def pair_id(query, ds, doc_key):
|
|
return hashlib.sha256(f"{query}\x00{ds}\x00{doc_key}".encode()).hexdigest()[:16]
|
|
|
|
def trunc(text):
|
|
text = re.sub(r"\n{3,}", "\n\n", str(text)).strip()
|
|
return (text[:MAX_CHARS], True) if len(text) > MAX_CHARS else (text, False)
|
|
|
|
def emit(recs):
|
|
done = set()
|
|
if os.path.exists(OUT):
|
|
for l in open(OUT, encoding="utf-8"):
|
|
done.add(json.loads(l)["pair_id"])
|
|
n_new = n_skip = 0
|
|
with open(OUT, "a", encoding="utf-8") as fout:
|
|
for r in recs:
|
|
if r["pair_id"] in done:
|
|
n_skip += 1
|
|
continue
|
|
fout.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
n_new += 1
|
|
print(f" 写入 {n_new} 新对 (重复跳过 {n_skip})")
|
|
|
|
def load_parquet(path):
|
|
import pyarrow.parquet as pq
|
|
return pq.read_table(path).to_pylist()
|
|
|
|
# ---------- ragbench: 句子支撑标注 → 文档级分级 ----------
|
|
GRADE_SOFT = {"full": 1.0, "partial": 0.5, "none": 0.0}
|
|
|
|
def do_ragbench():
|
|
recs = []
|
|
for domain, rawdir in [("techqa", "ragbench-techqa"), ("emanual", "ragbench-emanual")]:
|
|
path = os.path.join(RAW, rawdir, "train.parquet")
|
|
if not os.path.exists(path):
|
|
print(f" [跳过] {path} 不存在")
|
|
continue
|
|
rows = load_parquet(path)
|
|
n_pos = n_neg = 0
|
|
for row in rows:
|
|
q = (row.get("question") or "").strip()
|
|
docs = row.get("documents") or []
|
|
keys = row.get("all_relevant_sentence_keys") or []
|
|
if not q or not docs:
|
|
continue
|
|
total = len(keys)
|
|
per_doc = {}
|
|
for k in keys:
|
|
m = re.match(r"(\d+)", str(k))
|
|
if not m:
|
|
continue
|
|
idx = int(m.group(1))
|
|
per_doc[idx] = per_doc.get(idx, 0) + 1
|
|
graded, none_docs = {}, []
|
|
for i, dtext in enumerate(docs):
|
|
c = per_doc.get(i, 0)
|
|
if total and c >= 1 and (c == total or (c >= 3 and c / total >= 0.75)):
|
|
graded[i] = "full"
|
|
elif c >= 1:
|
|
graded[i] = "partial"
|
|
else:
|
|
none_docs.append(i)
|
|
neg_sample = random.sample(none_docs, min(2, len(none_docs)))
|
|
for i, grade in graded.items():
|
|
text, tr = trunc(docs[i])
|
|
recs.append({
|
|
"pair_id": pair_id(q, f"ragbench_{domain}", i),
|
|
"query": q, "query_type": "full", "doc_id": f"ragbench_{domain}/{row['id']}/doc{i}",
|
|
"doc_text": text, "doc_truncated": tr,
|
|
"hard_label": 1, "grade": grade, "soft_label": GRADE_SOFT[grade],
|
|
"judge_scores": [], "difficulty": "pos", "mutation": None,
|
|
"split": "train", "provenance": {"source": "public", "dataset": f"ragbench/{domain}",
|
|
"orig_id": row.get("id"), "license": LICENSES["ragbench"]}})
|
|
n_pos += 1
|
|
for i in neg_sample:
|
|
text, tr = trunc(docs[i])
|
|
recs.append({
|
|
"pair_id": pair_id(q, f"ragbench_{domain}neg", i),
|
|
"query": q, "query_type": "full", "doc_id": f"ragbench_{domain}/{row['id']}/doc{i}",
|
|
"doc_text": text, "doc_truncated": tr,
|
|
"hard_label": 0, "grade": "none", "soft_label": 0.0,
|
|
"judge_scores": [], "difficulty": "hard", "mutation": None,
|
|
"split": "train", "provenance": {"source": "public", "dataset": f"ragbench/{domain}",
|
|
"orig_id": row.get("id"), "license": LICENSES["ragbench"]}})
|
|
n_neg += 1
|
|
print(f" ragbench/{domain}: 行 {len(rows)}, 正例 {n_pos}, 同域难负例 {n_neg}")
|
|
emit(recs)
|
|
|
|
# ---------- mteb 系列(qrels+queries+corpus) ----------
|
|
def do_mteb(name, rawdir, grade_map, max_queries, negs_per_q, neg_pool_random=20000):
|
|
qp = os.path.join(RAW, rawdir, "queries.parquet")
|
|
cp = os.path.join(RAW, rawdir, "corpus.parquet")
|
|
dp = os.path.join(RAW, rawdir, "dev-data.parquet")
|
|
for p in (qp, cp, dp):
|
|
if not os.path.exists(p):
|
|
print(f" [跳过] {p} 不存在(语料可能仍在下载)")
|
|
return
|
|
queries = {r[list(r.keys())[0]]: r[list(r.keys())[1]] for r in load_parquet(qp)}
|
|
corpus = {}
|
|
for r in load_parquet(cp):
|
|
vals = list(r.values())
|
|
corpus[vals[0]] = (vals[1] or "", vals[2] if len(vals) > 2 else "") # (title?, text)
|
|
qrels = {}
|
|
for r in load_parquet(dp):
|
|
qrels.setdefault(r["query-id"], []).append((r["corpus-id"], r["score"]))
|
|
qids = [q for q, rels in qrels.items()
|
|
if q in queries and any(s in grade_map for _, s in rels)]
|
|
random.shuffle(qids)
|
|
qids = qids[:max_queries]
|
|
neg_pool = random.sample(list(corpus.keys()), min(neg_pool_random, len(corpus)))
|
|
recs = []
|
|
n_pos = n_neg = 0
|
|
for qid in qids:
|
|
q = queries[qid]
|
|
if not q:
|
|
continue
|
|
rels = qrels[qid]
|
|
hard_negs = [cid for cid, s in rels if grade_map.get(s) == "none" and cid in corpus]
|
|
random.shuffle(hard_negs)
|
|
for cid, s in rels:
|
|
if s not in grade_map or cid not in corpus:
|
|
continue
|
|
grade = grade_map[s]
|
|
title, text = corpus[cid]
|
|
full_text = (title + "\n" + text) if title and grade != "none" else text
|
|
t, tr = trunc(full_text)
|
|
recs.append({"pair_id": pair_id(q, name, cid),
|
|
"query": q, "query_type": "full",
|
|
"doc_id": f"{name}/{cid}", "doc_text": t, "doc_truncated": tr,
|
|
"hard_label": 0 if grade == "none" else 1, "grade": grade,
|
|
"soft_label": GRADE_SOFT[grade], "judge_scores": [],
|
|
"difficulty": "pos" if grade != "none" else "hard", "mutation": None,
|
|
"split": "train",
|
|
"provenance": {"source": "public", "dataset": name, "orig_id": f"{qid}/{cid}",
|
|
"license": LICENSES[name]}})
|
|
if grade != "none":
|
|
n_pos += 1
|
|
need = negs_per_q
|
|
pool = hard_negs if hard_negs else []
|
|
for cid in pool[:need]:
|
|
t, tr = trunc(corpus[cid][1])
|
|
recs.append({"pair_id": pair_id(q, name, cid),
|
|
"query": q, "query_type": "full", "doc_id": f"{name}/{cid}",
|
|
"doc_text": t, "doc_truncated": tr,
|
|
"hard_label": 0, "grade": "none", "soft_label": 0.0,
|
|
"judge_scores": [], "difficulty": "hard", "mutation": None,
|
|
"split": "train",
|
|
"provenance": {"source": "public", "dataset": name,
|
|
"orig_id": f"{qid}/{cid}", "license": LICENSES[name]}})
|
|
n_neg += 1
|
|
need -= len(pool[:need])
|
|
for cid in random.sample(neg_pool, min(need, len(neg_pool))):
|
|
t, tr = trunc(corpus[cid][1])
|
|
recs.append({"pair_id": pair_id(q, name, cid),
|
|
"query": q, "query_type": "full", "doc_id": f"{name}/{cid}",
|
|
"doc_text": t, "doc_truncated": tr,
|
|
"hard_label": 0, "grade": "none", "soft_label": 0.0,
|
|
"judge_scores": [], "difficulty": "easy", "mutation": None,
|
|
"split": "train",
|
|
"provenance": {"source": "public", "dataset": name,
|
|
"orig_id": f"{qid}/{cid}", "license": LICENSES[name]}})
|
|
n_neg += 1
|
|
print(f" {name}: 采样 {len(qids)} query, 正例 {n_pos}, 负例 {n_neg}")
|
|
emit(recs)
|
|
|
|
def main():
|
|
which = sys.argv[1] if len(sys.argv) > 1 else "all"
|
|
if which in ("ragbench", "all"):
|
|
print("[ragbench techqa+emanual]")
|
|
do_ragbench()
|
|
if which in ("t2reranking", "all"):
|
|
print("[t2reranking 二分类(仅正例qrels,负例取top_ranked缺省池)]")
|
|
do_mteb("t2reranking", "t2reranking", {1: "full"}, 3000, 2)
|
|
if which in ("t2retrieval", "all"):
|
|
print("[t2retrieval 分级 0/1/2 → none/partial/full]")
|
|
do_mteb("t2retrieval", "t2retrieval", {2: "full", 1: "partial", 0: "none"}, 1500, 2)
|
|
if which in ("duretrieval", "all"):
|
|
print("[duretrieval 二分类]")
|
|
do_mteb("duretrieval", "duretrieval", {1: "full"}, 1200, 2)
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|