208 lines
9.5 KiB
Python

#!/usr/bin/env python3
"""08_import_public.py — 公开数据集 → jev 统一训练样本
用法:
python3 08_import_public.py ragbench # techqa + emanual(句子支撑标注→文档级分级)
python3 08_import_public.py t2reranking
python3 08_import_public.py t2retrieval
python3 08_import_public.py duretrieval
输出: data/public_pairs.jsonl (追加式, 按 pair_id 去重)
schema: 见 README.md; provenance 记录数据集/切分/原始ID/许可
"""
import json, os, sys, hashlib, random, re
HERE = os.path.dirname(os.path.abspath(__file__))
RAW = os.path.join(HERE, "..", "data", "raw")
OUT = os.path.join(HERE, "..", "data", "public_pairs.jsonl")
random.seed(20261014)
MAX_CHARS = 6000
LICENSES = {
"ragbench": "CC BY 4.0 (Galileo AI RAGBench)",
"t2reranking": "T2Ranking (CC BY-SA 4.0, THUIR)",
"t2retrieval": "T2Ranking (CC BY-SA 4.0, THUIR)",
"duretrieval": "DuReader-Retrieval (Apache-2.0, Baidu)",
"qasper": "CC BY 4.0 (AllenAI)",
}
def pair_id(query, ds, doc_key):
return hashlib.sha256(f"{query}\x00{ds}\x00{doc_key}".encode()).hexdigest()[:16]
def trunc(text):
text = re.sub(r"\n{3,}", "\n\n", str(text)).strip()
return (text[:MAX_CHARS], True) if len(text) > MAX_CHARS else (text, False)
def emit(recs):
done = set()
if os.path.exists(OUT):
for l in open(OUT, encoding="utf-8"):
done.add(json.loads(l)["pair_id"])
n_new = n_skip = 0
with open(OUT, "a", encoding="utf-8") as fout:
for r in recs:
if r["pair_id"] in done:
n_skip += 1
continue
fout.write(json.dumps(r, ensure_ascii=False) + "\n")
n_new += 1
print(f" 写入 {n_new} 新对 (重复跳过 {n_skip})")
def load_parquet(path):
import pyarrow.parquet as pq
return pq.read_table(path).to_pylist()
# ---------- ragbench: 句子支撑标注 → 文档级分级 ----------
GRADE_SOFT = {"full": 1.0, "partial": 0.5, "none": 0.0}
def do_ragbench():
recs = []
for domain, rawdir in [("techqa", "ragbench-techqa"), ("emanual", "ragbench-emanual")]:
path = os.path.join(RAW, rawdir, "train.parquet")
if not os.path.exists(path):
print(f" [跳过] {path} 不存在")
continue
rows = load_parquet(path)
n_pos = n_neg = 0
for row in rows:
q = (row.get("question") or "").strip()
docs = row.get("documents") or []
keys = row.get("all_relevant_sentence_keys") or []
if not q or not docs:
continue
total = len(keys)
per_doc = {}
for k in keys:
m = re.match(r"(\d+)", str(k))
if not m:
continue
idx = int(m.group(1))
per_doc[idx] = per_doc.get(idx, 0) + 1
graded, none_docs = {}, []
for i, dtext in enumerate(docs):
c = per_doc.get(i, 0)
if total and c >= 1 and (c == total or (c >= 3 and c / total >= 0.75)):
graded[i] = "full"
elif c >= 1:
graded[i] = "partial"
else:
none_docs.append(i)
neg_sample = random.sample(none_docs, min(2, len(none_docs)))
for i, grade in graded.items():
text, tr = trunc(docs[i])
recs.append({
"pair_id": pair_id(q, f"ragbench_{domain}", i),
"query": q, "query_type": "full", "doc_id": f"ragbench_{domain}/{row['id']}/doc{i}",
"doc_text": text, "doc_truncated": tr,
"hard_label": 1, "grade": grade, "soft_label": GRADE_SOFT[grade],
"judge_scores": [], "difficulty": "pos", "mutation": None,
"split": "train", "provenance": {"source": "public", "dataset": f"ragbench/{domain}",
"orig_id": row.get("id"), "license": LICENSES["ragbench"]}})
n_pos += 1
for i in neg_sample:
text, tr = trunc(docs[i])
recs.append({
"pair_id": pair_id(q, f"ragbench_{domain}neg", i),
"query": q, "query_type": "full", "doc_id": f"ragbench_{domain}/{row['id']}/doc{i}",
"doc_text": text, "doc_truncated": tr,
"hard_label": 0, "grade": "none", "soft_label": 0.0,
"judge_scores": [], "difficulty": "hard", "mutation": None,
"split": "train", "provenance": {"source": "public", "dataset": f"ragbench/{domain}",
"orig_id": row.get("id"), "license": LICENSES["ragbench"]}})
n_neg += 1
print(f" ragbench/{domain}: 行 {len(rows)}, 正例 {n_pos}, 同域难负例 {n_neg}")
emit(recs)
# ---------- mteb 系列(qrels+queries+corpus) ----------
def do_mteb(name, rawdir, grade_map, max_queries, negs_per_q, neg_pool_random=20000):
qp = os.path.join(RAW, rawdir, "queries.parquet")
cp = os.path.join(RAW, rawdir, "corpus.parquet")
dp = os.path.join(RAW, rawdir, "dev-data.parquet")
for p in (qp, cp, dp):
if not os.path.exists(p):
print(f" [跳过] {p} 不存在(语料可能仍在下载)")
return
queries = {r[list(r.keys())[0]]: r[list(r.keys())[1]] for r in load_parquet(qp)}
corpus = {}
for r in load_parquet(cp):
vals = list(r.values())
corpus[vals[0]] = (vals[1] or "", vals[2] if len(vals) > 2 else "") # (title?, text)
qrels = {}
for r in load_parquet(dp):
qrels.setdefault(r["query-id"], []).append((r["corpus-id"], r["score"]))
qids = [q for q, rels in qrels.items()
if q in queries and any(s in grade_map for _, s in rels)]
random.shuffle(qids)
qids = qids[:max_queries]
neg_pool = random.sample(list(corpus.keys()), min(neg_pool_random, len(corpus)))
recs = []
n_pos = n_neg = 0
for qid in qids:
q = queries[qid]
if not q:
continue
rels = qrels[qid]
hard_negs = [cid for cid, s in rels if grade_map.get(s) == "none" and cid in corpus]
random.shuffle(hard_negs)
for cid, s in rels:
if s not in grade_map or cid not in corpus:
continue
grade = grade_map[s]
title, text = corpus[cid]
full_text = (title + "\n" + text) if title and grade != "none" else text
t, tr = trunc(full_text)
recs.append({"pair_id": pair_id(q, name, cid),
"query": q, "query_type": "full",
"doc_id": f"{name}/{cid}", "doc_text": t, "doc_truncated": tr,
"hard_label": 0 if grade == "none" else 1, "grade": grade,
"soft_label": GRADE_SOFT[grade], "judge_scores": [],
"difficulty": "pos" if grade != "none" else "hard", "mutation": None,
"split": "train",
"provenance": {"source": "public", "dataset": name, "orig_id": f"{qid}/{cid}",
"license": LICENSES[name]}})
if grade != "none":
n_pos += 1
need = negs_per_q
pool = hard_negs if hard_negs else []
for cid in pool[:need]:
t, tr = trunc(corpus[cid][1])
recs.append({"pair_id": pair_id(q, name, cid),
"query": q, "query_type": "full", "doc_id": f"{name}/{cid}",
"doc_text": t, "doc_truncated": tr,
"hard_label": 0, "grade": "none", "soft_label": 0.0,
"judge_scores": [], "difficulty": "hard", "mutation": None,
"split": "train",
"provenance": {"source": "public", "dataset": name,
"orig_id": f"{qid}/{cid}", "license": LICENSES[name]}})
n_neg += 1
need -= len(pool[:need])
for cid in random.sample(neg_pool, min(need, len(neg_pool))):
t, tr = trunc(corpus[cid][1])
recs.append({"pair_id": pair_id(q, name, cid),
"query": q, "query_type": "full", "doc_id": f"{name}/{cid}",
"doc_text": t, "doc_truncated": tr,
"hard_label": 0, "grade": "none", "soft_label": 0.0,
"judge_scores": [], "difficulty": "easy", "mutation": None,
"split": "train",
"provenance": {"source": "public", "dataset": name,
"orig_id": f"{qid}/{cid}", "license": LICENSES[name]}})
n_neg += 1
print(f" {name}: 采样 {len(qids)} query, 正例 {n_pos}, 负例 {n_neg}")
emit(recs)
def main():
which = sys.argv[1] if len(sys.argv) > 1 else "all"
if which in ("ragbench", "all"):
print("[ragbench techqa+emanual]")
do_ragbench()
if which in ("t2reranking", "all"):
print("[t2reranking 二分类(仅正例qrels,负例取top_ranked缺省池)]")
do_mteb("t2reranking", "t2reranking", {1: "full"}, 3000, 2)
if which in ("t2retrieval", "all"):
print("[t2retrieval 分级 0/1/2 → none/partial/full]")
do_mteb("t2retrieval", "t2retrieval", {2: "full", 1: "partial", 0: "none"}, 1500, 2)
if which in ("duretrieval", "all"):
print("[duretrieval 二分类]")
do_mteb("duretrieval", "duretrieval", {1: "full"}, 1200, 2)
if __name__ == "__main__":
sys.exit(main())