73 lines
3.3 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""12_build_v03.py — v0.3: v0.2 train + 真实问法对(×2 上采样) → 重打包 bundle
输出: data/jev_dataset_v0.3/ (四切分) + data/raw/jev-v0.3-bundle.tar.gz(重打包) + 分片目录 data/raw/parts_v03/
"""
import json, os, sys, hashlib, shutil, tarfile
HERE = os.path.dirname(os.path.abspath(__file__))
D = lambda n: os.path.join(HERE, "..", "data", n)
def main():
upweight = int(sys.argv[1]) if len(sys.argv) > 1 else 2
train = [json.loads(l) for l in open(D("jev_dataset_v0.2/train.jsonl"), encoding="utf-8")]
real = [json.loads(l) for l in open(D("real_query_pairs.jsonl"), encoding="utf-8")]
corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in open(D("corpus.jsonl"), encoding="utf-8")}
added = 0
for r in real:
for w in range(upweight):
train.append({
"pair_id": hashlib.sha256((r["query"] + "\x00" + r["doc_id"] + f"\x00w{w}").encode()).hexdigest()[:16],
"query": r["query"], "query_type": "real_query",
"doc_id": r["doc_id"], "doc_text": corpus[r["doc_id"]]["text"],
"doc_truncated": corpus[r["doc_id"]]["truncated"],
"hard_label": r["hard_label"], "grade": "full" if r["hard_label"] == 1 else "none",
"soft_label": r["soft_label"], "judge_scores": [j["p"] for j in r["judges"] if j],
"difficulty": "real", "mutation": None, "split": "train",
"provenance": {**r["provenance"], "upweight": w}})
added += 1
outdir = D("jev_dataset_v0.3")
if os.path.exists(outdir):
shutil.rmtree(outdir)
os.makedirs(outdir)
# train 重写; 其余三切分原样复制
for name in ["calibration", "eval", "injection"]:
src = D(f"jev_dataset_v0.2/{name}.jsonl")
dst = os.path.join(outdir, name + ".jsonl")
shutil.copy(src, dst)
if name == "eval":
shutil.copy(D("eval_v2.jsonl"), D("jev_dataset_v0.3/eval_v2.jsonl"))
with open(os.path.join(outdir, "train.jsonl"), "w", encoding="utf-8") as f:
for r in train:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
json.dump({"tag": "v0.3", "train": len(train), "real_query_added": added,
"upweight": upweight},
open(os.path.join(outdir, "manifest_delta.json"), "w"), ensure_ascii=False, indent=1)
print(f"v0.3: train {len(train)} 对 (新增真实问法 {added}, 上采样×{upweight})")
# 重打包
parts_dir = D("raw/parts_v03")
shutil.rmtree(parts_dir, ignore_errors=True)
os.makedirs(parts_dir)
bundle = D("raw/jev-v0.3-bundle.tar.gz")
with tarfile.open(bundle, "w:gz") as tf:
for name in ["train.jsonl", "calibration.jsonl", "eval.jsonl", "injection.jsonl", "manifest_delta.json"]:
tf.add(os.path.join(outdir, name), arcname=name)
import subprocess
size = os.path.getsize(bundle)
i = 0
with open(bundle, "rb") as f:
while True:
b = f.read(5 * 1024 * 1024)
if not b:
break
open(os.path.join(parts_dir, f"jev_part_{i:02d}"), "wb").write(b)
i += 1
md5 = hashlib.md5(open(bundle, "rb").read()).hexdigest()
print(f"bundle {size//1024//1024}MB → {i} 分片, md5={md5}, 目录 {os.path.normpath(parts_dir)}")
if __name__ == "__main__":
sys.exit(main())