73 lines
3.3 KiB
Python
73 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
||
"""12_build_v03.py — v0.3: v0.2 train + 真实问法对(×2 上采样) → 重打包 bundle
|
||
输出: data/jev_dataset_v0.3/ (四切分) + data/raw/jev-v0.3-bundle.tar.gz(重打包) + 分片目录 data/raw/parts_v03/
|
||
"""
|
||
import json, os, sys, hashlib, shutil, tarfile
|
||
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
D = lambda n: os.path.join(HERE, "..", "data", n)
|
||
|
||
def main():
|
||
upweight = int(sys.argv[1]) if len(sys.argv) > 1 else 2
|
||
train = [json.loads(l) for l in open(D("jev_dataset_v0.2/train.jsonl"), encoding="utf-8")]
|
||
real = [json.loads(l) for l in open(D("real_query_pairs.jsonl"), encoding="utf-8")]
|
||
corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in open(D("corpus.jsonl"), encoding="utf-8")}
|
||
|
||
added = 0
|
||
for r in real:
|
||
for w in range(upweight):
|
||
train.append({
|
||
"pair_id": hashlib.sha256((r["query"] + "\x00" + r["doc_id"] + f"\x00w{w}").encode()).hexdigest()[:16],
|
||
"query": r["query"], "query_type": "real_query",
|
||
"doc_id": r["doc_id"], "doc_text": corpus[r["doc_id"]]["text"],
|
||
"doc_truncated": corpus[r["doc_id"]]["truncated"],
|
||
"hard_label": r["hard_label"], "grade": "full" if r["hard_label"] == 1 else "none",
|
||
"soft_label": r["soft_label"], "judge_scores": [j["p"] for j in r["judges"] if j],
|
||
"difficulty": "real", "mutation": None, "split": "train",
|
||
"provenance": {**r["provenance"], "upweight": w}})
|
||
added += 1
|
||
|
||
outdir = D("jev_dataset_v0.3")
|
||
if os.path.exists(outdir):
|
||
shutil.rmtree(outdir)
|
||
os.makedirs(outdir)
|
||
# train 重写; 其余三切分原样复制
|
||
for name in ["calibration", "eval", "injection"]:
|
||
src = D(f"jev_dataset_v0.2/{name}.jsonl")
|
||
dst = os.path.join(outdir, name + ".jsonl")
|
||
shutil.copy(src, dst)
|
||
if name == "eval":
|
||
shutil.copy(D("eval_v2.jsonl"), D("jev_dataset_v0.3/eval_v2.jsonl"))
|
||
with open(os.path.join(outdir, "train.jsonl"), "w", encoding="utf-8") as f:
|
||
for r in train:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
json.dump({"tag": "v0.3", "train": len(train), "real_query_added": added,
|
||
"upweight": upweight},
|
||
open(os.path.join(outdir, "manifest_delta.json"), "w"), ensure_ascii=False, indent=1)
|
||
print(f"v0.3: train {len(train)} 对 (新增真实问法 {added}, 上采样×{upweight})")
|
||
|
||
# 重打包
|
||
parts_dir = D("raw/parts_v03")
|
||
shutil.rmtree(parts_dir, ignore_errors=True)
|
||
os.makedirs(parts_dir)
|
||
bundle = D("raw/jev-v0.3-bundle.tar.gz")
|
||
with tarfile.open(bundle, "w:gz") as tf:
|
||
for name in ["train.jsonl", "calibration.jsonl", "eval.jsonl", "injection.jsonl", "manifest_delta.json"]:
|
||
tf.add(os.path.join(outdir, name), arcname=name)
|
||
import subprocess
|
||
size = os.path.getsize(bundle)
|
||
i = 0
|
||
with open(bundle, "rb") as f:
|
||
while True:
|
||
b = f.read(5 * 1024 * 1024)
|
||
if not b:
|
||
break
|
||
open(os.path.join(parts_dir, f"jev_part_{i:02d}"), "wb").write(b)
|
||
i += 1
|
||
md5 = hashlib.md5(open(bundle, "rb").read()).hexdigest()
|
||
print(f"bundle {size//1024//1024}MB → {i} 分片, md5={md5}, 目录 {os.path.normpath(parts_dir)}")
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|