85 lines
3.9 KiB
Python
85 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
||
"""14_build_v04.py — v0.4: v0.3 train + 泛化页负例(×3) ; eval_v3 = eval_v2 + 标签修正
|
||
输出: data/jev_dataset_v0.4/ + data/raw/parts_v04/ + eval_v3.jsonl
|
||
"""
|
||
import json, os, sys, hashlib, shutil, tarfile
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
D = lambda n: os.path.join(HERE, "..", "data", n)
|
||
|
||
def main():
|
||
upweight = int(sys.argv[1]) if len(sys.argv) > 1 else 3
|
||
corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in open(D("corpus.jsonl"), encoding="utf-8")}
|
||
|
||
# 1. 标签修正 → eval_v3
|
||
fixes = json.load(open(D("eval_label_fixes.json"), encoding="utf-8"))
|
||
fix_by_pair = {f["pair_id"]: f for f in fixes}
|
||
eval_rows = [json.loads(l) for l in open(D("eval_v2.jsonl"), encoding="utf-8")]
|
||
n_flip = 0
|
||
eval_v3 = []
|
||
for r in eval_rows:
|
||
f = fix_by_pair.get(r["pair_id"])
|
||
if f and f["verdict"] == "flip_to_positive":
|
||
r = dict(r)
|
||
r["hard_label"], r["grade"] = 1, "full"
|
||
r["soft_label"] = round(sum(j["p"] for j in f["judges"] if j) / 2, 3)
|
||
r["review"]["iter3"] = "flip_to_positive"
|
||
n_flip += 1
|
||
eval_v3.append(r)
|
||
with open(D("eval_v3.jsonl"), "w", encoding="utf-8") as f:
|
||
for r in eval_v3:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
print(f"eval_v3: {len(eval_v3)} 对 (修正翻转 {n_flip})")
|
||
|
||
# 2. train = v0.3 train + 泛化负例 ×3
|
||
train = [json.loads(l) for l in open(D("jev_dataset_v0.3/train.jsonl"), encoding="utf-8")]
|
||
negs = [json.loads(l) for l in open(D("generic_page_negatives.jsonl"), encoding="utf-8")]
|
||
added = 0
|
||
for r in negs:
|
||
for w in range(upweight):
|
||
train.append({
|
||
"pair_id": hashlib.sha256((r["query"] + "\x00" + r["doc_id"] + f"\x00g{w}").encode()).hexdigest()[:16],
|
||
"query": r["query"], "query_type": "generic_page_negative",
|
||
"doc_id": r["doc_id"], "doc_text": corpus[r["doc_id"]]["text"],
|
||
"doc_truncated": corpus[r["doc_id"]]["truncated"],
|
||
"hard_label": 0, "grade": "none", "soft_label": r["soft_label"],
|
||
"judge_scores": [j["p"] for j in r["judges"] if j],
|
||
"difficulty": "generic_page", "mutation": None, "split": "train",
|
||
"provenance": {**r["provenance"], "upweight": w}})
|
||
added += 1
|
||
outdir = D("jev_dataset_v0.4")
|
||
if os.path.exists(outdir):
|
||
shutil.rmtree(outdir)
|
||
os.makedirs(outdir)
|
||
with open(os.path.join(outdir, "train.jsonl"), "w", encoding="utf-8") as f:
|
||
for r in train:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
for name in ["calibration", "injection"]:
|
||
shutil.copy(D(f"jev_dataset_v0.3/{name}.jsonl"), os.path.join(outdir, name + ".jsonl"))
|
||
shutil.copy(D("eval_v3.jsonl"), os.path.join(outdir, "eval.jsonl"))
|
||
json.dump({"tag": "v0.4", "train": len(train), "generic_neg_added": added, "upweight": upweight,
|
||
"eval_label_fixed": n_flip},
|
||
open(os.path.join(outdir, "manifest_delta.json"), "w"), ensure_ascii=False, indent=1)
|
||
print(f"v0.4: train {len(train)} 对 (泛化负例 {added}, ×{upweight})")
|
||
|
||
# 3. 打包分片
|
||
parts_dir = D("raw/parts_v04")
|
||
shutil.rmtree(parts_dir, ignore_errors=True)
|
||
os.makedirs(parts_dir)
|
||
bundle = D("raw/jev-v0.4-bundle.tar.gz")
|
||
with tarfile.open(bundle, "w:gz") as tf:
|
||
for name in ["train.jsonl", "calibration.jsonl", "eval.jsonl", "injection.jsonl", "manifest_delta.json"]:
|
||
tf.add(os.path.join(outdir, name), arcname=name)
|
||
i = 0
|
||
with open(bundle, "rb") as f:
|
||
while True:
|
||
b = f.read(5 * 1024 * 1024)
|
||
if not b:
|
||
break
|
||
open(os.path.join(parts_dir, f"jev_part_{i:02d}"), "wb").write(b)
|
||
i += 1
|
||
print(f"bundle → {i} 分片, 目录 {os.path.normpath(parts_dir)}")
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|