159 lines
7.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""13_iter3_data.py — 迭代3数据:
A. Module共享答案对复核(eval簇B) → 出标签修正建议
B. 泛化页面负例挖掘(下载页等 × 多样问题) → 出专项负例训练对
输出: data/eval_label_fixes.json + data/generic_page_negatives.jsonl
"""
import json, os, sys, time, re
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
HERE = os.path.dirname(os.path.abspath(__file__))
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
JUDGES = ["glm-5.2", "glm-4.6"]
PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
def norm(s): return PUNCT.sub("", s).lower()
PROMPT = """你是检索评测的复核员。判断"文档是否包含回答该问题所需的关键信息"。
规则:
1. 若文档能回答, 必须从文档原文中逐字摘录证据句(evidence, 15-80字的连续片段), 这是硬性要求
2. 注意产品型号匹配: X3 Module 套件的硬件规格与 X3 开发板共用同一套文档, 所以 X3 文档回答 X3 Module 套件问题是【有效的】; 反之其他型号(如 X5/S600)不共享
3. 通用下载页/工具页若只含链接列表而无该问题的具体参数, 不算回答
4. 若文档不含答案, evidence 留空字符串
只输出 JSON: {{"p": 0.0-1.0, "grade": "full|partial|none", "evidence": "...", "reason": "20字内"}}
问题: {query}
文档所属产品域: {products}
文档内容:
---
{text}
---"""
def chat(model, prompt):
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
"temperature": 0.0, "max_tokens": 300,
"thinking": {"type": "disabled"}}).encode()
req = urllib.request.Request(CFG["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
headers={"Content-Type": "application/json",
"Authorization": "Bearer " + CFG["gateway"]["api_key"]})
for i in range(4):
try:
with urllib.request.urlopen(req, timeout=120) as r:
return json.loads(r.read())["choices"][0]["message"]["content"]
except Exception:
if i == 3:
raise
time.sleep(2 ** i)
def parse(content):
try:
content = content.strip()
if content.startswith("```"):
content = content.split("```")[1].removeprefix("json").strip()
d = json.loads(content[content.index("{"):content.rindex("}") + 1])
return {"p": max(0.0, min(1.0, float(d.get("p", 0)))),
"evidence": str(d.get("evidence", ""))[:200]}
except Exception:
return None
def judge(query, doc, docs_norm):
rs = []
for jm in JUDGES:
try:
rs.append(parse(chat(jm, PROMPT.format(query=query, products=",".join(doc["products"]),
text=doc["text"][:4500]))))
except Exception as e:
print(f" [JUDGE-ERR] {jm}: {type(e).__name__}: {str(e)[:130]}", flush=True)
rs.append(None)
ev_ok = [bool(r and len(norm(r["evidence"])) >= 10 and norm(r["evidence"]) in docs_norm[doc["doc_id"]])
for r in rs]
return rs, ev_ok
def main():
corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in
open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")}
docs_norm = {k: {"products": v["products"], "norm_text": norm(v["text"]), "text": v["text"]}
for k, v in corpus.items()}
eval_recs = json.load(open(os.path.join(HERE, "..", "data", "eval_v2_aligned_v03.json"), encoding="utf-8"))
# ---- A. 簇B: Module问题 × X3文档的"负例"对复核 ----
bucket_b = [x for x in eval_recs
if x["label"] == 0 and x["score"] >= 0.7
and ("X3 Module" in x["query"] or "X3 套件" in x["query"])
and x["doc_id"].startswith(("products/x3/", "families/"))]
# 兜底: 其他≥0.9 且文档域与问题型号可能共享的
fixes, work = [], []
for x in bucket_b:
work.append(("fix", x))
print(f"簇B(Module共享答案)候选: {len(work)} 对, 复核中...")
# ---- B. 簇A: 泛化页面负例挖掘 ----
GENERIC = ["families/rdk-36d06335122f-download",
"common/rdk-de6f5c9383d0-hardware_and_system",
"common/rdk-1448ede55dc8-interface",
"common/rdk-b50e209f37c0-multimedia",
"families/rdk-f1ef2d3c3151-accessory",
"common/rdk-adc377bb3b0d-self_start",
"families/rdk-ccea7f4a8744-driver_pwm"]
real_queries = json.load(open(os.path.join(HERE, "..", "data", "real_queries.json"), encoding="utf-8"))
eval_qs = {x["query"] for x in eval_recs}
diverse = [q for q in real_queries if q not in eval_qs][:40]
neg_work = [("neg", q, did) for q in diverse for did in GENERIC if did in corpus]
print(f"簇A(泛化页负例)候选: {len(neg_work)} 对, 标注中...")
results = {}
def do(task):
if task[0] == "fix":
_, x = task
doc = {"doc_id": x["doc_id"], "products": corpus[x["doc_id"]]["products"],
"text": corpus[x["doc_id"]]["text"]}
rs, ev_ok = judge(x["query"], doc, docs_norm)
return ("fix", x, rs, ev_ok)
else:
_, q, did = task
doc = {"doc_id": did, "products": corpus[did]["products"], "text": corpus[did]["text"]}
rs, ev_ok = judge(q, doc, docs_norm)
return ("neg", q, did, rs, ev_ok)
n_fix, n_neg = 0, 0
neg_out = []
fix_out = []
with ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
for r in as_completed([ex.submit(do, t) for t in work + neg_work]):
if r.result()[0] == "fix":
_, x, rs, ev_ok = r.result()
ps = [q["p"] for q in rs if q]
ev_ok_any = any(ev_ok)
verdict = None
if len(ps) == 2 and all(p >= 0.7 for p in ps) and ev_ok_any:
verdict = "flip_to_positive"
elif len(ps) == 2 and all(p <= 0.3 for p in ps):
verdict = "confirm_negative"
if verdict:
n_fix += 1
fix_out.append({"pair_id": x["pair_id"], "query": x["query"], "doc_id": x["doc_id"],
"qid": x.get("qid"), "old_label": x["label"], "verdict": verdict,
"judges": rs})
else:
_, q, did, rs, ev_ok = r.result()
ps = [z["p"] for z in rs if z]
if len(ps) == 2 and all(p <= 0.3 for p in ps):
n_neg += 1
neg_out.append({"query": q, "doc_id": did, "hard_label": 0,
"soft_label": round(sum(ps) / 2, 3), "judges": rs,
"provenance": {"source": "generic_page_negative"}})
if (n_fix + n_neg) % 50 == 0 and (n_fix + n_neg) > 0:
print(f" 已完成 {n_fix + n_neg} (标签修正 {n_fix}, 泛化负例 {n_neg})", flush=True)
json.dump(fix_out, open(os.path.join(HERE, "..", "data", "eval_label_fixes.json"), "w", encoding="utf-8"),
ensure_ascii=False, indent=1)
with open(os.path.join(HERE, "..", "data", "generic_page_negatives.jsonl"), "w", encoding="utf-8") as f:
for r in neg_out:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
print(f"完成: 标签修正 {n_fix} 条 (eval_label_fixes.json), 泛化页负例 {n_neg} 条 (generic_page_negatives.jsonl)")
if __name__ == "__main__":
sys.exit(main())