159 lines
7.6 KiB
Python
159 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
||
"""13_iter3_data.py — 迭代3数据:
|
||
A. Module共享答案对复核(eval簇B) → 出标签修正建议
|
||
B. 泛化页面负例挖掘(下载页等 × 多样问题) → 出专项负例训练对
|
||
输出: data/eval_label_fixes.json + data/generic_page_negatives.jsonl
|
||
"""
|
||
import json, os, sys, time, re
|
||
import urllib.request
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
|
||
JUDGES = ["glm-5.2", "glm-4.6"]
|
||
PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
|
||
def norm(s): return PUNCT.sub("", s).lower()
|
||
|
||
PROMPT = """你是检索评测的复核员。判断"文档是否包含回答该问题所需的关键信息"。
|
||
|
||
规则:
|
||
1. 若文档能回答, 必须从文档原文中逐字摘录证据句(evidence, 15-80字的连续片段), 这是硬性要求
|
||
2. 注意产品型号匹配: X3 Module 套件的硬件规格与 X3 开发板共用同一套文档, 所以 X3 文档回答 X3 Module 套件问题是【有效的】; 反之其他型号(如 X5/S600)不共享
|
||
3. 通用下载页/工具页若只含链接列表而无该问题的具体参数, 不算回答
|
||
4. 若文档不含答案, evidence 留空字符串
|
||
|
||
只输出 JSON: {{"p": 0.0-1.0, "grade": "full|partial|none", "evidence": "...", "reason": "20字内"}}
|
||
|
||
问题: {query}
|
||
文档所属产品域: {products}
|
||
文档内容:
|
||
---
|
||
{text}
|
||
---"""
|
||
|
||
def chat(model, prompt):
|
||
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
|
||
"temperature": 0.0, "max_tokens": 300,
|
||
"thinking": {"type": "disabled"}}).encode()
|
||
req = urllib.request.Request(CFG["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
|
||
headers={"Content-Type": "application/json",
|
||
"Authorization": "Bearer " + CFG["gateway"]["api_key"]})
|
||
for i in range(4):
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=120) as r:
|
||
return json.loads(r.read())["choices"][0]["message"]["content"]
|
||
except Exception:
|
||
if i == 3:
|
||
raise
|
||
time.sleep(2 ** i)
|
||
|
||
def parse(content):
|
||
try:
|
||
content = content.strip()
|
||
if content.startswith("```"):
|
||
content = content.split("```")[1].removeprefix("json").strip()
|
||
d = json.loads(content[content.index("{"):content.rindex("}") + 1])
|
||
return {"p": max(0.0, min(1.0, float(d.get("p", 0)))),
|
||
"evidence": str(d.get("evidence", ""))[:200]}
|
||
except Exception:
|
||
return None
|
||
|
||
def judge(query, doc, docs_norm):
|
||
rs = []
|
||
for jm in JUDGES:
|
||
try:
|
||
rs.append(parse(chat(jm, PROMPT.format(query=query, products=",".join(doc["products"]),
|
||
text=doc["text"][:4500]))))
|
||
except Exception as e:
|
||
print(f" [JUDGE-ERR] {jm}: {type(e).__name__}: {str(e)[:130]}", flush=True)
|
||
rs.append(None)
|
||
ev_ok = [bool(r and len(norm(r["evidence"])) >= 10 and norm(r["evidence"]) in docs_norm[doc["doc_id"]])
|
||
for r in rs]
|
||
return rs, ev_ok
|
||
|
||
def main():
|
||
corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in
|
||
open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")}
|
||
docs_norm = {k: {"products": v["products"], "norm_text": norm(v["text"]), "text": v["text"]}
|
||
for k, v in corpus.items()}
|
||
eval_recs = json.load(open(os.path.join(HERE, "..", "data", "eval_v2_aligned_v03.json"), encoding="utf-8"))
|
||
|
||
# ---- A. 簇B: Module问题 × X3文档的"负例"对复核 ----
|
||
bucket_b = [x for x in eval_recs
|
||
if x["label"] == 0 and x["score"] >= 0.7
|
||
and ("X3 Module" in x["query"] or "X3 套件" in x["query"])
|
||
and x["doc_id"].startswith(("products/x3/", "families/"))]
|
||
# 兜底: 其他≥0.9 且文档域与问题型号可能共享的
|
||
fixes, work = [], []
|
||
for x in bucket_b:
|
||
work.append(("fix", x))
|
||
print(f"簇B(Module共享答案)候选: {len(work)} 对, 复核中...")
|
||
|
||
# ---- B. 簇A: 泛化页面负例挖掘 ----
|
||
GENERIC = ["families/rdk-36d06335122f-download",
|
||
"common/rdk-de6f5c9383d0-hardware_and_system",
|
||
"common/rdk-1448ede55dc8-interface",
|
||
"common/rdk-b50e209f37c0-multimedia",
|
||
"families/rdk-f1ef2d3c3151-accessory",
|
||
"common/rdk-adc377bb3b0d-self_start",
|
||
"families/rdk-ccea7f4a8744-driver_pwm"]
|
||
real_queries = json.load(open(os.path.join(HERE, "..", "data", "real_queries.json"), encoding="utf-8"))
|
||
eval_qs = {x["query"] for x in eval_recs}
|
||
diverse = [q for q in real_queries if q not in eval_qs][:40]
|
||
neg_work = [("neg", q, did) for q in diverse for did in GENERIC if did in corpus]
|
||
print(f"簇A(泛化页负例)候选: {len(neg_work)} 对, 标注中...")
|
||
|
||
results = {}
|
||
def do(task):
|
||
if task[0] == "fix":
|
||
_, x = task
|
||
doc = {"doc_id": x["doc_id"], "products": corpus[x["doc_id"]]["products"],
|
||
"text": corpus[x["doc_id"]]["text"]}
|
||
rs, ev_ok = judge(x["query"], doc, docs_norm)
|
||
return ("fix", x, rs, ev_ok)
|
||
else:
|
||
_, q, did = task
|
||
doc = {"doc_id": did, "products": corpus[did]["products"], "text": corpus[did]["text"]}
|
||
rs, ev_ok = judge(q, doc, docs_norm)
|
||
return ("neg", q, did, rs, ev_ok)
|
||
|
||
n_fix, n_neg = 0, 0
|
||
neg_out = []
|
||
fix_out = []
|
||
with ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
|
||
for r in as_completed([ex.submit(do, t) for t in work + neg_work]):
|
||
if r.result()[0] == "fix":
|
||
_, x, rs, ev_ok = r.result()
|
||
ps = [q["p"] for q in rs if q]
|
||
ev_ok_any = any(ev_ok)
|
||
verdict = None
|
||
if len(ps) == 2 and all(p >= 0.7 for p in ps) and ev_ok_any:
|
||
verdict = "flip_to_positive"
|
||
elif len(ps) == 2 and all(p <= 0.3 for p in ps):
|
||
verdict = "confirm_negative"
|
||
if verdict:
|
||
n_fix += 1
|
||
fix_out.append({"pair_id": x["pair_id"], "query": x["query"], "doc_id": x["doc_id"],
|
||
"qid": x.get("qid"), "old_label": x["label"], "verdict": verdict,
|
||
"judges": rs})
|
||
else:
|
||
_, q, did, rs, ev_ok = r.result()
|
||
ps = [z["p"] for z in rs if z]
|
||
if len(ps) == 2 and all(p <= 0.3 for p in ps):
|
||
n_neg += 1
|
||
neg_out.append({"query": q, "doc_id": did, "hard_label": 0,
|
||
"soft_label": round(sum(ps) / 2, 3), "judges": rs,
|
||
"provenance": {"source": "generic_page_negative"}})
|
||
if (n_fix + n_neg) % 50 == 0 and (n_fix + n_neg) > 0:
|
||
print(f" 已完成 {n_fix + n_neg} (标签修正 {n_fix}, 泛化负例 {n_neg})", flush=True)
|
||
|
||
json.dump(fix_out, open(os.path.join(HERE, "..", "data", "eval_label_fixes.json"), "w", encoding="utf-8"),
|
||
ensure_ascii=False, indent=1)
|
||
with open(os.path.join(HERE, "..", "data", "generic_page_negatives.jsonl"), "w", encoding="utf-8") as f:
|
||
for r in neg_out:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
print(f"完成: 标签修正 {n_fix} 条 (eval_label_fixes.json), 泛化页负例 {n_neg} 条 (generic_page_negatives.jsonl)")
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|