#!/usr/bin/env python3 """13_iter3_data.py — 迭代3数据: A. Module共享答案对复核(eval簇B) → 出标签修正建议 B. 泛化页面负例挖掘(下载页等 × 多样问题) → 出专项负例训练对 输出: data/eval_label_fixes.json + data/generic_page_negatives.jsonl """ import json, os, sys, time, re import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed HERE = os.path.dirname(os.path.abspath(__file__)) CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8")) JUDGES = ["glm-5.2", "glm-4.6"] PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+") def norm(s): return PUNCT.sub("", s).lower() PROMPT = """你是检索评测的复核员。判断"文档是否包含回答该问题所需的关键信息"。 规则: 1. 若文档能回答, 必须从文档原文中逐字摘录证据句(evidence, 15-80字的连续片段), 这是硬性要求 2. 注意产品型号匹配: X3 Module 套件的硬件规格与 X3 开发板共用同一套文档, 所以 X3 文档回答 X3 Module 套件问题是【有效的】; 反之其他型号(如 X5/S600)不共享 3. 通用下载页/工具页若只含链接列表而无该问题的具体参数, 不算回答 4. 若文档不含答案, evidence 留空字符串 只输出 JSON: {{"p": 0.0-1.0, "grade": "full|partial|none", "evidence": "...", "reason": "20字内"}} 问题: {query} 文档所属产品域: {products} 文档内容: --- {text} ---""" def chat(model, prompt): body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}], "temperature": 0.0, "max_tokens": 300, "thinking": {"type": "disabled"}}).encode() req = urllib.request.Request(CFG["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body, headers={"Content-Type": "application/json", "Authorization": "Bearer " + CFG["gateway"]["api_key"]}) for i in range(4): try: with urllib.request.urlopen(req, timeout=120) as r: return json.loads(r.read())["choices"][0]["message"]["content"] except Exception: if i == 3: raise time.sleep(2 ** i) def parse(content): try: content = content.strip() if content.startswith("```"): content = content.split("```")[1].removeprefix("json").strip() d = json.loads(content[content.index("{"):content.rindex("}") + 1]) return {"p": max(0.0, min(1.0, float(d.get("p", 0)))), "evidence": str(d.get("evidence", ""))[:200]} except Exception: return None def judge(query, doc, docs_norm): rs = [] for jm in JUDGES: try: rs.append(parse(chat(jm, PROMPT.format(query=query, products=",".join(doc["products"]), text=doc["text"][:4500])))) except Exception as e: print(f" [JUDGE-ERR] {jm}: {type(e).__name__}: {str(e)[:130]}", flush=True) rs.append(None) ev_ok = [bool(r and len(norm(r["evidence"])) >= 10 and norm(r["evidence"]) in docs_norm[doc["doc_id"]]) for r in rs] return rs, ev_ok def main(): corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")} docs_norm = {k: {"products": v["products"], "norm_text": norm(v["text"]), "text": v["text"]} for k, v in corpus.items()} eval_recs = json.load(open(os.path.join(HERE, "..", "data", "eval_v2_aligned_v03.json"), encoding="utf-8")) # ---- A. 簇B: Module问题 × X3文档的"负例"对复核 ---- bucket_b = [x for x in eval_recs if x["label"] == 0 and x["score"] >= 0.7 and ("X3 Module" in x["query"] or "X3 套件" in x["query"]) and x["doc_id"].startswith(("products/x3/", "families/"))] # 兜底: 其他≥0.9 且文档域与问题型号可能共享的 fixes, work = [], [] for x in bucket_b: work.append(("fix", x)) print(f"簇B(Module共享答案)候选: {len(work)} 对, 复核中...") # ---- B. 簇A: 泛化页面负例挖掘 ---- GENERIC = ["families/rdk-36d06335122f-download", "common/rdk-de6f5c9383d0-hardware_and_system", "common/rdk-1448ede55dc8-interface", "common/rdk-b50e209f37c0-multimedia", "families/rdk-f1ef2d3c3151-accessory", "common/rdk-adc377bb3b0d-self_start", "families/rdk-ccea7f4a8744-driver_pwm"] real_queries = json.load(open(os.path.join(HERE, "..", "data", "real_queries.json"), encoding="utf-8")) eval_qs = {x["query"] for x in eval_recs} diverse = [q for q in real_queries if q not in eval_qs][:40] neg_work = [("neg", q, did) for q in diverse for did in GENERIC if did in corpus] print(f"簇A(泛化页负例)候选: {len(neg_work)} 对, 标注中...") results = {} def do(task): if task[0] == "fix": _, x = task doc = {"doc_id": x["doc_id"], "products": corpus[x["doc_id"]]["products"], "text": corpus[x["doc_id"]]["text"]} rs, ev_ok = judge(x["query"], doc, docs_norm) return ("fix", x, rs, ev_ok) else: _, q, did = task doc = {"doc_id": did, "products": corpus[did]["products"], "text": corpus[did]["text"]} rs, ev_ok = judge(q, doc, docs_norm) return ("neg", q, did, rs, ev_ok) n_fix, n_neg = 0, 0 neg_out = [] fix_out = [] with ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex: for r in as_completed([ex.submit(do, t) for t in work + neg_work]): if r.result()[0] == "fix": _, x, rs, ev_ok = r.result() ps = [q["p"] for q in rs if q] ev_ok_any = any(ev_ok) verdict = None if len(ps) == 2 and all(p >= 0.7 for p in ps) and ev_ok_any: verdict = "flip_to_positive" elif len(ps) == 2 and all(p <= 0.3 for p in ps): verdict = "confirm_negative" if verdict: n_fix += 1 fix_out.append({"pair_id": x["pair_id"], "query": x["query"], "doc_id": x["doc_id"], "qid": x.get("qid"), "old_label": x["label"], "verdict": verdict, "judges": rs}) else: _, q, did, rs, ev_ok = r.result() ps = [z["p"] for z in rs if z] if len(ps) == 2 and all(p <= 0.3 for p in ps): n_neg += 1 neg_out.append({"query": q, "doc_id": did, "hard_label": 0, "soft_label": round(sum(ps) / 2, 3), "judges": rs, "provenance": {"source": "generic_page_negative"}}) if (n_fix + n_neg) % 50 == 0 and (n_fix + n_neg) > 0: print(f" 已完成 {n_fix + n_neg} (标签修正 {n_fix}, 泛化负例 {n_neg})", flush=True) json.dump(fix_out, open(os.path.join(HERE, "..", "data", "eval_label_fixes.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=1) with open(os.path.join(HERE, "..", "data", "generic_page_negatives.jsonl"), "w", encoding="utf-8") as f: for r in neg_out: f.write(json.dumps(r, ensure_ascii=False) + "\n") print(f"完成: 标签修正 {n_fix} 条 (eval_label_fixes.json), 泛化页负例 {n_neg} 条 (generic_page_negatives.jsonl)") if __name__ == "__main__": sys.exit(main())