126 lines
5.9 KiB
Python

#!/usr/bin/env python3
"""05_label_llm.py — 双裁判 LLM 标注 (query, doc) 相关性, 输出软标签
用法: python3 05_label_llm.py --pairs data/pairs_candidate.jsonl [--judge2]
输出: data/labels.jsonl {pair_id, judge1:{p,grade}, judge2:{p,grade}, soft_label, agree}
可断点续跑(按 pair_id 跳过)。双裁判分歧>0.35 的样本进 data/labels_disagreement.jsonl 供人工复核。
"""
import json, os, sys, time, argparse, hashlib
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
HERE = os.path.dirname(os.path.abspath(__file__))
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
PROMPT = """判断以下文档对用户问题的相关程度。你是校准过的评审员: 输出的 p 值应代表"该文档确实有助于回答该问题"的真实概率, 而不是主观倾向。
评分标准:
- p=1.0 full: 文档包含回答该问题所需的全部关键信息
- p=0.5 partial: 文档只覆盖答案的一部分, 还需其他文档
- p=0.05 none: 文档与问题同领域但不含答案 / 完全无关
只输出 JSON: {{"p": 0.0-1.0, "grade": "full|partial|none", "reason": "15字内"}}
用户问题: {query}
文档内容:
---
{text}
---"""
def chat(cfg, model, prompt):
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
"temperature": 0.0, "max_tokens": 120, "thinking": {"type": "disabled"}}).encode()
req = urllib.request.Request(cfg["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
headers={"Content-Type": "application/json",
"Authorization": "Bearer " + cfg["gateway"]["api_key"]})
for i in range(4):
try:
with urllib.request.urlopen(req, timeout=cfg["gateway"].get("timeout", 120)) as r:
return json.loads(r.read())["choices"][0]["message"]["content"]
except Exception:
if i == 3:
raise
time.sleep(2 ** i)
def parse(content):
try:
content = content.strip()
if content.startswith("```"):
content = content.split("```")[1].removeprefix("json").strip()
d = json.loads(content[content.index("{"):content.rindex("}") + 1])
p = max(0.0, min(1.0, float(d.get("p", 0))))
g = d.get("grade") if d.get("grade") in ("full", "partial", "none") else ("full" if p >= 0.8 else "partial" if p >= 0.3 else "none")
return {"p": p, "grade": g}
except Exception:
return None # 模型未按JSON返回, 由调用方记错误并跳过
def pair_id(query, doc_id):
return hashlib.sha256((query + "\x00" + doc_id).encode()).hexdigest()[:16]
def label_one(cfg, judge_model, query, text):
r = parse(chat(cfg, judge_model, PROMPT.format(query=query, text=text[:6000])))
if r is None:
r = parse(chat(cfg, judge_model, PROMPT.format(query=query, text=text[:4000]))) # 重试一次
return r
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--pairs", default=os.path.join(HERE, "..", "data", "pairs_candidate.jsonl"))
ap.add_argument("--judge2", action="store_true", help="同时跑第二裁判")
args = ap.parse_args()
out_path = os.path.join(HERE, "..", "data", "labels.jsonl")
dis_path = os.path.join(HERE, "..", "data", "labels_disagreement.jsonl")
done = set()
if os.path.exists(out_path):
for l in open(out_path, encoding="utf-8"):
done.add(json.loads(l)["pair_id"])
pairs = [json.loads(l) for l in open(args.pairs, encoding="utf-8")]
docs = {json.loads(l)["doc_id"]: json.loads(l) for l in
open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")}
todo = [p for p in pairs if pair_id(p["query"], p["doc_id"]) not in done]
print(f"待标注 {len(todo)} / 共 {len(pairs)} 对")
def work(p):
try:
text = docs[p["doc_id"]]["text"]
j1 = label_one(CFG, CFG["models"]["judge1"], p["query"], text)
if j1 is None:
return {"error": "parse", "pair_key": p["pair_key"], "query": p["query"][:50]}
rec = {"pair_id": pair_id(p["query"], p["doc_id"]), "query": p["query"], "doc_id": p["doc_id"],
"judge1": j1, "judge2": None, "soft_label": j1["p"], "agree": None}
if args.judge2:
j2 = label_one(CFG, CFG["models"]["judge2"], p["query"], text)
if j2 is not None:
rec["judge2"] = j2
rec["soft_label"] = round((j1["p"] + j2["p"]) / 2, 4)
rec["agree"] = abs(j1["p"] - j2["p"]) <= 0.35
return rec
except Exception as e:
return {"error": str(e)[:150], "pair_key": p["pair_key"], "query": p["query"][:50]}
n, n_dis, n_err = 0, 0, 0
with open(out_path, "a", encoding="utf-8") as fout, \
ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
for f in as_completed([ex.submit(work, p) for p in todo]):
rec = f.result()
if "error" in rec:
n_err += 1
with open(os.path.join(HERE, "..", "data", "labels_errors.jsonl"), "a", encoding="utf-8") as fe:
fe.write(json.dumps(rec, ensure_ascii=False) + "\n")
continue
fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
fout.flush()
n += 1
if rec["agree"] is False:
n_dis += 1
with open(dis_path, "a", encoding="utf-8") as fd:
fd.write(json.dumps(rec, ensure_ascii=False) + "\n")
if n % 100 == 0:
print(f" 已标注 {n} (分歧 {n_dis}, 错误 {n_err})", flush=True)
print(f"完成 {n} 对, 分歧待复核 {n_dis}, 解析/请求失败 {n_err} → {os.path.normpath(out_path)}")
if __name__ == "__main__":
sys.exit(main())