78 lines
3.9 KiB
Python

#!/usr/bin/env python3
"""02b_llm_anchor.py — LLM 锚定队列求解: 从 BM25 候选中挑选真正包含答案的文档
用法: python3 02b_llm_anchor.py
输入: data/llm_anchor_queue.json 输出: data/llm_anchor_results.jsonl
每题一次调用: 问题 + 参考答案 + ≤10 个候选(每候选前600字符)。模型从候选中选含答案文档, 可多选, 可答"none"。
"""
import json, os, sys, time
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
HERE = os.path.dirname(os.path.abspath(__file__))
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
PROMPT = """给定用户问题、标准参考答案、以及候选文档列表。请判断哪些候选文档包含能推导出参考答案的内容。
规则: 只依据内容判断, 参考答案的表述可以与文档不同但事实必须来自该文档; 若无候选包含答案, 返回空数组。
只输出 JSON: {{"answer_doc_ids": ["...", ...]}} (从候选编号中选择)
用户问题: {question}
参考答案: {ref_answer}
候选文档:
{candidates}
"""
def chat(cfg, model, prompt):
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
"temperature": 0.0, "max_tokens": 200, "thinking": {"type": "disabled"}}).encode()
req = urllib.request.Request(cfg["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
headers={"Content-Type": "application/json",
"Authorization": "Bearer " + cfg["gateway"]["api_key"]})
for i in range(4):
try:
with urllib.request.urlopen(req, timeout=cfg["gateway"].get("timeout", 120)) as r:
return json.loads(r.read())["choices"][0]["message"]["content"]
except Exception:
if i == 3:
raise
time.sleep(2 ** i)
def main():
docs = {json.loads(l)["doc_id"]: json.loads(l) for l in
open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")}
queue = json.load(open(os.path.join(HERE, "..", "data", "llm_anchor_queue.json"), encoding="utf-8"))["questions"]
out = os.path.join(HERE, "..", "data", "llm_anchor_results.jsonl")
print(f"LLM 锚定 {len(queue)} 题 (模型 {CFG['models']['anchor']})")
def work(q):
cands = "\n".join(
f"[{i}] doc_id={cid}\n标题: {cid.split('/')[-1]}\n内容: {docs[cid]['text'][:600]}"
for i, cid in enumerate(q["candidates"]))
p = PROMPT.format(question=q["question"], ref_answer=(q.get("reference_answer") or "")[:500],
candidates=cands)
try:
content = chat(CFG, CFG["models"]["anchor"], p)
content = content[content.index("{"):content.rindex("}") + 1]
ids = json.loads(content).get("answer_doc_ids", [])
picked = [q["candidates"][int(i)] for i in ids
if str(i).lstrip("[").rstrip("]").isdigit() and int(i) < len(q["candidates"])]
return {"qid": q["qid"], "question": q["question"], "reference_answer": q.get("reference_answer"),
"model": q.get("model"), "matched_docs": picked, "match_type": "llm_anchor"}
except Exception as e:
return {"qid": q["qid"], "error": str(e)[:200]}
n_ok, n_none, n_err = 0, 0, 0
with open(out, "w", encoding="utf-8") as fout, \
ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
for f in as_completed([ex.submit(work, q) for q in queue]):
r = f.result()
fout.write(json.dumps(r, ensure_ascii=False) + "\n"); fout.flush()
if "error" in r: n_err += 1
elif r["matched_docs"]: n_ok += 1
else: n_none += 1
print(f"锚定成功 {n_ok}, 确认无候选 {n_none}, 失败 {n_err} → {os.path.normpath(out)}")
if __name__ == "__main__":
sys.exit(main())