78 lines
3.9 KiB
Python
78 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""02b_llm_anchor.py — LLM 锚定队列求解: 从 BM25 候选中挑选真正包含答案的文档
|
|
用法: python3 02b_llm_anchor.py
|
|
输入: data/llm_anchor_queue.json 输出: data/llm_anchor_results.jsonl
|
|
每题一次调用: 问题 + 参考答案 + ≤10 个候选(每候选前600字符)。模型从候选中选含答案文档, 可多选, 可答"none"。
|
|
"""
|
|
import json, os, sys, time
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
|
|
|
|
PROMPT = """给定用户问题、标准参考答案、以及候选文档列表。请判断哪些候选文档包含能推导出参考答案的内容。
|
|
规则: 只依据内容判断, 参考答案的表述可以与文档不同但事实必须来自该文档; 若无候选包含答案, 返回空数组。
|
|
|
|
只输出 JSON: {{"answer_doc_ids": ["...", ...]}} (从候选编号中选择)
|
|
|
|
用户问题: {question}
|
|
参考答案: {ref_answer}
|
|
|
|
候选文档:
|
|
{candidates}
|
|
"""
|
|
|
|
def chat(cfg, model, prompt):
|
|
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
|
|
"temperature": 0.0, "max_tokens": 200, "thinking": {"type": "disabled"}}).encode()
|
|
req = urllib.request.Request(cfg["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
|
|
headers={"Content-Type": "application/json",
|
|
"Authorization": "Bearer " + cfg["gateway"]["api_key"]})
|
|
for i in range(4):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=cfg["gateway"].get("timeout", 120)) as r:
|
|
return json.loads(r.read())["choices"][0]["message"]["content"]
|
|
except Exception:
|
|
if i == 3:
|
|
raise
|
|
time.sleep(2 ** i)
|
|
|
|
def main():
|
|
docs = {json.loads(l)["doc_id"]: json.loads(l) for l in
|
|
open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")}
|
|
queue = json.load(open(os.path.join(HERE, "..", "data", "llm_anchor_queue.json"), encoding="utf-8"))["questions"]
|
|
out = os.path.join(HERE, "..", "data", "llm_anchor_results.jsonl")
|
|
print(f"LLM 锚定 {len(queue)} 题 (模型 {CFG['models']['anchor']})")
|
|
|
|
def work(q):
|
|
cands = "\n".join(
|
|
f"[{i}] doc_id={cid}\n标题: {cid.split('/')[-1]}\n内容: {docs[cid]['text'][:600]}"
|
|
for i, cid in enumerate(q["candidates"]))
|
|
p = PROMPT.format(question=q["question"], ref_answer=(q.get("reference_answer") or "")[:500],
|
|
candidates=cands)
|
|
try:
|
|
content = chat(CFG, CFG["models"]["anchor"], p)
|
|
content = content[content.index("{"):content.rindex("}") + 1]
|
|
ids = json.loads(content).get("answer_doc_ids", [])
|
|
picked = [q["candidates"][int(i)] for i in ids
|
|
if str(i).lstrip("[").rstrip("]").isdigit() and int(i) < len(q["candidates"])]
|
|
return {"qid": q["qid"], "question": q["question"], "reference_answer": q.get("reference_answer"),
|
|
"model": q.get("model"), "matched_docs": picked, "match_type": "llm_anchor"}
|
|
except Exception as e:
|
|
return {"qid": q["qid"], "error": str(e)[:200]}
|
|
|
|
n_ok, n_none, n_err = 0, 0, 0
|
|
with open(out, "w", encoding="utf-8") as fout, \
|
|
ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
|
|
for f in as_completed([ex.submit(work, q) for q in queue]):
|
|
r = f.result()
|
|
fout.write(json.dumps(r, ensure_ascii=False) + "\n"); fout.flush()
|
|
if "error" in r: n_err += 1
|
|
elif r["matched_docs"]: n_ok += 1
|
|
else: n_none += 1
|
|
print(f"锚定成功 {n_ok}, 确认无候选 {n_none}, 失败 {n_err} → {os.path.normpath(out)}")
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|