96 lines
4.3 KiB
Python
96 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""04_gen_queries_llm.py — LLM 为每篇文档合成训练 query(正例+部分例+邻域例)
|
|
用法: python3 04_gen_queries_llm.py [--limit N] [--model M]
|
|
输出: data/syn_queries.jsonl {doc_id, queries:[{query,type,expected_relevance}]}
|
|
可断点续跑(按 doc_id 跳过已生成)
|
|
"""
|
|
import json, os, sys, time, argparse
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
|
|
|
|
PROMPT = """你是检索训练数据构建员。下面是一篇技术文档。请基于文档内容生成 {k} 条用户可能提出的中文检索问题, 用于训练相关性打分模型。
|
|
|
|
要求:
|
|
1. 2 条 "full": 文档能完整回答的问题(具体、像真实开发者会问的, 不要抄标题)
|
|
2. 1 条 "partial": 文档只覆盖答案一部分的问题(答案还需要其他文档补充)
|
|
3. 1 条 "adjacent": 与文档同产品领域、但本文档不包含答案的问题
|
|
|
|
只输出 JSON 数组, 每项 {{"query": "...", "type": "full|partial|adjacent", "expected_relevance": 1.0或0.5或0.05}}, 不要其他文字。
|
|
|
|
文档路径: {doc_id}
|
|
文档内容:
|
|
---
|
|
{text}
|
|
---"""
|
|
|
|
def chat(cfg, model, prompt, max_retries=3):
|
|
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
|
|
"temperature": 0.7, "max_tokens": 800, "thinking": {"type": "disabled"}}).encode()
|
|
req = urllib.request.Request(cfg["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
|
|
headers={"Content-Type": "application/json",
|
|
"Authorization": "Bearer " + cfg["gateway"]["api_key"]})
|
|
for i in range(max_retries):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=cfg["gateway"].get("timeout", 120)) as r:
|
|
data = json.loads(r.read())
|
|
return data["choices"][0]["message"]["content"]
|
|
except Exception as e:
|
|
if i == max_retries - 1:
|
|
raise
|
|
time.sleep(2 ** i)
|
|
|
|
def parse_queries(content):
|
|
content = content.strip()
|
|
if content.startswith("```"):
|
|
content = content.split("```")[1].removeprefix("json").strip()
|
|
arr = json.loads(content[content.index("["):content.rindex("]") + 1])
|
|
out = []
|
|
for q in arr:
|
|
if isinstance(q, dict) and q.get("query") and q.get("type") in ("full", "partial", "adjacent"):
|
|
out.append({"query": str(q["query"])[:200], "type": q["type"],
|
|
"expected_relevance": float(q.get("expected_relevance", 1.0))})
|
|
return out[:5]
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--limit", type=int, default=CFG.get("pilot", {}).get("docs", 50))
|
|
ap.add_argument("--model", default=CFG["models"]["query_gen"])
|
|
args = ap.parse_args()
|
|
|
|
out_path = os.path.join(HERE, "..", "data", "syn_queries.jsonl")
|
|
done = set()
|
|
if os.path.exists(out_path):
|
|
for l in open(out_path, encoding="utf-8"):
|
|
done.add(json.loads(l)["doc_id"])
|
|
docs = [json.loads(l) for l in open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")]
|
|
todo = [d for d in docs if d["doc_id"] not in done][:args.limit]
|
|
print(f"待合成 {len(todo)} 篇 (模型 {args.model}, 已完成 {len(done)})")
|
|
|
|
def work(d):
|
|
p = PROMPT.format(k=4, doc_id=d["doc_id"], text=d["text"][:6000])
|
|
try:
|
|
return d["doc_id"], parse_queries(chat(CFG, args.model, p))
|
|
except Exception as e:
|
|
return d["doc_id"], {"error": str(e)[:200]}
|
|
|
|
n_ok, n_err = 0, 0
|
|
with open(out_path, "a", encoding="utf-8") as fout, \
|
|
ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
|
|
futs = [ex.submit(work, d) for d in todo]
|
|
for f in as_completed(futs):
|
|
doc_id, qs = f.result()
|
|
if isinstance(qs, dict):
|
|
n_err += 1
|
|
fout.write(json.dumps({"doc_id": doc_id, "error": qs["error"]}, ensure_ascii=False) + "\n")
|
|
else:
|
|
n_ok += 1
|
|
fout.write(json.dumps({"doc_id": doc_id, "queries": qs}, ensure_ascii=False) + "\n")
|
|
fout.flush()
|
|
print(f"完成: 成功 {n_ok}, 失败 {n_err} → {os.path.normpath(out_path)}")
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|