d-wiki-eval/judge.py
dingbin.wang e1c3f2493e 替换测评集为官方文档集(91 条:7 型号 × 10 文本 + 3 图片)
- evalset.jsonl: 由 rdk_eval_dataset.json 转换,含 model/type/difficulty/reference_answer
- prepare_dataset.py: 改为官方文档集 -> evalset.jsonl(图片转绝对路径)
- run_agent_eval.py: 图片题用 -i 传图(prompt 走 stdin),轨迹记录 sid/model/type
- judge.py: 字段改 reference_answer,按题型/型号拆分汇总
- prompts/judge_prompt.md: 适配规格类短答案评分;agent_prompt 加图片题说明
- results/smoke_official/: 官方集冒烟(1 文本 + 1 图片),替换旧论坛集冒烟
2026-08-15 01:06:26 +08:00

167 lines
7.4 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""准确性评分(LLM-as-judge)。
读取 results/<run_id>/ 下每题的回答,调用 judge 模型按 1-5 分评分,
输出 judged.jsonl 与汇总(平均分、分布、按分类的均分)。
judge 模型默认走本机 codex-proxy(与 codex 同一网关),可用环境变量覆盖:
JUDGE_BASE_URL / JUDGE_API_KEY / JUDGE_MODEL
"""
import argparse, concurrent.futures, json, os, re, time
from openai import OpenAI
BASE = "/Users/d-robotics/agent/知识检索/rdk_eval/agent_eval"
EVALSET = os.path.join(BASE, "evalset.jsonl")
TPL = open(os.path.join(BASE, "prompts/judge_prompt.md")).read()
MAX_A_CHARS = 2000 # 参考答案截断(太长会稀释 judge 注意力)
MAX_R_CHARS = 4000 # 待评分回答截断
def make_client():
# 默认继承环境里的 OPENAI_BASE_URL / OPENAI_API_KEY(与 codex 同一网关),
# 否则回退到本机 codex-proxy。
base = os.environ.get("JUDGE_BASE_URL") or os.environ.get("OPENAI_BASE_URL") or "http://127.0.0.1:8421/v1"
key = os.environ.get("JUDGE_API_KEY") or os.environ.get("OPENAI_API_KEY") or os.environ.get("LLM_API_KEY") or "sk-empty"
return OpenAI(base_url=base, api_key=key, timeout=180, max_retries=2)
def judge_one(client, model, item, response):
q_text = item["question"]
if item.get("type") == "image":
q_text = "(图片识别题,已给 agent 提供板卡图片)" + q_text
prompt = (TPL
.replace("{question}", q_text[:4000])
.replace("{answer}", item["reference_answer"][:MAX_A_CHARS])
.replace("{response}", response[:MAX_R_CHARS]))
resp = client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": prompt}],
temperature=0,
max_tokens=1500,
).choices[0].message
raw = (resp.content or "").strip()
if not raw: # 部分模型把输出放在 reasoning_content,追加说明后重试一次
raw = client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": prompt},
{"role": "assistant", "content": getattr(resp, "reasoning_content", "") or ""},
{"role": "user", "content": "请按要求的格式直接输出评分结果。"}],
temperature=0, max_tokens=1000,
).choices[0].message.content or ""
raw = raw.strip()
m = re.search(r"分数[::]\s*([1-5])", raw)
return {
"qid": item["qid"],
"category": item.get("category"), "model": item.get("model"), "type": item.get("type"), "difficulty": item.get("difficulty"),
"score": int(m.group(1)) if m else None,
"raw": raw,
"parse_ok": bool(m),
}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-id", required=True)
ap.add_argument("--model", default=os.environ.get("JUDGE_MODEL", "glm-5.2"))
ap.add_argument("--workers", type=int, default=6)
ap.add_argument("--qid", type=int, nargs="*", default=None)
args = ap.parse_args()
run_dir = os.path.join(BASE, "results", args.run_id)
out_path = os.path.join(run_dir, "judged.jsonl")
items = {json.loads(l)["qid"]: json.loads(l) for l in open(EVALSET)}
# 只评本次运行实际跑过的题目(answers/ 或 traces/ 里有记录的)
ran_qids = set()
for d in ("answers", "traces"):
dd = os.path.join(run_dir, d)
if os.path.isdir(dd):
for fn in os.listdir(dd):
stem = os.path.splitext(fn)[0]
if stem.isdigit():
ran_qids.add(int(stem))
if ran_qids:
items = {q: it for q, it in items.items() if q in ran_qids}
# 已评过的跳过(续跑)
done = {}
if os.path.exists(out_path):
for l in open(out_path):
try:
r = json.loads(l)
if r.get("parse_ok"):
done[r["qid"]] = r
except json.JSONDecodeError:
pass
todo = []
for qid, item in sorted(items.items()):
if qid in done or (args.qid and qid not in set(args.qid)):
continue
ans_file = os.path.join(run_dir, "answers", f"{qid:04d}.txt")
resp = open(ans_file).read().strip() if os.path.exists(ans_file) else ""
if not resp:
rec = {"qid": qid, "category": item.get("category"), "model": item.get("model"), "type": item.get("type"), "difficulty": item.get("difficulty"), "score": None,
"raw": "(agent 无回答)", "parse_ok": True}
with open(out_path, "a") as f:
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
done[qid] = rec
continue
todo.append(item)
print(f"judge: {len(todo)} 题待评(已有 {len(done)}) model={args.model}", flush=True)
client = make_client()
t0 = time.time()
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as ex:
futs = {ex.submit(judge_one, client, args.model, it,
open(os.path.join(run_dir, "answers", f"{it['qid']:04d}.txt")).read()): it
for it in todo}
n = 0
for fut in concurrent.futures.as_completed(futs):
it = futs[fut]
try:
rec = fut.result()
except Exception as e:
rec = {"qid": it["qid"], "category": it["category"], "score": None,
"raw": f"judge error: {e}", "parse_ok": False}
with open(out_path, "a") as f:
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
done[rec["qid"]] = rec
n += 1
print(f" [{n}/{len(todo)}] qid={it['qid']:>3} score={rec['score']}", flush=True)
# 汇总
scores = [r["score"] for r in done.values() if r.get("score")]
by_cat = {}
by_model = {}
by_type = {}
for r in done.values():
if r.get("score"):
by_cat.setdefault(r.get("category") or "?", []).append(r["score"])
by_model.setdefault(r.get("model") or "?", []).append(r["score"])
by_type.setdefault(r.get("type") or "text", []).append(r["score"])
print(f"\n== 汇总({args.run_id})==")
print(f"有效评分 {len(scores)}/{len(done)},平均分 {sum(scores)/len(scores):.2f}" if scores else "无有效评分")
dist = {s: scores.count(s) for s in range(1, 6)}
print("分数分布:", {k: v for k, v in sorted(dist.items())})
print("按题型均分:", {c: round(sum(v)/len(v), 2) for c, v in sorted(by_type.items())})
print("按型号均分:", {c: round(sum(v)/len(v), 2) for c, v in sorted(by_model.items())})
print(f"耗时 {time.time()-t0:.0f}s")
# 若已有轨迹统计,合并输出检索速率
tr_path = os.path.join(run_dir, "trace_report.json")
if os.path.exists(tr_path):
rows = json.load(open(tr_path))
avg_call = [r["avg_kb_call_s"] for r in rows if r.get("avg_kb_call_s") is not None]
ratio = [r["kb_time_ratio"] for r in rows if r.get("kb_time_ratio") is not None]
lat = [r["latency_s"] for r in rows if r.get("latency_s")]
print(f"\n== 检索速率 ==")
print(f"知识库调用: {sum(r['kb_calls'] for r in rows)} 次,单次平均 {sum(avg_call)/len(avg_call):.2f}s" if avg_call else "无知识库调用")
if ratio:
print(f"检索耗时占比平均 {sum(ratio)/len(ratio)*100:.0f}%,每题总耗时平均 {sum(lat)/len(lat):.1f}s")
if __name__ == "__main__":
main()