- evalset.jsonl: 由 rdk_eval_dataset.json 转换,含 model/type/difficulty/reference_answer - prepare_dataset.py: 改为官方文档集 -> evalset.jsonl(图片转绝对路径) - run_agent_eval.py: 图片题用 -i 传图(prompt 走 stdin),轨迹记录 sid/model/type - judge.py: 字段改 reference_answer,按题型/型号拆分汇总 - prompts/judge_prompt.md: 适配规格类短答案评分;agent_prompt 加图片题说明 - results/smoke_official/: 官方集冒烟(1 文本 + 1 图片),替换旧论坛集冒烟
167 lines
7.4 KiB
Python
167 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""准确性评分(LLM-as-judge)。
|
||
|
||
读取 results/<run_id>/ 下每题的回答,调用 judge 模型按 1-5 分评分,
|
||
输出 judged.jsonl 与汇总(平均分、分布、按分类的均分)。
|
||
|
||
judge 模型默认走本机 codex-proxy(与 codex 同一网关),可用环境变量覆盖:
|
||
JUDGE_BASE_URL / JUDGE_API_KEY / JUDGE_MODEL
|
||
"""
|
||
import argparse, concurrent.futures, json, os, re, time
|
||
|
||
from openai import OpenAI
|
||
|
||
BASE = "/Users/d-robotics/agent/知识检索/rdk_eval/agent_eval"
|
||
EVALSET = os.path.join(BASE, "evalset.jsonl")
|
||
TPL = open(os.path.join(BASE, "prompts/judge_prompt.md")).read()
|
||
MAX_A_CHARS = 2000 # 参考答案截断(太长会稀释 judge 注意力)
|
||
MAX_R_CHARS = 4000 # 待评分回答截断
|
||
|
||
|
||
def make_client():
|
||
# 默认继承环境里的 OPENAI_BASE_URL / OPENAI_API_KEY(与 codex 同一网关),
|
||
# 否则回退到本机 codex-proxy。
|
||
base = os.environ.get("JUDGE_BASE_URL") or os.environ.get("OPENAI_BASE_URL") or "http://127.0.0.1:8421/v1"
|
||
key = os.environ.get("JUDGE_API_KEY") or os.environ.get("OPENAI_API_KEY") or os.environ.get("LLM_API_KEY") or "sk-empty"
|
||
return OpenAI(base_url=base, api_key=key, timeout=180, max_retries=2)
|
||
|
||
|
||
def judge_one(client, model, item, response):
|
||
q_text = item["question"]
|
||
if item.get("type") == "image":
|
||
q_text = "(图片识别题,已给 agent 提供板卡图片)" + q_text
|
||
prompt = (TPL
|
||
.replace("{question}", q_text[:4000])
|
||
.replace("{answer}", item["reference_answer"][:MAX_A_CHARS])
|
||
.replace("{response}", response[:MAX_R_CHARS]))
|
||
resp = client.chat.completions.create(
|
||
model=model,
|
||
messages=[{"role": "user", "content": prompt}],
|
||
temperature=0,
|
||
max_tokens=1500,
|
||
).choices[0].message
|
||
raw = (resp.content or "").strip()
|
||
if not raw: # 部分模型把输出放在 reasoning_content,追加说明后重试一次
|
||
raw = client.chat.completions.create(
|
||
model=model,
|
||
messages=[{"role": "user", "content": prompt},
|
||
{"role": "assistant", "content": getattr(resp, "reasoning_content", "") or ""},
|
||
{"role": "user", "content": "请按要求的格式直接输出评分结果。"}],
|
||
temperature=0, max_tokens=1000,
|
||
).choices[0].message.content or ""
|
||
raw = raw.strip()
|
||
m = re.search(r"分数[::]\s*([1-5])", raw)
|
||
return {
|
||
"qid": item["qid"],
|
||
"category": item.get("category"), "model": item.get("model"), "type": item.get("type"), "difficulty": item.get("difficulty"),
|
||
"score": int(m.group(1)) if m else None,
|
||
"raw": raw,
|
||
"parse_ok": bool(m),
|
||
}
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--run-id", required=True)
|
||
ap.add_argument("--model", default=os.environ.get("JUDGE_MODEL", "glm-5.2"))
|
||
ap.add_argument("--workers", type=int, default=6)
|
||
ap.add_argument("--qid", type=int, nargs="*", default=None)
|
||
args = ap.parse_args()
|
||
|
||
run_dir = os.path.join(BASE, "results", args.run_id)
|
||
out_path = os.path.join(run_dir, "judged.jsonl")
|
||
items = {json.loads(l)["qid"]: json.loads(l) for l in open(EVALSET)}
|
||
# 只评本次运行实际跑过的题目(answers/ 或 traces/ 里有记录的)
|
||
ran_qids = set()
|
||
for d in ("answers", "traces"):
|
||
dd = os.path.join(run_dir, d)
|
||
if os.path.isdir(dd):
|
||
for fn in os.listdir(dd):
|
||
stem = os.path.splitext(fn)[0]
|
||
if stem.isdigit():
|
||
ran_qids.add(int(stem))
|
||
if ran_qids:
|
||
items = {q: it for q, it in items.items() if q in ran_qids}
|
||
|
||
# 已评过的跳过(续跑)
|
||
done = {}
|
||
if os.path.exists(out_path):
|
||
for l in open(out_path):
|
||
try:
|
||
r = json.loads(l)
|
||
if r.get("parse_ok"):
|
||
done[r["qid"]] = r
|
||
except json.JSONDecodeError:
|
||
pass
|
||
|
||
todo = []
|
||
for qid, item in sorted(items.items()):
|
||
if qid in done or (args.qid and qid not in set(args.qid)):
|
||
continue
|
||
ans_file = os.path.join(run_dir, "answers", f"{qid:04d}.txt")
|
||
resp = open(ans_file).read().strip() if os.path.exists(ans_file) else ""
|
||
if not resp:
|
||
rec = {"qid": qid, "category": item.get("category"), "model": item.get("model"), "type": item.get("type"), "difficulty": item.get("difficulty"), "score": None,
|
||
"raw": "(agent 无回答)", "parse_ok": True}
|
||
with open(out_path, "a") as f:
|
||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||
done[qid] = rec
|
||
continue
|
||
todo.append(item)
|
||
|
||
print(f"judge: {len(todo)} 题待评(已有 {len(done)}) model={args.model}", flush=True)
|
||
client = make_client()
|
||
t0 = time.time()
|
||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||
futs = {ex.submit(judge_one, client, args.model, it,
|
||
open(os.path.join(run_dir, "answers", f"{it['qid']:04d}.txt")).read()): it
|
||
for it in todo}
|
||
n = 0
|
||
for fut in concurrent.futures.as_completed(futs):
|
||
it = futs[fut]
|
||
try:
|
||
rec = fut.result()
|
||
except Exception as e:
|
||
rec = {"qid": it["qid"], "category": it["category"], "score": None,
|
||
"raw": f"judge error: {e}", "parse_ok": False}
|
||
with open(out_path, "a") as f:
|
||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||
done[rec["qid"]] = rec
|
||
n += 1
|
||
print(f" [{n}/{len(todo)}] qid={it['qid']:>3} score={rec['score']}", flush=True)
|
||
|
||
# 汇总
|
||
scores = [r["score"] for r in done.values() if r.get("score")]
|
||
by_cat = {}
|
||
by_model = {}
|
||
by_type = {}
|
||
for r in done.values():
|
||
if r.get("score"):
|
||
by_cat.setdefault(r.get("category") or "?", []).append(r["score"])
|
||
by_model.setdefault(r.get("model") or "?", []).append(r["score"])
|
||
by_type.setdefault(r.get("type") or "text", []).append(r["score"])
|
||
print(f"\n== 汇总({args.run_id})==")
|
||
print(f"有效评分 {len(scores)}/{len(done)},平均分 {sum(scores)/len(scores):.2f}" if scores else "无有效评分")
|
||
dist = {s: scores.count(s) for s in range(1, 6)}
|
||
print("分数分布:", {k: v for k, v in sorted(dist.items())})
|
||
print("按题型均分:", {c: round(sum(v)/len(v), 2) for c, v in sorted(by_type.items())})
|
||
print("按型号均分:", {c: round(sum(v)/len(v), 2) for c, v in sorted(by_model.items())})
|
||
print(f"耗时 {time.time()-t0:.0f}s")
|
||
|
||
# 若已有轨迹统计,合并输出检索速率
|
||
tr_path = os.path.join(run_dir, "trace_report.json")
|
||
if os.path.exists(tr_path):
|
||
rows = json.load(open(tr_path))
|
||
avg_call = [r["avg_kb_call_s"] for r in rows if r.get("avg_kb_call_s") is not None]
|
||
ratio = [r["kb_time_ratio"] for r in rows if r.get("kb_time_ratio") is not None]
|
||
lat = [r["latency_s"] for r in rows if r.get("latency_s")]
|
||
print(f"\n== 检索速率 ==")
|
||
print(f"知识库调用: {sum(r['kb_calls'] for r in rows)} 次,单次平均 {sum(avg_call)/len(avg_call):.2f}s" if avg_call else "无知识库调用")
|
||
if ratio:
|
||
print(f"检索耗时占比平均 {sum(ratio)/len(ratio)*100:.0f}%,每题总耗时平均 {sum(lat)/len(lat):.1f}s")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|