- evalset.jsonl: 由 rdk_eval_dataset.json 转换,含 model/type/difficulty/reference_answer - prepare_dataset.py: 改为官方文档集 -> evalset.jsonl(图片转绝对路径) - run_agent_eval.py: 图片题用 -i 传图(prompt 走 stdin),轨迹记录 sid/model/type - judge.py: 字段改 reference_answer,按题型/型号拆分汇总 - prompts/judge_prompt.md: 适配规格类短答案评分;agent_prompt 加图片题说明 - results/smoke_official/: 官方集冒烟(1 文本 + 1 图片),替换旧论坛集冒烟
55 lines
2.1 KiB
Python
55 lines
2.1 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""把官方文档评测集 rdk_eval_dataset.json 转成 runner 用的 evalset.jsonl。
|
||
|
||
源:rdk_eval/rdk_eval_dataset.json(7 型号 × 10 文本题 + 3 图片题 = 91 条,
|
||
基于 D-Robotics 官方文档中心构建,图片题为接口识别,image 为相对 rdk_eval/ 的路径)。
|
||
输出每行:qid, sid, model, type(text/image), category, difficulty,
|
||
question, reference_answer, image(图片题才有,绝对路径), source_pages。
|
||
"""
|
||
import json, os, argparse
|
||
|
||
BASE = "/Users/d-robotics/agent/知识检索/rdk_eval"
|
||
SRC = os.path.join(BASE, "rdk_eval_dataset.json")
|
||
OUT = "/Users/d-robotics/agent/知识检索/rdk_eval/agent_eval/evalset.jsonl"
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--out", default=OUT)
|
||
args = ap.parse_args()
|
||
|
||
d = json.load(open(SRC))
|
||
qid = 0
|
||
n_text = n_img = 0
|
||
with open(args.out, "w") as f:
|
||
for md in d["models"]:
|
||
for q in md.get("text_questions", []) + md.get("image_questions", []):
|
||
qid += 1
|
||
rec = {
|
||
"qid": qid,
|
||
"sid": q["id"],
|
||
"model": q["model"],
|
||
"type": q["type"],
|
||
"category": q.get("category"),
|
||
"difficulty": q.get("difficulty"),
|
||
"question": q["question"],
|
||
"reference_answer": q["reference_answer"],
|
||
"source_pages": md.get("source_pages", []),
|
||
}
|
||
if q["type"] == "image":
|
||
rec["image"] = os.path.join(BASE, q["image"])
|
||
n_img += 1
|
||
else:
|
||
n_text += 1
|
||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||
print(f"共 {qid} 条(文本 {n_text} + 图片 {n_img})-> {args.out}")
|
||
# 校验图片存在
|
||
miss = [l for l in open(args.out)
|
||
if json.loads(l)["type"] == "image" and not os.path.exists(json.loads(l)["image"])]
|
||
print(f"图片缺失: {len(miss)}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|