165 lines
6.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""deeper_metrics.py — 对比评测的补充指标(离线重分析 compare_detail.jsonl)
1. 分型号明细(错误是否集中在 S100/S600)
2. 跨型号混淆率 wrong-model@1(jev 已知弱点,量化两臂)
3. MAP / nDCG@5/10 / Recall-all-rel@k(MRR 只看第一个命中,信息利用不足)
4. 配对 bootstrap 置信区间(n=63 差 1-2 题必须给不确定性)
5. 判官敏感性:严格判官(计数题需 数字+主题词 共现)下结论是否翻转
6. 空检索臂不对称报告(7 题原始链路直接失败)
"""
import json, re, random
rows = [json.loads(l) for l in open('/tmp/jev_compare/compare_detail.jsonl', encoding='utf-8')]
valid = [r for r in rows if r['chunks']]
print(f'questions={len(rows)} valid={len(valid)} empty_retrieval={len(rows)-len(valid)}')
def q_model(qid):
for pre, m in [('X3M-', 'x3m'), ('X5M-', 'x5m'), ('S1P-', 's100p'),
('S1-', 's100'), ('S6-', 's600'), ('X5-', 'x5'), ('X3-', 'x3')]:
if qid.startswith(pre):
return m
return '?'
def doc_model(fname, sec=''):
t = (fname + ' ' + (sec or '')).lower()
if 's100p' in t: return 's100p'
if 's600' in t: return 's600'
if 'x3_module' in t or 'x3m' in t: return 'x3m'
if 's100' in t: return 's100'
if 'x5_module' in t or 'x5m' in t: return 'x5m'
if 'x3' in t: return 'x3'
if 'x5' in t: return 'x5'
return 'none'
# ---- 1. 分型号 ----
print('\n== 1. 分型号 @1/@3 (A=原始, B=jev) ==')
models = sorted({q_model(r['qid']) for r in valid})
for m in models:
sub = [r for r in valid if q_model(r['qid']) == m]
a1 = sum(1 for r in sub if r['a_rank'] == 1) / len(sub)
b1 = sum(1 for r in sub if r['b_rank'] == 1) / len(sub)
a3 = sum(1 for r in sub if r['a_rank'] and r['a_rank'] <= 3) / len(sub)
b3 = sum(1 for r in sub if r['b_rank'] and r['b_rank'] <= 3) / len(sub)
print(f' {m:6s} n={len(sub):2d} @1: A={a1:.2f} B={b1:.2f} @3: A={a3:.2f} B={b3:.2f}')
# ---- 2. 跨型号混淆率 (top-1 页面型号 != 题目型号, 不计 none) ----
print('\n== 2. wrong-model@1 (top1 页面型号与题目型号不符) ==')
for name, key in [('A 原始', 'a'), ('B jev', 'b')]:
wrong = 0; scored = 0
for r in valid:
chs = r['chunks']
if key == 'a':
top = chs[0]
else:
top = sorted(chs, key=lambda c: -c['pfinal'])[0]
dm = doc_model(top['f'], top['sec'])
if dm != 'none':
scored += 1
if dm != q_model(r['qid']):
wrong += 1
print(f' {name}: {wrong}/{scored} = {wrong/max(1,scored):.2%}')
# ---- 3. MAP / nDCG / Recall-all ----
print('\n== 3. 排序质量 (hit=判官二值) ==')
import math
def eval_ranking(hits, order):
"""order: chunk 索引按排名; hits: bool per chunk. 返回 AP, ndcg5, ndcg10, recall1/3/5"""
rel = [1 if hits[i] else 0 for i in order]
nrel = sum(rel)
if nrel == 0:
return None
# AP
cum, ap = 0, 0.0
for k, r in enumerate(rel, 1):
if r:
cum += 1
ap += cum / k
ap /= nrel
# nDCG@k
def ndcg(k):
dcg = sum(r / math.log2(i + 2) for i, r in enumerate(rel[:k]))
idcg = sum(1 / math.log2(i + 2) for i in range(min(nrel, k)))
return dcg / idcg if idcg else 0.0
rec = {k: sum(rel[:k]) / nrel for k in (1, 3, 5)}
return ap, ndcg(5), ndcg(10), rec[1], rec[3], rec[5]
for name, order_of in [('A 原始', lambda chs: list(range(len(chs)))),
('B jev', lambda chs: sorted(range(len(chs)), key=lambda k: -chs[k]['pfinal']))]:
aps, n5s, n10s = [], [], []
r1 = r3 = r5 = 0.0
nq = 0
for r in valid:
out = eval_ranking([c['hit'] for c in r['chunks']], order_of(r['chunks']))
if out is None:
continue
ap, n5, n10, rr1, rr3, rr5 = out
aps.append(ap); n5s.append(n5); n10s.append(n10)
r1 += rr1; r3 += rr3; r5 += rr5
nq += 1
print(f' {name}: MAP={sum(aps)/nq:.3f} nDCG@5={sum(n5s)/nq:.3f} nDCG@10={sum(n10s)/nq:.3f} '
f'RecallAll@1={r1/nq:.3f} @3={r3/nq:.3f} @5={r5/nq:.3f}')
# ---- 4. 配对 bootstrap ----
print('\n== 4. 配对 bootstrap 95% CI (B-A, 10000 次) ==')
def boot(delta_fn, n=10000):
rng = random.Random(7)
ds = []
for _ in range(n):
sample = [valid[rng.randrange(len(valid))] for _ in range(len(valid))]
ds.append(delta_fn(sample))
ds.sort()
return ds[int(0.025 * n)], ds[int(0.975 * n)]
def d_at1(sample):
a = sum(1 for r in sample if r['a_rank'] == 1) / len(sample)
b = sum(1 for r in sample if r['b_rank'] == 1) / len(sample)
return b - a
def d_mrr(sample):
a = sum(1 / r['a_rank'] for r in sample if r['a_rank']) / len(sample)
b = sum(1 / r['b_rank'] for r in sample if r['b_rank']) / len(sample)
return b - a
def d_fuse_at1(sample):
K = 60
a = 0; b = 0
for r in sample:
chs = r['chunks']
rj = sorted(range(len(chs)), key=lambda k: -chs[k]['pfinal'])
rankj = [0] * len(chs)
for pos, k in enumerate(rj): rankj[k] = pos
def fscore(i): return 0.5 / (K + rankj[i] + 1) + 0.5 / (K + i + 1)
order = sorted(range(len(chs)), key=lambda i: -fscore(i))
if chs[0]['hit']: a += 1
if chs[order[0]]['hit']: b += 1
return b / len(sample) - a / len(sample)
lo, hi = boot(d_at1); print(f' Δ@1 (jev-A): [{lo:+.3f}, {hi:+.3f}]')
lo, hi = boot(d_mrr); print(f' ΔMRR (jev-A): [{lo:+.3f}, {hi:+.3f}]')
lo, hi = boot(d_fuse_at1); print(f' Δ@1 (RRF-A): [{lo:+.3f}, {hi:+.3f}]')
# ---- 5. 判官敏感性: 严格模式 (纯数字答案需同 chunk 含题目主题英文词) ----
print('\n== 5. 严格判官敏感性 ==')
def strict_hits(r):
"""纯数字/短答案题: hit 需 数字 且 题目主题词(英文token≥3字母) 同时出现"""
toks = r['tokens']
plain = toks['plain']
qt = {w.lower() for w in re.findall(r'[A-Za-z][A-Za-z0-9\-]{2,}', r['question'])}
weak = len(plain) <= 2 and all(t.isdigit() or len(t) <= 3 for t in plain)
out = []
for c in r['chunks']:
t = (c['f'] + ' ' + c['sec'] + ' ') + '' # 内容不可用(未存), 用 filename+section 代理
h = c['hit']
if weak:
# 内容没存 — 严格判定退化为: 要求 sec/filename 含主题词
h = c['hit'] and any(w in (c['sec'] or '').lower() or w in c['f'].lower() for w in qt)
out.append(h)
return out
for name, order_of in [('A 原始', lambda h: list(range(len(h)))),
('B jev', lambda h: sorted(range(len(h)), key=lambda k: -h[k][1]))]:
pass # 严格判定需要 chunk 内容, detail 里没存 — 报告局限
print(' (严格判官需 chunk 全文 — compare_detail 未存 content, 见局限说明)')
# ---- 6. 空检索题 ----
print('\n== 6. 空检索题 (原始链路直接无输出) ==')
for r in rows:
if not r['chunks']:
print(f" {r['qid']}: {r['question'][:44]}")