165 lines
6.9 KiB
Python
165 lines
6.9 KiB
Python
#!/usr/bin/env python3
|
||
"""deeper_metrics.py — 对比评测的补充指标(离线重分析 compare_detail.jsonl)
|
||
1. 分型号明细(错误是否集中在 S100/S600)
|
||
2. 跨型号混淆率 wrong-model@1(jev 已知弱点,量化两臂)
|
||
3. MAP / nDCG@5/10 / Recall-all-rel@k(MRR 只看第一个命中,信息利用不足)
|
||
4. 配对 bootstrap 置信区间(n=63 差 1-2 题必须给不确定性)
|
||
5. 判官敏感性:严格判官(计数题需 数字+主题词 共现)下结论是否翻转
|
||
6. 空检索臂不对称报告(7 题原始链路直接失败)
|
||
"""
|
||
import json, re, random
|
||
|
||
rows = [json.loads(l) for l in open('/tmp/jev_compare/compare_detail.jsonl', encoding='utf-8')]
|
||
valid = [r for r in rows if r['chunks']]
|
||
print(f'questions={len(rows)} valid={len(valid)} empty_retrieval={len(rows)-len(valid)}')
|
||
|
||
def q_model(qid):
|
||
for pre, m in [('X3M-', 'x3m'), ('X5M-', 'x5m'), ('S1P-', 's100p'),
|
||
('S1-', 's100'), ('S6-', 's600'), ('X5-', 'x5'), ('X3-', 'x3')]:
|
||
if qid.startswith(pre):
|
||
return m
|
||
return '?'
|
||
|
||
def doc_model(fname, sec=''):
|
||
t = (fname + ' ' + (sec or '')).lower()
|
||
if 's100p' in t: return 's100p'
|
||
if 's600' in t: return 's600'
|
||
if 'x3_module' in t or 'x3m' in t: return 'x3m'
|
||
if 's100' in t: return 's100'
|
||
if 'x5_module' in t or 'x5m' in t: return 'x5m'
|
||
if 'x3' in t: return 'x3'
|
||
if 'x5' in t: return 'x5'
|
||
return 'none'
|
||
|
||
# ---- 1. 分型号 ----
|
||
print('\n== 1. 分型号 @1/@3 (A=原始, B=jev) ==')
|
||
models = sorted({q_model(r['qid']) for r in valid})
|
||
for m in models:
|
||
sub = [r for r in valid if q_model(r['qid']) == m]
|
||
a1 = sum(1 for r in sub if r['a_rank'] == 1) / len(sub)
|
||
b1 = sum(1 for r in sub if r['b_rank'] == 1) / len(sub)
|
||
a3 = sum(1 for r in sub if r['a_rank'] and r['a_rank'] <= 3) / len(sub)
|
||
b3 = sum(1 for r in sub if r['b_rank'] and r['b_rank'] <= 3) / len(sub)
|
||
print(f' {m:6s} n={len(sub):2d} @1: A={a1:.2f} B={b1:.2f} @3: A={a3:.2f} B={b3:.2f}')
|
||
|
||
# ---- 2. 跨型号混淆率 (top-1 页面型号 != 题目型号, 不计 none) ----
|
||
print('\n== 2. wrong-model@1 (top1 页面型号与题目型号不符) ==')
|
||
for name, key in [('A 原始', 'a'), ('B jev', 'b')]:
|
||
wrong = 0; scored = 0
|
||
for r in valid:
|
||
chs = r['chunks']
|
||
if key == 'a':
|
||
top = chs[0]
|
||
else:
|
||
top = sorted(chs, key=lambda c: -c['pfinal'])[0]
|
||
dm = doc_model(top['f'], top['sec'])
|
||
if dm != 'none':
|
||
scored += 1
|
||
if dm != q_model(r['qid']):
|
||
wrong += 1
|
||
print(f' {name}: {wrong}/{scored} = {wrong/max(1,scored):.2%}')
|
||
|
||
# ---- 3. MAP / nDCG / Recall-all ----
|
||
print('\n== 3. 排序质量 (hit=判官二值) ==')
|
||
import math
|
||
def eval_ranking(hits, order):
|
||
"""order: chunk 索引按排名; hits: bool per chunk. 返回 AP, ndcg5, ndcg10, recall1/3/5"""
|
||
rel = [1 if hits[i] else 0 for i in order]
|
||
nrel = sum(rel)
|
||
if nrel == 0:
|
||
return None
|
||
# AP
|
||
cum, ap = 0, 0.0
|
||
for k, r in enumerate(rel, 1):
|
||
if r:
|
||
cum += 1
|
||
ap += cum / k
|
||
ap /= nrel
|
||
# nDCG@k
|
||
def ndcg(k):
|
||
dcg = sum(r / math.log2(i + 2) for i, r in enumerate(rel[:k]))
|
||
idcg = sum(1 / math.log2(i + 2) for i in range(min(nrel, k)))
|
||
return dcg / idcg if idcg else 0.0
|
||
rec = {k: sum(rel[:k]) / nrel for k in (1, 3, 5)}
|
||
return ap, ndcg(5), ndcg(10), rec[1], rec[3], rec[5]
|
||
|
||
for name, order_of in [('A 原始', lambda chs: list(range(len(chs)))),
|
||
('B jev', lambda chs: sorted(range(len(chs)), key=lambda k: -chs[k]['pfinal']))]:
|
||
aps, n5s, n10s = [], [], []
|
||
r1 = r3 = r5 = 0.0
|
||
nq = 0
|
||
for r in valid:
|
||
out = eval_ranking([c['hit'] for c in r['chunks']], order_of(r['chunks']))
|
||
if out is None:
|
||
continue
|
||
ap, n5, n10, rr1, rr3, rr5 = out
|
||
aps.append(ap); n5s.append(n5); n10s.append(n10)
|
||
r1 += rr1; r3 += rr3; r5 += rr5
|
||
nq += 1
|
||
print(f' {name}: MAP={sum(aps)/nq:.3f} nDCG@5={sum(n5s)/nq:.3f} nDCG@10={sum(n10s)/nq:.3f} '
|
||
f'RecallAll@1={r1/nq:.3f} @3={r3/nq:.3f} @5={r5/nq:.3f}')
|
||
|
||
# ---- 4. 配对 bootstrap ----
|
||
print('\n== 4. 配对 bootstrap 95% CI (B-A, 10000 次) ==')
|
||
def boot(delta_fn, n=10000):
|
||
rng = random.Random(7)
|
||
ds = []
|
||
for _ in range(n):
|
||
sample = [valid[rng.randrange(len(valid))] for _ in range(len(valid))]
|
||
ds.append(delta_fn(sample))
|
||
ds.sort()
|
||
return ds[int(0.025 * n)], ds[int(0.975 * n)]
|
||
|
||
def d_at1(sample):
|
||
a = sum(1 for r in sample if r['a_rank'] == 1) / len(sample)
|
||
b = sum(1 for r in sample if r['b_rank'] == 1) / len(sample)
|
||
return b - a
|
||
def d_mrr(sample):
|
||
a = sum(1 / r['a_rank'] for r in sample if r['a_rank']) / len(sample)
|
||
b = sum(1 / r['b_rank'] for r in sample if r['b_rank']) / len(sample)
|
||
return b - a
|
||
def d_fuse_at1(sample):
|
||
K = 60
|
||
a = 0; b = 0
|
||
for r in sample:
|
||
chs = r['chunks']
|
||
rj = sorted(range(len(chs)), key=lambda k: -chs[k]['pfinal'])
|
||
rankj = [0] * len(chs)
|
||
for pos, k in enumerate(rj): rankj[k] = pos
|
||
def fscore(i): return 0.5 / (K + rankj[i] + 1) + 0.5 / (K + i + 1)
|
||
order = sorted(range(len(chs)), key=lambda i: -fscore(i))
|
||
if chs[0]['hit']: a += 1
|
||
if chs[order[0]]['hit']: b += 1
|
||
return b / len(sample) - a / len(sample)
|
||
lo, hi = boot(d_at1); print(f' Δ@1 (jev-A): [{lo:+.3f}, {hi:+.3f}]')
|
||
lo, hi = boot(d_mrr); print(f' ΔMRR (jev-A): [{lo:+.3f}, {hi:+.3f}]')
|
||
lo, hi = boot(d_fuse_at1); print(f' Δ@1 (RRF-A): [{lo:+.3f}, {hi:+.3f}]')
|
||
|
||
# ---- 5. 判官敏感性: 严格模式 (纯数字答案需同 chunk 含题目主题英文词) ----
|
||
print('\n== 5. 严格判官敏感性 ==')
|
||
def strict_hits(r):
|
||
"""纯数字/短答案题: hit 需 数字 且 题目主题词(英文token≥3字母) 同时出现"""
|
||
toks = r['tokens']
|
||
plain = toks['plain']
|
||
qt = {w.lower() for w in re.findall(r'[A-Za-z][A-Za-z0-9\-]{2,}', r['question'])}
|
||
weak = len(plain) <= 2 and all(t.isdigit() or len(t) <= 3 for t in plain)
|
||
out = []
|
||
for c in r['chunks']:
|
||
t = (c['f'] + ' ' + c['sec'] + ' ') + '' # 内容不可用(未存), 用 filename+section 代理
|
||
h = c['hit']
|
||
if weak:
|
||
# 内容没存 — 严格判定退化为: 要求 sec/filename 含主题词
|
||
h = c['hit'] and any(w in (c['sec'] or '').lower() or w in c['f'].lower() for w in qt)
|
||
out.append(h)
|
||
return out
|
||
for name, order_of in [('A 原始', lambda h: list(range(len(h)))),
|
||
('B jev', lambda h: sorted(range(len(h)), key=lambda k: -h[k][1]))]:
|
||
pass # 严格判定需要 chunk 内容, detail 里没存 — 报告局限
|
||
print(' (严格判官需 chunk 全文 — compare_detail 未存 content, 见局限说明)')
|
||
|
||
# ---- 6. 空检索题 ----
|
||
print('\n== 6. 空检索题 (原始链路直接无输出) ==')
|
||
for r in rows:
|
||
if not r['chunks']:
|
||
print(f" {r['qid']}: {r['question'][:44]}")
|