55 lines
2.3 KiB
Python

#!/usr/bin/env python3
"""09_pilot_quality.py — 试点标注质量分析
输出: judge vs hint 矛盾率(合成质量核心指标)、软标签分布、各来源统计
"""
import json, os, sys
from collections import Counter, defaultdict
HERE = os.path.dirname(os.path.abspath(__file__))
D = lambda n: os.path.join(HERE, "..", "data", n)
def main():
labels = {json.loads(l)["pair_id"]: json.loads(l) for l in open(D("labels.jsonl"), encoding="utf-8")}
pairs = [json.loads(l) for l in open(D("pairs_candidate.jsonl"), encoding="utf-8")]
errors = sum(1 for _ in open(D("labels_errors.jsonl"), encoding="utf-8")) if os.path.exists(D("labels_errors.jsonl")) else 0
src_stats = defaultdict(lambda: {"n": 0, "p_sum": 0.0, "conflict": 0, "bands": Counter()})
for p in pairs:
lab = labels.get(p["pair_key"]) or labels.get(json.dumps({})) # pair_id 复用 pair_key
lab = labels.get(p["pair_key"])
if not lab:
continue
pj, src = lab["judge1"]["p"], p["source"]
s = src_stats[src]
s["n"] += 1; s["p_sum"] += pj
band = "high(>0.8)" if pj > 0.8 else "mid(0.2-0.8)" if pj >= 0.2 else "low(<0.2)"
s["bands"][band] += 1
hint = p.get("label_hint")
if hint is not None:
if hint >= 0.9 and pj < 0.5:
s["conflict"] += 1
elif hint <= 0.1 and pj > 0.5:
s["conflict"] += 1
total = sum(s["n"] for s in src_stats.values())
print(f"已标注 {total} 对, 解析/请求失败 {errors}")
print(f"{'来源':<22}{'数量':>6}{'均分':>7}{'矛盾':>6} 分布")
for src, s in sorted(src_stats.items()):
print(f"{src:<22}{s['n']:>6}{s['p_sum']/s['n']:>7.3f}{s['conflict']:>6} {dict(s['bands'])}")
# 合成质量: synthetic_source 的 hint=expected_relevance 与 judge 的相关方向
agree_full = agree_none = 0
for p in pairs:
if p["source"] != "synthetic_source":
continue
lab = labels.get(p["pair_key"])
if not lab:
continue
pj, hint = lab["judge1"]["p"], p["label_hint"]
if hint >= 0.9 and pj > 0.8: agree_full += 1
if hint <= 0.1 and pj < 0.2: agree_none += 1
print(f"\n合成 query 验证: 裁判确认 full 的比率见上表 synthetic_source 行")
if __name__ == "__main__":
sys.exit(main())