55 lines
2.3 KiB
Python
55 lines
2.3 KiB
Python
#!/usr/bin/env python3
|
|
"""09_pilot_quality.py — 试点标注质量分析
|
|
输出: judge vs hint 矛盾率(合成质量核心指标)、软标签分布、各来源统计
|
|
"""
|
|
import json, os, sys
|
|
from collections import Counter, defaultdict
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
D = lambda n: os.path.join(HERE, "..", "data", n)
|
|
|
|
def main():
|
|
labels = {json.loads(l)["pair_id"]: json.loads(l) for l in open(D("labels.jsonl"), encoding="utf-8")}
|
|
pairs = [json.loads(l) for l in open(D("pairs_candidate.jsonl"), encoding="utf-8")]
|
|
errors = sum(1 for _ in open(D("labels_errors.jsonl"), encoding="utf-8")) if os.path.exists(D("labels_errors.jsonl")) else 0
|
|
|
|
src_stats = defaultdict(lambda: {"n": 0, "p_sum": 0.0, "conflict": 0, "bands": Counter()})
|
|
for p in pairs:
|
|
lab = labels.get(p["pair_key"]) or labels.get(json.dumps({})) # pair_id 复用 pair_key
|
|
lab = labels.get(p["pair_key"])
|
|
if not lab:
|
|
continue
|
|
pj, src = lab["judge1"]["p"], p["source"]
|
|
s = src_stats[src]
|
|
s["n"] += 1; s["p_sum"] += pj
|
|
band = "high(>0.8)" if pj > 0.8 else "mid(0.2-0.8)" if pj >= 0.2 else "low(<0.2)"
|
|
s["bands"][band] += 1
|
|
hint = p.get("label_hint")
|
|
if hint is not None:
|
|
if hint >= 0.9 and pj < 0.5:
|
|
s["conflict"] += 1
|
|
elif hint <= 0.1 and pj > 0.5:
|
|
s["conflict"] += 1
|
|
|
|
total = sum(s["n"] for s in src_stats.values())
|
|
print(f"已标注 {total} 对, 解析/请求失败 {errors}")
|
|
print(f"{'来源':<22}{'数量':>6}{'均分':>7}{'矛盾':>6} 分布")
|
|
for src, s in sorted(src_stats.items()):
|
|
print(f"{src:<22}{s['n']:>6}{s['p_sum']/s['n']:>7.3f}{s['conflict']:>6} {dict(s['bands'])}")
|
|
|
|
# 合成质量: synthetic_source 的 hint=expected_relevance 与 judge 的相关方向
|
|
agree_full = agree_none = 0
|
|
for p in pairs:
|
|
if p["source"] != "synthetic_source":
|
|
continue
|
|
lab = labels.get(p["pair_key"])
|
|
if not lab:
|
|
continue
|
|
pj, hint = lab["judge1"]["p"], p["label_hint"]
|
|
if hint >= 0.9 and pj > 0.8: agree_full += 1
|
|
if hint <= 0.1 and pj < 0.2: agree_none += 1
|
|
print(f"\n合成 query 验证: 裁判确认 full 的比率见上表 synthetic_source 行")
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|