Eval debug

评估系统调试分析

目标文件：$ARGUMENTS（留空取最新）

---

Step 1：定位文件

ls -lt rag/code/logs/*.jsonl | head -5

Step 2：运行诊断脚本

python3 - << 'EOF'
import json, sys
from pathlib import Path
from collections import defaultdict, Counter

log_path = "$ARGUMENTS" or sorted(
    Path("rag/code/logs").glob("*.jsonl"),
    key=lambda p: p.stat().st_mtime
)[-1]

records = []
for line in Path(log_path).read_text().splitlines():
    try:
        r = json.loads(line)
        if "question_id" in r:
            records.append(r)
    except:
        pass

# ── 1. 模型可用性 ──
print("=== 模型可用性 ===")
model_status = defaultdict(lambda: {"total": 0, "failed": 0, "errors": []})
for r in records:
    for e in r.get("model_evals", []):
        m = e["model"]
        model_status[m]["total"] += 1
        if e["score"] < 0:
            model_status[m]["failed"] += 1
            err = e.get("error", e.get("feedback", ""))[:60]
            if err not in model_status[m]["errors"]:
                model_status[m]["errors"].append(err)

for model, stat in model_status.items():
    fail_rate = stat["failed"] / stat["total"] * 100 if stat["total"] else 0
    status = "✗ 全部失败" if stat["failed"] == stat["total"] else \
             f"⚠ {stat['failed']}/{stat['total']} 失败" if stat["failed"] else "✓ 正常"
    print(f"  {model.split('/')[-1]:<20} {status}")
    for err in stat["errors"][:1]:
        print(f"    原因: {err}")

# ── 2. 主裁判健康度 ──
print("\n=== 主裁判（primary model）健康度 ===")
primary_failures = 0
for r in records:
    evals = r.get("model_evals", [])
    if evals and evals[0]["score"] < 0:
        primary_failures += 1

print(f"  主裁判失败次数: {primary_failures}/{len(records)}")
if primary_failures > 0:
    print(f"  ⚠ BUG: key_points_hit/missed/feedback 来自主裁判，主裁判失败时这些字段为空")
    print(f"  影响: 所有题目的定性分析（命中/遗漏要点）不可信")

# ── 3. 得分准确性审查 ──
print("\n=== 得分合理性审查 ===")
for r in records:
    qid = r["question_id"]
    user_ans = r["user_answer"][:80]
    agg_score = r["score"]
    max_s = r["max_score"]
    evals = r.get("model_evals", [])
    valid_evals = [e for e in evals if e["score"] >= 0]
    scores = [e["score"] for e in valid_evals]

    issues = []

    # 模型分歧
    if len(scores) >= 2 and max(scores) - min(scores) >= 2:
        issues.append(f"模型分歧大 {[f'{e[\"model\"].split(\"/\")[-1]}:{e[\"score\"]}' for e in valid_evals]}")

    # 仅1个有效模型
    if len(valid_evals) == 1:
        issues.append(f"仅1个模型有效，得分可信度低")

    # 得分为0但回答非空
    if agg_score == 0 and len(user_ans.strip()) > 10:
        issues.append("得分0但有实质回答，可能误判")

    # 有 errors 但 score 不为0
    primary = evals[0] if evals else {}
    for e in valid_evals:
        if e.get("errors") and e["score"] > 0:
            issues.append(f"{e['model'].split('/')[-1']} 标记了错误但仍给分，逻辑需确认")

    print(f"\n[{qid}] {agg_score}/{max_s}")
    print(f"  回答: {user_ans}")
    if issues:
        for iss in issues:
            print(f"  ⚠ {iss}")
    else:
        print(f"  ✓ 评估稳定")

# ── 4. 误判检测 ──
print("\n=== 潜在误判（False Positive errors）===")
for r in records:
    for e in r.get("model_evals", []):
        if e.get("errors") and e["score"] >= 0:
            print(f"  [{r['question_id']}] {e['model'].split('/')[-1]}")
            print(f"  回答: {r['user_answer'][:80]}")
            for err in e["errors"]:
                print(f"  标记错误: {err}")
            print()

# ── 5. 程序逻辑校验 ──
print("=== 程序逻辑校验 ===")
for r in records:
    evals = r.get("model_evals", [])
    valid = [e for e in evals if e["score"] >= 0]

    # 校验 aggregated score 是否等于 valid scores 均值
    if valid:
        expected = round(sum(e["score"] for e in valid) / len(valid), 1)
        actual = r["score"]
        if abs(expected - actual) > 0.15:
            print(f"  ⚠ [{r['question_id']}] 得分聚合异常: 期望={expected} 实际={actual}")

    # 校验 key_points_hit + key_points_missed == all key_points (仅当主裁判正常时)
    if evals and evals[0]["score"] >= 0:
        all_kp = set(r.get("key_points_hit", []) + r.get("key_points_missed", []))
        # 只能粗判，不强要求

print("  (得分聚合逻辑校验完成)")

EOF

Step 3：输出结构化 Bug 报告

格式：

## 模型可用性
| 模型 | 状态 | 失败原因 |

## 发现的 Bug
| 严重级别 | 问题 | 影响 | 建议修复 |
| P0 | 主裁判失败时 key_points_hit/missed 为空 | 定性分析不可信 | fallback 到第一个成功的模型 |

## 得分可信度
每题：得分 / 有效模型数 / 是否有分歧

## 误判风险
列出 errors 字段中可疑的"误判"条目

## 改进建议
优先级排序的修复清单