학습 전에 먼저 잰다 — 기본 LAYA 는 우리 문의를 55% 맞혔다

"""평가 세트(JSONL)로 체크포인트 하나를 잰다 — 팀 정확도·팀별 정확도·평균 확신·ECE·시간.

python evaluate.py base               # 학습 전 다국어 체크포인트
python evaluate.py ../checkpoints/v1  # 내가 학습한 체크포인트
"""
import json
import sys
import time
from collections import Counter

import numpy as np
import laya
from laya.common import ece_score

target = sys.argv[1]
eval_path = sys.argv[2] if len(sys.argv) > 2 else "tickets-eval.jsonl"

if target == "base":
    agent = laya.load("convaiinnovations/laya", subfolder="multilingual")
else:
    agent = laya.load(target)

cases = [json.loads(line) for line in open(eval_path, encoding="utf-8")]
correct, confidence, misses = [], [], Counter()
per_team = {}
start = time.time()
for case in cases:
    result = agent.predict(case["state"], case["questions"])
    team = result["answers"]["team"]
    gold = case["expected"]["team"]
    ok = team["choice"] == gold
    correct.append(ok)
    confidence.append(team["probabilities"][team["choice"]])
    per_team.setdefault(gold, []).append(ok)
    if not ok:
        misses[f"{gold} -> {team['choice']}"] += 1
elapsed = time.time() - start

correct = np.array(correct, dtype=float)
confidence = np.array(confidence)
print(f"model: {target}")
print(f"accuracy: {correct.mean():.3f} ({int(correct.sum())}/{len(cases)})")
for gold, oks in sorted(per_team.items()):
    print(f"  {gold:<9} {sum(oks) / len(oks):.3f} ({sum(oks)}/{len(oks)})")
print(f"mean confidence: {confidence.mean():.3f}")
print(f"ECE: {ece_score(confidence, correct, bins=10):.3f}")
print("most common mistakes:", ", ".join(f"{k} ×{v}" for k, v in misses.most_common(3)) or "none")
print(f"time: {elapsed / len(cases) * 1000:.0f} ms per case")

model: base
accuracy: 0.550 (110/200)
  account   0.222 (10/45)
  billing   0.397 (25/63)
  delivery  0.622 (28/45)
  technical 1.000 (47/47)
mean confidence: 0.846
ECE: 0.297
most common mistakes: billing -> technical ×34, account -> technical ×24, delivery -> technical ×17
time: 37 ms per case