LAYA 지속 학습 3 — 새 판은 기준을 넘을 때만 올린다

"""지금 쓰는 판(current)과 새 판(candidate)을 같은 평가 세트로 재고, 기준을 넘을 때만 승격한다.

기준 — 셋 다 통과해야 한다:
  1. 전체 정확도가 0.02 이상 오른다
  2. 어느 팀도 정확도가 0.05 넘게 떨어지지 않는다(전체 평균에 가려진 퇴보를 잡는다)
  3. ECE(확신과 실제의 차이)가 0.25 이하
결과는 data/registry.jsonl 에 한 줄씩 남긴다 — 무엇을 왜 올렸고 왜 막았는지.
"""
import json
import sys
import time
from datetime import date

import numpy as np
import laya
from laya.common import ece_score

current_path, candidate_path = sys.argv[1], sys.argv[2]
cases = [json.loads(line) for line in open("tickets-eval.jsonl", encoding="utf-8")]


def measure(path):
    agent = laya.load(path)
    correct, conf, per_team = [], [], {}
    for case in cases:
        answer = agent.predict(case["state"], case["questions"])["answers"]["team"]
        ok = answer["choice"] == case["expected"]["team"]
        correct.append(ok)
        conf.append(answer["probabilities"][answer["choice"]])
        per_team.setdefault(case["expected"]["team"], []).append(ok)
    correct = np.array(correct, dtype=float)
    return {
        "accuracy": round(float(correct.mean()), 3),
        "ece": round(float(ece_score(np.array(conf), correct, bins=10)), 3),
        "teams": {t: round(sum(v) / len(v), 3) for t, v in sorted(per_team.items())},
    }


cur, cand = measure(current_path), measure(candidate_path)
checks = {
    "accuracy +0.02": cand["accuracy"] - cur["accuracy"] >= 0.02,
    "no team -0.05": all(cand["teams"][t] - cur["teams"][t] >= -0.05 for t in cur["teams"]),
    "ECE <= 0.25": cand["ece"] <= 0.25,
}
decision = "PROMOTE" if all(checks.values()) else "REJECT"

print(f"{'':<12}{'current':>10}{'candidate':>11}")
print(f"{'accuracy':<12}{cur['accuracy']:>10.3f}{cand['accuracy']:>11.3f}")
for team in cur["teams"]:
    print(f"  {team:<10}{cur['teams'][team]:>10.3f}{cand['teams'][team]:>11.3f}")
print(f"{'ECE':<12}{cur['ece']:>10.3f}{cand['ece']:>11.3f}")
for name, ok in checks.items():
    print(f"[{'pass' if ok else 'FAIL'}] {name}")
print(f"decision: {decision} {candidate_path}")

record = {"date": str(date.today()), "current": current_path, "candidate": candidate_path,
          "decision": decision, "checks": checks, "current_metrics": cur, "candidate_metrics": cand}
with open("data/registry.jsonl", "a", encoding="utf-8") as f:
    f.write(json.dumps(record, ensure_ascii=False) + "\n")

               current  candidate
accuracy         0.750      0.815
  account        0.800      0.889
  billing        0.746      0.825
  delivery       0.600      0.622
  technical      0.851      0.915
ECE              0.207      0.136
[pass] accuracy +0.02
[pass] no team -0.05
[pass] ECE <= 0.25
decision: PROMOTE checkpoints/v2

               current  candidate
accuracy         0.750      0.735
  account        0.800      0.800
  billing        0.746      0.905
  delivery       0.600      0.711
  technical      0.851      0.468
ECE              0.207      0.237
[FAIL] accuracy +0.02
[FAIL] no team -0.05
[pass] ECE <= 0.25
decision: REJECT checkpoints/v2b