LAYA 쇼핑 검색 3 — v1 결과 읽기: 배운 단어는 80%, 처음 보는 단어는 0%

"""검색어 평가 세트로 판 여러 개를 나란히 잰다 — 완전 일치(여섯 칸 모두 맞음) · 어휘 묶음별 · 칸별.

python shop_eval.py base checkpoints/shop-v1
"""
import json
import sys
import time
from collections import Counter

import laya

FIELDS = ["price", "category", "item", "winter", "gender", "color"]
POOLS = ["train", "ops", "unseen"]


def load(target):
    if target == "base":
        return laya.load("convaiinnovations/laya", subfolder="multilingual")
    return laya.load(target)


def filters(answers):
    """LAYA 답을 필터 값으로 — choice 는 고른 값, noul 은 0.5 이상이면 True."""
    return {f: (answers[f]["noul"] >= 0.5 if answers[f]["type"] == "noul" else answers[f]["choice"]) for f in FIELDS}


def measure(target, cases):
    agent = load(target)
    start = time.time()
    preds = [filters(agent.predict(c["state"], c["questions"])["answers"]) for c in cases]
    ms = (time.time() - start) / len(cases) * 1000
    exact = [all(p[f] == c["expected"][f] for f in FIELDS) for c, p in zip(cases, preds)]
    row = {"exact match": sum(exact) / len(cases)}
    for pool in POOLS:
        hits = [e for c, e in zip(cases, exact) if c["pool"] == pool]
        row[f"  {pool} words"] = sum(hits) / len(hits)
    for f in FIELDS:
        row[f] = sum(p[f] == c["expected"][f] for c, p in zip(cases, preds)) / len(cases)
    row["ms per query"] = ms
    misses = Counter(f"{f} {c['expected'][f]} -> {p[f]}" for c, p in zip(cases, preds)
                     for f in FIELDS if p[f] != c["expected"][f])
    return row, misses


if __name__ == "__main__":
    targets = sys.argv[1:]
    cases = [json.loads(line) for line in open("shop-eval.jsonl", encoding="utf-8")]
    results = [measure(t, cases) for t in targets]
    names = [t.split("/")[-1] for t in targets]
    print(f"{len(cases)} queries · " + " · ".join(f"{p} words {sum(c['pool'] == p for c in cases)}" for p in POOLS))
    print(f"{'':<16}" + "".join(f"{n:>10}" for n in names))
    for key in results[0][0]:
        fmt = "{:>10.0f}" if key == "ms per query" else "{:>10.3f}"
        print(f"{key:<16}" + "".join(fmt.format(r[key]) for r, _ in results))
    print(f"most common mistakes ({names[-1]}):")
    for k, v in results[-1][1].most_common(5):
        print(f"  {k} ×{v}")

210 queries · train words 70 · ops words 70 · unseen words 70
                      base   shop-v1
exact match          0.014     0.305
  train words        0.029     0.800
  ops words          0.014     0.114
  unseen words       0.000     0.000
price                0.590     0.800
category             0.138     0.671
item                 0.595     0.638
winter               0.457     0.852
gender               0.629     0.743
color                0.505     0.605
ms per query            92        90
most common mistakes (shop-v1):
  gender men -> unknown ×33
  winter True -> False ×30
  gender women -> unknown ×21
  item sneakers -> none ×18
  color gray -> none ×17

검색어: 50만원이하 패딩 상품중 갈색
  price    under50   확신 0.99
  category top       확신 0.99
  item     padding   확신 0.99
  winter   1.00      확신 1.00
  gender   unknown   확신 0.95
  color    brown     확신 0.99
  filters: {"price_under50": true, "category_top": true, "item_padding": true, "winter": true, "gender": null, "color_brown": true}
검색어: 아빠 겨울 부츠 브라운으로
  price    none      확신 0.99
  category shoes     확신 0.98
  item     boots     확신 0.98
  winter   1.00      확신 1.00
  gender   unknown   확신 0.87
  color    brown     확신 0.98
  filters: {"price": null, "category_shoes": true, "item_boots": true, "winter": true, "gender": null, "color_brown": true}
검색어: 언니 선물로 십만 원 이하 밤색 터틀넥
  price    none      확신 0.81
  category top       확신 0.96
  item     none      확신 0.97
  winter   0.00      확신 1.00
  gender   unknown   확신 0.87
  color    none      확신 0.89
  filters: {"price": null, "category_top": true, "item": null, "winter": false, "gender": null, "color": null}

"""확신 문턱마다 — 필터가 몇 개 걸리고, 걸린 필터 중 몇 개가 틀리며, 틀린 필터가 하나라도 걸린 검색어가 몇인지.

틀린 필터가 하나라도 걸리면 그 검색은 엉뚱한 결과(대개 0건)를 낸다. 문턱은 이 숫자를 보고 정한다.
python threshold.py checkpoints/shop-v1
"""
import json
import sys

import laya

FIELDS = ["price", "category", "item", "winter", "gender", "color"]
EMPTY = {"none", "unknown", False}
agent = laya.load(sys.argv[1])
cases = [json.loads(line) for line in open("shop-eval.jsonl", encoding="utf-8")]

answers = []   # 검색어마다 [(칸, 값, 확신, 맞았나)]
for c in cases:
    a = agent.predict(c["state"], c["questions"])["answers"]
    row = []
    for f in FIELDS:
        value = a[f]["noul"] >= 0.5 if a[f]["type"] == "noul" else a[f]["choice"]
        row.append((f, value, a[f]["answer_confidence"], value == c["expected"][f]))
    answers.append((c, row))

wanted = sum(c["expected"][f] not in EMPTY for c in cases for f in FIELDS)
print(f"queries {len(cases)} · filters the queries ask for {wanted}")
print(f"{'threshold':>9}{'applied':>9}{'wrong':>7}{'wrong %':>9}{'queries hit':>13}{'  unseen hit':>13}")
for t in (0.5, 0.7, 0.9, 0.99):
    applied = wrong = hit = unseen_hit = 0
    for c, row in answers:
        bad = False
        for f, value, conf, ok in row:
            if value in EMPTY or conf < t:
                continue          # 걸지 않는 칸
            applied += 1
            if not ok:
                wrong += 1
                bad = True
        hit += bad
        unseen_hit += bad and c["pool"] == "unseen"
    print(f"{t:>9}{applied:>9}{wrong:>7}{wrong / applied:>9.1%}{hit:>13}{unseen_hit:>13}")