LAYA 쇼핑 검색 4 — 필터 칩을 고친 검색어로 지속 학습: 0.322 에서 0.532 로

"""운영 흉내 — 검색어마다 v1 이 만든 필터를 보여 주고, 사용자가 필터 칩을 고친 검색어를 정답으로 모은다.

사용자 행동은 흉내다. 필터가 틀렸으면 FIX_RATE 만큼은 칩을 고치고(고친 결과 = 정답), 나머지는 그냥 떠난다.
맞았으면 아무도 고치지 않는다 — 그래서 고친 기록에는 v1 이 틀린 검색어만 담긴다.

python collect.py checkpoints/shop-v1 0.6
"""
import json
import random
import sys
from collections import Counter

import laya

FIELDS = ["price", "category", "item", "winter", "gender", "color"]
model, fix_rate = sys.argv[1], float(sys.argv[2])
rng = random.Random(7)
agent = laya.load(model)
incoming = [json.loads(line) for line in open("shop-incoming-300.jsonl", encoding="utf-8")]
first = [json.loads(line) for line in open("shop-first-150.jsonl", encoding="utf-8")]

fixed, left, right, fields = [], 0, 0, Counter()
for case in incoming:
    a = agent.predict(case["state"], case["questions"])["answers"]
    got = {f: (a[f]["noul"] >= 0.5 if a[f]["type"] == "noul" else a[f]["choice"]) for f in FIELDS}
    wrong = [f for f in FIELDS if got[f] != case["expected"][f]]
    if not wrong:
        right += 1
    elif rng.random() < fix_rate:
        fixed.append(case)
        fields.update(wrong)
    else:
        left += 1

pools = Counter(c["pool"] for c in incoming)
print(f"incoming: {len(incoming)} (ops words {pools['ops']} · train words {pools['train']})")
print(f"filters all right: {right} ({right / len(incoming):.0%})")
print(f"wrong: {len(fixed) + left} — user fixed chips {len(fixed)}, left without fixing {left}")
print("fields fixed:", ", ".join(f"{f} {fields[f]}" for f in FIELDS))
for name, rows in [("data/shop-feedback.jsonl", fixed), ("data/shop-train-v2.jsonl", first + fixed),
                   ("data/shop-train-all.jsonl", first + incoming)]:
    with open(name, "w", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print(f"saved {len(rows):>3} rows -> {name}")

"""이어 가는 방법 넷을 씨앗만 바꿔 세 번씩 학습해 같은 평가 세트로 잰다(맥 GPU 학습은 한 번 숫자가 흔들린다)."""
import json
import shutil
import statistics
import subprocess

import laya

FIELDS = ["price", "category", "item", "winter", "gender", "color"]
APPROACHES = {
    "v1 again (base + first 150)": ("shop-first-150.jsonl", "multilingual"),
    "continue v1 + fixed only": ("data/shop-feedback.jsonl", "checkpoints/shop-v1"),
    "retrain base + first + fixed": ("data/shop-train-v2.jsonl", "multilingual"),
    "retrain base + all 450 labeled": ("data/shop-train-all.jsonl", "multilingual"),
}
SEEDS = [1, 2, 3]
cases = [json.loads(line) for line in open("shop-eval.jsonl", encoding="utf-8")]


def exact_by_pool(path):
    agent = laya.load(path)
    by = {"all": [], "train": [], "ops": [], "unseen": []}
    for c in cases:
        a = agent.predict(c["state"], c["questions"])["answers"]
        ok = all((a[f]["noul"] >= 0.5 if a[f]["type"] == "noul" else a[f]["choice"]) == c["expected"][f] for f in FIELDS)
        by["all"].append(ok)
        by[c["pool"]].append(ok)
    return {k: sum(v) / len(v) for k, v in by.items()}


print(f"{'approach':<32}{'seed 1':>8}{'seed 2':>8}{'seed 3':>8}{'mean':>8}{'range':>8}{'train':>8}{'ops':>8}{'unseen':>8}")
for name, (data, base) in APPROACHES.items():
    runs = []
    for seed in SEEDS:
        out = f"checkpoints/shop-repeat-{seed}"
        subprocess.run(["laya-train", "--data", data, "--base", base, "--out", out, "--seed", str(seed)],
                       check=True, capture_output=True)
        runs.append(exact_by_pool(out))
        shutil.rmtree(out)
    alls = [r["all"] for r in runs]
    pools = "".join(f"{statistics.mean(r[p] for r in runs):>8.3f}" for p in ("train", "ops", "unseen"))
    cells = "".join(f"{s:>8.3f}" for s in alls)
    print(f"{name:<32}{cells}{statistics.mean(alls):>8.3f}{max(alls) - min(alls):>8.3f}{pools}", flush=True)

=== Evaluation (before vs after fine-tuning on 1260 held-out items from shop-eval.jsonl) ===
Metric                 Before         After          Delta         
-----------------------------------------------------------------
Accuracy               0.4857         0.8563         +0.3706       
Loss                   1.5558         0.5511         -1.0047       
ECE                    0.1940         0.0786         -0.1154       
Brier (multi-class)    0.7368         0.2358         -0.5010       
Brier (top-1)          0.2975         0.1049         -0.1926       
Mean Confidence        0.6769         0.9315         +0.2546       
-----------------------------------------------------------------

Training complete! Model saved to checkpoints/shop-v2

"""지금 쓰는 판(current)과 새 판(candidate)을 같은 검색어 평가 세트로 재고, 기준을 넘을 때만 승격한다.

기준 — 셋 다 통과해야 한다:
  1. 완전 일치(여섯 칸 모두 맞음)가 0.03 이상 오른다
  2. 어느 칸도 0.03 넘게 떨어지지 않는다(평균에 가려진 퇴보를 잡는다)
  3. 처음 학습한 단어(train words)의 완전 일치가 0.03 넘게 떨어지지 않는다(새 것을 배우다 옛것을 잊는지)
결과는 data/shop-registry.jsonl 에 한 줄씩 남긴다.
"""
import json
import sys
from datetime import date

import laya

FIELDS = ["price", "category", "item", "winter", "gender", "color"]
current_path, candidate_path = sys.argv[1], sys.argv[2]
cases = [json.loads(line) for line in open("shop-eval.jsonl", encoding="utf-8")]


def measure(path):
    agent = laya.load(path)
    preds = []
    for c in cases:
        a = agent.predict(c["state"], c["questions"])["answers"]
        preds.append({f: (a[f]["noul"] >= 0.5 if a[f]["type"] == "noul" else a[f]["choice"]) for f in FIELDS})
    exact = [all(p[f] == c["expected"][f] for f in FIELDS) for c, p in zip(cases, preds)]
    train = [e for c, e in zip(cases, exact) if c["pool"] == "train"]
    return {"exact": round(sum(exact) / len(exact), 3), "train words": round(sum(train) / len(train), 3),
            "fields": {f: round(sum(p[f] == c["expected"][f] for c, p in zip(cases, preds)) / len(cases), 3)
                       for f in FIELDS}}


cur, cand = measure(current_path), measure(candidate_path)
checks = {
    "exact +0.03": cand["exact"] - cur["exact"] >= 0.03,
    "no field -0.03": all(cand["fields"][f] - cur["fields"][f] >= -0.03 for f in FIELDS),
    "train words -0.03": cand["train words"] - cur["train words"] >= -0.03,
}
decision = "PROMOTE" if all(checks.values()) else "REJECT"

print(f"{'':<14}{'current':>10}{'candidate':>11}")
print(f"{'exact match':<14}{cur['exact']:>10.3f}{cand['exact']:>11.3f}")
print(f"{'  train words':<14}{cur['train words']:>10.3f}{cand['train words']:>11.3f}")
for f in FIELDS:
    print(f"  {f:<12}{cur['fields'][f]:>10.3f}{cand['fields'][f]:>11.3f}")
for name, ok in checks.items():
    print(f"[{'pass' if ok else 'FAIL'}] {name}")
print(f"decision: {decision} {candidate_path}")

record = {"date": str(date.today()), "current": current_path, "candidate": candidate_path,
          "decision": decision, "checks": checks, "current_metrics": cur, "candidate_metrics": cand}
with open("data/shop-registry.jsonl", "a", encoding="utf-8") as f:
    f.write(json.dumps(record, ensure_ascii=False) + "\n")

                 current  candidate
exact match        0.305      0.510
  train words      0.800      0.929
  price            0.800      0.933
  category         0.671      0.838
  item             0.638      0.833
  winter           0.852      0.867
  gender           0.743      0.952
  color            0.605      0.710
[pass] exact +0.03
[pass] no field -0.03
[pass] train words -0.03
decision: PROMOTE checkpoints/shop-v2

검색어: 50만원이하 패딩 상품중 갈색
  price    under50   확신 0.99
  category top       확신 0.99
  item     padding   확신 0.98
  winter   0.96      확신 0.96
  gender   unknown   확신 1.00
  color    brown     확신 0.98
  filters: {"price_under50": true, "category_top": true, "item_padding": true, "winter": true, "gender": null, "color_brown": true}
검색어: 아빠 겨울 부츠 브라운으로
  price    none      확신 0.99
  category shoes     확신 0.99
  item     boots     확신 0.98
  winter   0.95      확신 0.95
  gender   men       확신 1.00
  color    brown     확신 0.98
  filters: {"price": null, "category_shoes": true, "item_boots": true, "winter": true, "gender_men": true, "color_brown": true}
검색어: 언니 선물로 십만 원 이하 밤색 터틀넥
  price    under30   확신 0.95
  category top       확신 0.98
  item     tshirt    확신 0.71
  winter   0.24      확신 0.76
  gender   men       확신 0.83
  color    navy      확신 0.95
  filters: {"price_under30": true, "category_top": true, "item_tshirt": true, "winter": false, "gender_men": true, "color_navy": true}