#!/usr/bin/env python3
"""Phase 2 triage: separate real editorial changes from OCR noise across the
66 reliable books, and emit a consolidated, prioritized review report.

GOLD = original scan text (truth). VEDA = vedabase text (audited).
A "real" diff = both sides are clean language but differ in wording/meaning.
Noise = the GOLD side is garbled OCR, or the two are near-identical OCR variants.
"""
import json, glob, os, re, difflib

BASE = os.path.dirname(os.path.abspath(__file__))
EXCLUDE = {"tlc", "ejop"}  # edition-mismatch outliers

GARBAGE = re.compile(r'[<>=~|{}\\^_]|[A-Za-z][0-9]|[0-9][A-Za-z]')

def clean(s):
    return ' '.join(str(s).split())

def is_garbled(s):
    """True if s looks like OCR garbage (symbols, letter/digit mash, no real words)."""
    if GARBAGE.search(s):
        return True
    letters = re.findall(r'[A-Za-z]{2,}', s)
    if not letters and re.search(r'[A-Za-z]', s):
        return True  # only stray single letters
    return False

def char_ratio(a, b):
    a2 = re.sub(r'[^a-z]', '', a.lower())
    b2 = re.sub(r'[^a-z]', '', b.lower())
    if not a2 and not b2:
        return 1.0
    return difflib.SequenceMatcher(None, a2, b2).ratio()

def classify(o, v):
    o, v = clean(o), clean(v)
    # pure punctuation / empty
    if not re.search(r'[A-Za-z]', o) and not re.search(r'[A-Za-z]', v):
        return "trivial"
    # GOLD garbled → OCR noise on the scan side
    if is_garbled(o):
        return "noise"
    # near-identical (OCR variant or capitalization/punct only)
    if char_ratio(o, v) >= 0.82:
        return "trivial"
    return "real"

def main():
    rows = []
    real_by_book = {}
    for f in sorted(glob.glob(os.path.join(BASE, "reports", "*_diffs.json"))):
        key = os.path.basename(f).replace("_diffs.json", "")
        if key in EXCLUDE:
            continue
        d = json.load(open(f))
        reals = []
        n_noise = n_triv = 0
        for x in d.get("differences", []):
            c = classify(x.get("original", ""), x.get("vedabase", ""))
            if c == "real":
                reals.append(x)
            elif c == "noise":
                n_noise += 1
            else:
                n_triv += 1
        rows.append((len(reals), n_noise, n_triv, key, d["book"]))
        if reals:
            real_by_book[key] = (d["book"], reals)

    rows.sort(reverse=True)
    total_real = sum(r[0] for r in rows)
    total_noise = sum(r[1] for r in rows)
    total_triv = sum(r[2] for r in rows)

    # ---- summary table ----
    lines = ["# Fase 2 — Revisión consolidada (66 libros fiables)\n",
             f"GOLD = escaneo original (verdad). VEDA = vedabase (auditado).\n",
             f"**{total_real} cambios reales** · {total_noise} ruido-OCR descartado · {total_triv} triviales descartados\n",
             "\n## Resumen por libro (solo cambios reales)\n",
             "| Libro | Reales | Ruido | Triv |", "|---|---|---|---|"]
    for nreal, nnoise, ntriv, key, name in rows:
        if nreal:
            lines.append(f"| {key} — {name} | **{nreal}** | {nnoise} | {ntriv} |")

    # ---- detailed real changes, by book, top-down ----
    lines.append("\n---\n\n## Cambios reales por libro\n")
    for nreal, nnoise, ntriv, key, name in rows:
        if not nreal:
            continue
        bookname, reals = real_by_book[key]
        lines.append(f"\n### {key} — {bookname}  ({nreal} cambios)\n")
        for x in reals:
            o = clean(x.get("original", "")); v = clean(x.get("vedabase", ""))
            cb = clean(x.get("context_before", ""))[-40:]
            ca = clean(x.get("context_after", ""))[:40]
            lines.append(f"- …{cb} **「GOLD: {o}」 → 「VEDA: {v}」** {ca}…")

    out = os.path.join(BASE, "reports", "PHASE2_review.md")
    open(out, "w").write("\n".join(lines))
    print(f"REAL={total_real}  noise={total_noise}  trivial={total_triv}")
    print(f"report -> {out}")
    print("\nTop libros por cambios reales:")
    for nreal, nnoise, ntriv, key, name in rows[:15]:
        if nreal:
            print(f"  {key:9s} real={nreal:4d}  noise={nnoise:4d}  triv={ntriv:4d}")

if __name__ == "__main__":
    main()