#!/usr/bin/env python3
"""Clean the reversion list: drop corrections whose GOLD (scan) is garbled OCR
(mangled Sanskrit etc.), keep only reliable ones. Re-accent the kept GOLD.
Conservative: when in doubt, drop (better to miss a fix than corrupt the master).
"""
import re, json, unicodedata, collections, os

BASE = os.path.dirname(os.path.abspath(__file__))
amap = json.load(open(os.path.join(BASE, "accent_map.json")))
WORDS = set(w.strip().lower() for w in open("/usr/share/dict/words"))
IAST = "āīūṛṝḷḹṅñṭḍṇśṣṁḥ"

def fold(s):
    s = unicodedata.normalize('NFKD', s)
    return ''.join(c for c in s if not unicodedata.combining(c)).lower()
def reaccent(s):
    return re.sub(r"[A-Za-z']+", lambda m: amap.get(fold(m.group(0)), m.group(0)), s)

def is_garbage(gold):
    g = gold.strip()
    # OCR-garbage markers (strong signals)
    # 1) internal uppercase: a letter (lower) immediately followed by an uppercase
    if re.search(r'[a-zāīūṛṝḷḹṅñṭḍṇśṣṁḥ][A-ZĀĪŪṚṜḶḸṄÑṬḌṆŚṢṀḤ]', g):
        return True
    # 2) '?' or stray symbols sitting inside/next to letters (OCR confusion)
    if re.search(r'[A-Za-zāīūṛṇśṣṁḥ][?;~^][A-Za-zāīū]|[A-Za-z][?]', g):
        return True
    # 3) odd char clusters
    if re.search(r'[fJ]\w*[A-Z]|[a-z]{2}[A-Z][a-z]', g):
        return True
    # 4) low dictionary ratio on alpha tokens (mangled words, not clean English/IAST)
    tk = re.findall(r"[A-Za-zāīūṛṝḷḹṅñṭḍṇśṣṁḥ]{2,}", g)
    if tk:
        dq = sum(1 for t in tk if fold(t) in WORDS) / len(tk)
        # allow short proper-noun-ish phrases; drop multi-word low-dict phrases
        if dq < 0.40 and len(tk) >= 2:
            return True
    return False

def main():
    revs = collections.defaultdict(list); cur = None
    for line in open(os.path.join(BASE, "reversiones_master.md")):
        h = re.match(r'^## (\S+)', line)
        if h: cur = h.group(1)
        d = re.match(r'^- en vedabase «(.*?)»  ->  restaurar «(.*?)»', line)
        if d and cur: revs[cur].append((d.group(1), d.group(2)))

    kept = collections.defaultdict(list); dropped = []
    nk = nd = 0
    for k, items in revs.items():
        for veda, gold in items:
            if is_garbage(gold):
                dropped.append((k, veda, gold)); nd += 1
            else:
                kept[k].append((veda, reaccent(gold))); nk += 1

    out = ["# Vedabase maestro — correcciones LIMPIAS para LaTeX",
           f"# {nk} correcciones fiables ({nd} descartadas por OCR-basura en el escaneo)", "",
           "Buscar «VEDA» y reemplazar por «GOLD» (re-acentuado) en el .tex.", ""]
    for k in sorted(kept):
        if not kept[k]: continue
        out.append(f"\n## {k}")
        for veda, gold in kept[k]:
            out.append(f"- buscar «{veda}»  →  reemplazar por «{gold}»")
    open(os.path.join(BASE, "correcciones_LIMPIAS.md"), "w").write("\n".join(out))

    dl = ["# Descartadas (GOLD = OCR basura del escaneo, NO aplicar)", ""]
    for k, veda, gold in dropped:
        dl.append(f"- [{k}] maestro «{veda[:50]}» → escaneo «{gold[:50]}»")
    open(os.path.join(BASE, "descartadas.md"), "w").write("\n".join(dl))

    print(f"FIABLES (conservadas): {nk}")
    print(f"DESCARTADAS (basura) : {nd}")
    print(f"-> correcciones_LIMPIAS.md  /  descartadas.md")
    print("\nMuestra de descartadas:")
    for k, v, g in dropped[:8]:
        print(f"  [{k}] «{v[:35]}» → «{g[:35]}»")

if __name__ == "__main__":
    main()