#!/usr/bin/env python3
"""Scan-vs-D1 localizer: diff the original-edition SCAN (gold) against the actual
D1 verse text, so every difference lands NATIVELY on a (ref, column) — no
whole-book ambiguity, and edition-mismatch misalignment can't place itself.

GOLD = scan (truth). VEDA = D1 (to be corrected toward GOLD).
Emits localized diffs with a quality filter; does NOT write to D1 (preview only).
"""
import fitz, re, json, sys, unicodedata, difflib, os

BASE = os.path.dirname(os.path.abspath(__file__))
ENG = set(w.strip().lower() for w in open('/usr/share/dict/words'))

def fold(s):
    s = unicodedata.normalize('NFKD', str(s))
    return ''.join(c for c in s if not unicodedata.combining(c)).lower()

def book_cfg(key):
    src = open(os.path.join(BASE, 'compare.py')).read()
    m = re.search(r'"'+key+r'":\s*\{(.+?)\}', src, re.S)
    b = m.group(1)
    g = lambda k: re.search(k+r'":\s*"?([^,"\n]+)', b).group(1).strip()
    return g('original'), int(g('orig_start_page')), int(g('orig_end_page'))

def resolve_orig(fn):
    for p in (os.path.join(BASE,'improved',fn), os.path.join(BASE,'originals',fn)):
        if os.path.exists(p): return p
    return None

def scan_words(key):
    fn, p0, p1 = book_cfg(key)
    path = resolve_orig(fn)
    d = fitz.open(path)
    txt = []
    for i in range(p0, min(p1, len(d))):
        t = d[i].get_text()
        t = re.sub(r'-\n([a-z])', r'\1', t)         # dehyphenate
        txt.append(t)
    words = ' '.join(txt).split()
    return words, os.path.basename(path)

def natkey(ref):
    return [int(x) if x.isdigit() else x for x in re.split(r'(\d+)', ref)]

def d1_words(key, d1rows):
    rows = sorted([r for r in d1rows if r['book']==key], key=lambda r: natkey(r['ref']))
    words=[]   # (word, ref, col)
    for r in rows:
        for col in ('verse_text','translation','purport'):
            t = r.get(col) or ''
            for w in t.split():
                words.append((w, r['ref'], col))
    return words

def is_garbage_gold(words):
    """Reject OCR-garbage GOLD: many 1-char tokens, low dict ratio, symbols."""
    if not words: return False
    toks = [w for w in words if re.search(r'[A-Za-z]', w)]
    if not toks: return True
    one_char = sum(1 for w in toks if len(re.sub(r'[^A-Za-z]','',w))<=1)
    if one_char/len(toks) >= 0.5: return True            # ": e o a o he"
    dictish = sum(1 for w in toks if fold(re.sub(r'[^A-Za-z]','',w)) in ENG)
    if len(toks)>=3 and dictish/len(toks) < 0.34: return True
    if any(re.search(r'[a-z][A-Z]|[A-Za-z][?;~^|{}]', w) for w in words): return True
    return False

def localize(key, d1rows, ctx=6):
    sw, scanfile = scan_words(key)
    dw = d1_words(key, d1rows)
    sf = [fold(w) for w in sw]
    df = [fold(w) for w in dw]
    sm = difflib.SequenceMatcher(None, sf, df, autojunk=False)
    diffs=[]
    for tag,i1,i2,j1,j2 in sm.get_opcodes():
        if tag=='equal': continue
        gold = sw[i1:i2]                       # scan side
        veda = [dw[k][0] for k in range(j1,j2)]# D1 side
        # ref/col from the D1 side (or neighbor for pure inserts/deletes)
        loc = dw[j1] if j1<len(dw) else (dw[-1] if dw else (None,None,None))
        ref, col = loc[1], loc[2]
        diffs.append({'tag':tag,'ref':ref,'col':col,
                      'gold':' '.join(gold),'veda':' '.join(veda)})
    return diffs, scanfile, len(dw)

def classify(d):
    g, v = d['gold'].strip(), d['veda'].strip()
    if not re.search(r'[A-Za-z]',g) and not re.search(r'[A-Za-z]',v): return 'trivial'
    if is_garbage_gold(g.split()): return 'noise'
    a=re.sub(r'[^a-z]','',fold(g)); b=re.sub(r'[^a-z]','',fold(v))
    if a and b and difflib.SequenceMatcher(None,a,b).ratio()>=0.85: return 'trivial'
    if not g and len(v.split())>12: return 'big_insert'   # vedabase added a lot -> suspect/misalign
    if not v and len(g.split())>12: return 'big_delete'
    return 'real'

if __name__=='__main__':
    key=sys.argv[1]
    d1=json.load(open('/tmp/d1_en.json'))[0]['results']
    diffs, scanfile, nd = localize(key, d1)
    from collections import Counter
    cnt=Counter(classify(d) for d in diffs)
    print(f'{key}: scan={scanfile} D1words={nd} rawdiffs={len(diffs)} | {dict(cnt)}')
    print('--- REAL (localized) ---')
    n=0
    for d in diffs:
        if classify(d)!='real': continue
        n+=1
        if n>25: break
        print(f"  [{d['ref']}/{d['col']}] «{d['veda'][:42]}» → «{d['gold'][:42]}»")