#!/usr/bin/env python3
"""FRESH PASS 2026-06-12: direct scan↔D1(en, post-corrections) comparison.
GOLD = scan (original edition, truth). VEDA = current D1 text.
Per small book / per sb-canto / per cc-lila. autojunk=True (Canto-10 lesson).
Output: reports/fresh/<group>.json + summary line per group.
"""
import fitz, re, json, sys, os, unicodedata, difflib
from collections import Counter
BASE=os.path.dirname(os.path.abspath(__file__))
ENG=set(w.strip().lower() for w in open('/usr/share/dict/words'))
def fold(s):
    s=str(s).replace('\u2014','-').replace('\u2019',"'").replace('\u201c','"').replace('\u201d','"')
    s=unicodedata.normalize('NFKD',s)
    return ''.join(c for c in s if not unicodedata.combining(c)).lower()
def book_cfg(key):
    src=open(os.path.join(BASE,'compare.py')).read()
    m=re.search(r'"'+key+r'":\s*\{(.+?)\n    \}',src,re.S) or re.search(r'"'+key+r'":\s*\{(.+?)\}',src,re.S)
    b=m.group(1)
    lm=re.search(r'"original":\s*\[(.+?)\]',b,re.S)
    if lm: files=re.findall(r'"([^"]+)"',lm.group(1))
    else: files=[re.search(r'"original":\s*"([^"]+)"',b).group(1)]
    def gi(k,d):
        mm=re.search(r'"'+k+r'":\s*(\d+)',b)
        return int(mm.group(1)) if mm else d
    return files, gi('orig_start_page',0), gi('orig_end_page',10**6)
def resolve_orig(fn):
    for p in (os.path.join(BASE,'improved',fn),os.path.join(BASE,'originals',fn)):
        if os.path.exists(p): return p
    return None
def scan_words(key):
    files,p0,p1=book_cfg(key)
    allw=[]; names=[]
    for fi,fn in enumerate(files):
        path=resolve_orig(fn)
        if not path: raise FileNotFoundError(key+': '+fn)
        w,nm=_scan_one(path, p0 if fi==0 else 0, p1 if fi==len(files)-1 else 10**6)
        allw+=w; names.append(nm)
    return allw, '+'.join(names)
def _scan_one(path,p0,p1):
    d=fitz.open(path)
    pages=[]
    from collections import Counter as _C
    freq=_C()
    for i in range(p0,min(p1,len(d))):
        t=d[i].get_text(); t=re.sub(r'-\n([a-z])',r'\1',t)
        lines=t.split('\n'); pages.append(lines)
        for ln in lines[:3]+lines[-3:]:
            k=re.sub(r'\d+','',fold(ln)).strip()
            if 0<len(k.split())<=8: freq[k]+=1
    npages=max(len(pages),1)
    heads={k for k,c in freq.items() if c>=max(3,int(0.10*npages))}
    txt=[]
    for lines in pages:
        keep=[]
        for ln in lines:
            k=re.sub(r'\d+','',fold(ln)).strip()
            if re.fullmatch(r'\s*\d+\s*', ln or ' '): continue
            if k in heads and len(k.split())<=8: continue
            keep.append(ln)
        txt.append('\n'.join(keep))
    return ' '.join(txt).split(), os.path.basename(path)
def natkey(ref): return [int(x) if x.isdigit() else x for x in re.split(r'(\d+)',ref)]
def clean_md(w): return w.replace('*','').replace('**','')
COLS=('translation','purport')
def d1_words(rows):
    out=[]
    for r in sorted(rows,key=lambda r:natkey(r['ref'])):
        for col in COLS:
            t=r.get(col) or ''
            for w in t.split():
                wc=clean_md(w)
                if wc.strip(): out.append((wc,r['ref'],col))
    return out
def is_garbage_gold(words):
    if not words: return False
    toks=[w for w in words if re.search(r'[A-Za-z]',w)]
    if not toks: return True
    one=sum(1 for w in toks if len(re.sub(r'[^A-Za-z]','',w))<=1)
    if one/len(toks)>=0.5: return True
    dic=sum(1 for w in toks if fold(re.sub(r'[^A-Za-z]','',w)) in ENG)
    if len(toks)>=3 and dic/len(toks)<0.34: return True
    if any(re.search(r'[a-z][A-Z]|[A-Za-z][?;~^|{}]',w) for w in words): return True
    return False
LABELS=re.compile(r'^(TRANSLATION|PURPORT|SYNONYMS|TEXT[S]?|MANTRA|CHAPTER|INVOCATION|[0-9IVXLC.\\s]+)$')
def classify(g,v):
    g,v=g.strip(),v.strip()
    if not re.search(r'[A-Za-z]',g) and not re.search(r'[A-Za-z]',v): return 'trivial'
    gl=' '.join(w for w in g.split() if not LABELS.match(w))
    if not gl and not v: return 'trivial'
    if is_garbage_gold(g.split()): return 'noise'
    a=re.sub(r'[^a-z]','',fold(g)); b=re.sub(r'[^a-z]','',fold(v))
    if a==b: return 'trivial'
    if a and b and difflib.SequenceMatcher(None,a,b).ratio()>=0.85: return 'trivial'
    if not v: return 'scan_only'   # bloque solo en escaneo (sánscrito/sinónimos/etiquetas/front) -> no accionable
    if not g: return 'd1_only' if len(v.split())<=12 else 'big_insert'
    if len(g.split())>30 or len(v.split())>30: return 'big_replace'
    gt=[re.sub(r"[^a-z]",'',fold(w)) for w in g.split()]
    vt=[re.sub(r"[^a-z]",'',fold(w)) for w in v.split()]
    gt=[w for w in gt if w]; vt=[w for w in vt if w]
    if gt and vt and all(w not in ENG for w in gt) and all(w not in ENG for w in vt): return 'translit'
    return 'real'
def run_group(name, scan_keys, rows):
    sw=[]; files=[]
    for k in scan_keys:
        w,f=scan_words(k); sw+=w; files.append(f)
    dw=d1_words(rows)
    sf=[fold(w) for w in sw]; df=[fold(w[0]) for w in dw]
    sm=difflib.SequenceMatcher(None,sf,df,autojunk=True)
    diffs=[]
    for tag,i1,i2,j1,j2 in sm.get_opcodes():
        if tag=='equal': continue
        gold=' '.join(sw[i1:i2]); veda=' '.join(dw[k][0] for k in range(j1,j2))
        loc=dw[j1] if j1<len(dw) else (dw[-1] if dw else ('',None,None))
        cb=' '.join(dw[k][0] for k in range(max(0,j1-8),j1))
        ca=' '.join(dw[k][0] for k in range(j2,min(len(dw),j2+8)))
        c=classify(gold,veda)
        if c in ('trivial',): continue
        diffs.append({'cls':c,'ref':loc[1],'col':loc[2],'gold':gold,'veda':veda,'cb':cb,'ca':ca})
    cnt=Counter(d['cls'] for d in diffs)
    json.dump({'group':name,'scans':files,'d1_rows':len(rows),'counts':dict(cnt),'diffs':diffs},
              open(os.path.join(BASE,'reports/fresh',name+'.json'),'w'),ensure_ascii=False)
    print(f"{name:12} d1rows={len(rows):5} | real={cnt.get('real',0):4} d1only={cnt.get('d1_only',0):4} bigrep={cnt.get('big_replace',0):3} bigins={cnt.get('big_insert',0):3} scanonly={cnt.get('scan_only',0):4} noise={cnt.get('noise',0):4} translit={cnt.get('translit',0):4} | {','.join(files)[:70]}", flush=True)
def main():
    small=json.load(open('/tmp/fp_small.json'))[0]['results']
    KEY2D1={'bg':'bg','iso':'iso','noi':'noi','poy':'poy','pop':'pop','lob':'lob','ekc':'ekc',
            'rvidya':'rv','bbd':'bbd','owk':'owk','pqpa':'pqpa','tqk':'tqk','nod':'nod',
            'ssr':'ssr','ejop':'ej','krp':'krp','kb':'kb','tlc':'tlc'}
    only=sys.argv[1:] or None
    for key,b in KEY2D1.items():
        if only and key not in only: continue
        rows=[r for r in small if r['book']==b]
        try: run_group(key,[key],rows)
        except Exception as e: print(f"{key:12} ERROR {e}", flush=True)
    if only and not any(k.startswith(('cc','sb')) for k in only): return
    cc=json.load(open('/tmp/fp_cc.json'))[0]['results']
    src=open(os.path.join(BASE,'compare.py')).read()
    cckeys=re.findall(r'"(cc_\w+)":\s*\{',src)
    for lila,pref in (('adi','cc/adi/'),('madhya','cc/madhya/'),('antya','cc/antya/')):
        ks=sorted([k for k in cckeys if k.split('_')[1].startswith(lila[:3])],key=natkey)
        if only and ('cc_'+lila) not in only and 'cc' not in only: continue
        rows=[r for r in cc if r['ref'].startswith(pref)]
        try: run_group('cc_'+lila,ks,rows)
        except Exception as e: print(f"cc_{lila:9} ERROR {e}", flush=True)
    del cc
    sb=json.load(open('/tmp/fp_sb.json'))[0]['results']
    sbkeys=re.findall(r'"(sb\d+)":\s*\{',src)
    for canto in range(1,11):
        ks=sorted([k for k in sbkeys if (k[2:-1]==str(canto))],key=natkey)
        if not ks: continue
        if only and ('sb'+str(canto)) not in only and 'sb' not in only: continue
        rows=[r for r in sb if r['ref'].startswith(f'sb/{canto}/')]
        try: run_group('sb'+str(canto),ks,rows)
        except Exception as e: print(f"sb{canto:<10} ERROR {e}", flush=True)
if __name__=='__main__': main()