#!/usr/bin/env python3
import json, re, os, glob, unicodedata, difflib
from collections import Counter, defaultdict
BASE=os.path.dirname(os.path.abspath(__file__))
ENG=set(w.strip().lower() for w in open('/usr/share/dict/words'))
def fold(s):
    s=str(s).replace('—','-').replace('’',"'")
    s=unicodedata.normalize('NFKD',s)
    return ''.join(c for c in s if not unicodedata.combining(c)).lower()
def foldsp(s): return re.sub(r'\s+',' ',re.sub(r'[^a-z0-9 ]',' ',fold(s))).strip()
GARB=re.compile(r'[a-z][A-Z]|[A-Za-z][0-9]|[0-9][A-Za-z]|[}{|<>~^]|[A-Za-z][?;)(!][A-Za-z]')
def dictratio(s):
    toks=[re.sub(r"[^A-Za-z']",'',w) for w in s.split()]
    toks=[t for t in toks if t]
    if not toks: return 0.0
    return sum(1 for t in toks if fold(t) in ENG)/len(toks)
def tier(g,v):
    if GARB.search(g): return 'baja'
    dg=dictratio(g)
    if dg>=0.75 and len(g.split())<=12 and len(v.split())<=12: return 'alta'
    if dg>=0.5: return 'media'
    return 'baja'
# conocidos: pares aplicados + pendientes manuales
known=set()
for line in open('correcciones_LIMPIAS.md',encoding='utf-8'):
    m=re.search(r'buscar «(.+?)»\s*→\s*reemplazar por «(.+?)»',line)
    if m: known.add((foldsp(m.group(2)),foldsp(m.group(1))))  # (veda,gold)
for f in ('/tmp/unapplied_all.json','/tmp/applyrun/remaining.json','/tmp/approved_canon.json'):
    try:
        for c in json.load(open(f)): known.add((foldsp(c['veda']),foldsp(c['gold'])))
    except Exception: pass
# D1 words por grupo para densidad
d1w={}
for f,books in (('/tmp/fp_small.json',None),('/tmp/fp_cc.json',None),('/tmp/fp_sb.json',None)):
    for r in json.load(open(f))[0]['results']:
        b=r['book']
        n=len(((r.get('translation') or '')+' '+(r.get('purport') or '')).split())
        if b=='sb':
            g='sb'+r['ref'].split('/')[1]
        elif b=='cc':
            g='cc_'+r['ref'].split('/')[1]
        else: g=b
        d1w[g]=d1w.get(g,0)+n
ALIAS={'rvidya':'rv','ejop':'ej'}
rows=[]; cands=[]
seen=set()
for fp in sorted(glob.glob('reports/fresh/*.json')):
    grp=os.path.basename(fp)[:-5]
    d=json.load(open(fp))
    reals=[x for x in d['diffs'] if x['cls']=='real']
    dgrp=ALIAS.get(grp,grp)
    words=d1w.get(dgrp,0)
    t=Counter(); new_alta=0
    for x in reals:
        k=(x['ref'],x['col'],foldsp(x['gold']),foldsp(x['veda']))
        if k in seen: continue
        seen.add(k)
        tr=tier(x['gold'],x['veda'])
        kn=(foldsp(x['veda']),foldsp(x['gold'])) in known
        t[tr]+=1
        if tr=='alta':
            if not kn: new_alta+=1
            cands.append({'group':grp,'ref':x['ref'],'col':x['col'],'gold':x['gold'],'veda':x['veda'],
                          'cb':x['cb'],'ca':x['ca'],'known':kn})
    dens=round(1000*sum(t.values())/max(words,1),1)
    rows.append((dens,grp,sum(t.values()),t['alta'],new_alta,t['media'],t['baja'],words))
rows.sort(reverse=True)
print(f"{'grupo':12} {'real':>5} {'alta':>5} {'altaNUEVA':>9} {'media':>6} {'baja':>5} {'dens/1k':>8}")
for dens,grp,tot,a,na,m,b,w in rows:
    print(f"{grp:12} {tot:5} {a:5} {na:9} {m:6} {b:5} {dens:8}")
print('\nTOTAL real(dedup):',sum(r[2] for r in rows),'| alta:',sum(r[3] for r in rows),'| alta NUEVA:',sum(r[4] for r in rows))
json.dump(cands, open('reports/fresh/CANDIDATES_alta.json','w'), ensure_ascii=False)
print('guardado reports/fresh/CANDIDATES_alta.json')