#!/usr/bin/env python3 """ Vedabase Text Comparison Pipeline v2 Compares original scanned editions with Vedabase PDF text. Splits by chapter using page-level detection, then compares verse by verse. Filters out OCR/diacritics noise, reports only real editorial changes. """ import fitz # PyMuPDF import re import difflib import json import unicodedata from pathlib import Path from dataclasses import dataclass, asdict # ============================================================ # CONFIGURATION # ============================================================ BASE_DIR = Path(__file__).parent ORIGINALS_DIR = BASE_DIR / "originals" IMPROVED_DIR = BASE_DIR / "improved" def resolve_orig(filename: str) -> str: """Path to a book's original scan, preferring the re-OCR'd version in improved/ when it exists (better text layer for scanned books).""" improved = IMPROVED_DIR / filename if improved.exists(): return str(improved) return str(ORIGINALS_DIR / filename) VEDABASE_PDF = Path("/Users/jaganat/MEGA/vedabase/veda_pdf_scan/vedabase1.pdf") OUTPUT_DIR = BASE_DIR / "reports" BOOKS = { "bg": { "name": "Bhagavad-gita As It Is", "original": "1972_Bhagavad_gita-As_It_Is-Original_authorized_Macmillan_edition.pdf", "vedabase_start": 86, "vedabase_end": 355, "chapters": 18, "type": "verse_by_chapter", }, "iso": { "name": "Sri Isopanisad", "original": "Sri-Isopanisad-scans-of-original-1969-edition.pdf", "vedabase_start": 10778, "vedabase_end": 10802, "mantras": 18, "orig_start_page": 17, "orig_end_page": 130, "type": "mantra", }, "noi": { "name": "The Nectar of Instruction", "original": "The_Nectar_of_Instruction-Original_1976_SCAN.pdf", "vedabase_start": 14118, "vedabase_end": 14138, "orig_start_page": 18, "orig_end_page": 140, "type": "text_flat", # TEXT ONE/TWO/... with no chapters }, "poy": { "name": "The Perfection of Yoga", "original": "1972_Perfection_of_Yoga_2008_Original.pdf", "vedabase_start": 14185, "vedabase_end": 14196, "orig_start_page": 5, "orig_end_page": 65, "type": "chapter_prose", # Roman numeral chapters, prose only }, "pop": { "name": "The Path of Perfection", "original": "The-Path-of-Perfection-SCAN.pdf", "vedabase_start": 14139, # TOC: pop p14139 .. poy p14185 "vedabase_end": 14184, "orig_start_page": 14, # scan: Introduction p14, Chapter One p16, ads from p116 "orig_end_page": 116, "type": "chapter_prose", }, "lob": { "name": "Light of the Bhagavata", "original": "Light_of_the_Bhagavata-1984_first_edition_SCAN.pdf", "vedabase_start": 8519, # TOC: lob p8519 .. On the Way to Krsna p8536 "vedabase_end": 8535, "orig_start_page": 9, # scan: Preface p9, Text 1 p12, glossary/index from p158 "orig_end_page": 158, "type": "chapter_prose", # 48 numbered texts ("LoB N"), prose + purport }, "ekc": { "name": "Elevation to Krsna Consciousness", "original": "1973_Elevation_to_Krsna_Consciousness.pdf", "vedabase_start": 373, "vedabase_end": 393, "orig_start_page": 3, "orig_end_page": 100, "type": "chapter_prose", }, "rvidya": { "name": "Raja-Vidya The King of Knowledge", "original": "1973_Raja-Vidya_The_King_of_Knowledge.pdf", "vedabase_start": 8580, "vedabase_end": 8603, "orig_start_page": 3, "orig_end_page": 100, "type": "chapter_prose", }, "bbd": { "name": "Beyond Birth and Death", "original": "Beyond_Birth_and_Death-Original_1974_edition_scan.pdf", "vedabase_start": 61, "vedabase_end": 73, "orig_start_page": 3, "orig_end_page": 80, "type": "chapter_prose", }, "owk": { "name": "On the Way to Krsna", "original": "On_the_Way_to_Krsna-1973_book_SCAN.pdf", "vedabase_start": 8536, "vedabase_end": 8552, "orig_start_page": 5, "orig_end_page": 80, "type": "chapter_prose", }, "pqpa": { "name": "Perfect Questions Perfect Answers", "original": "Perfect_Questions_Perfect_Answers-Original_1977_Edition-SCAN.pdf", "vedabase_start": 8553, "vedabase_end": 8579, "orig_start_page": 20, "orig_end_page": 140, "type": "chapter_prose", }, "kcty": { "name": "Krsna Consciousness The Topmost Yoga System", "original": "KRSNA Consciousness - The Topmost Yoga System - Original 1970 edition Scan.pdf", "vedabase_start": 394, "vedabase_end": 413, "orig_start_page": 8, "orig_end_page": 58, # 58+ is glossary/index/book ads "type": "chapter_prose", }, "ttp": { "name": "Transcendental Teachings of Prahlada Maharaja", "original": "Transcendental_Teachings_of_Prahlad_Maharaj-Original_book-SCAN.pdf", "vedabase_start": 14299, "vedabase_end": 14309, "orig_start_page": 3, "orig_end_page": 32, "type": "chapter_prose", }, "tlc": { "name": "Teachings of Lord Caitanya", "original": "Teachings_of_Lord_Chaitanya-1968_first_edition-SCAN.pdf", "vedabase_start": 13759, "vedabase_end": 13857, "orig_start_page": 20, "orig_end_page": 350, "type": "chapter_prose", }, "tlk": { "name": "Teachings of Lord Kapila", "original": "Teachings_of_Lord_Kapila-SCAN.pdf", "vedabase_start": 13858, "vedabase_end": 13937, "orig_start_page": 5, "orig_end_page": 333, "type": "chapter_prose", }, "tqk": { "name": "Teachings of Queen Kunti", "original": "Teachings_of_Queen_Kunti-SCAN.pdf", "vedabase_start": 13938, "vedabase_end": 13999, "orig_start_page": 5, "orig_end_page": 274, "type": "chapter_prose", }, "nod": { "name": "The Nectar of Devotion", "original": "The_Nectar_of_Devotion-1970_ISKCON_Press_edition-Hardcover-SCAN.pdf", "vedabase_start": 14000, "vedabase_end": 14117, "orig_start_page": 5, "orig_end_page": 458, "type": "chapter_prose", }, "ssr": { "name": "The Science of Self Realization", "original": "Science-of-Self-Realization-1977.pdf", "vedabase_start": 14197, "vedabase_end": 14298, "orig_start_page": 5, "orig_end_page": 410, "type": "chapter_prose", }, "ejop": { "name": "Easy Journey to Other Planets", "original": "Easy-Journey-to-Other-Planets-1972-krishnaorg-SCAN.pdf", "vedabase_start": 356, "vedabase_end": 372, "orig_start_page": 5, "orig_end_page": 46, "type": "chapter_prose", }, "krp": { "name": "Krsna the Reservoir of Pleasure", "original": "KRSNA-the-Reservoir-of-Pleasure-1970.pdf", "vedabase_start": 414, "vedabase_end": 416, "orig_start_page": 8, "orig_end_page": 50, "type": "chapter_prose", }, "kb": { "name": "KRSNA The Supreme Personality of Godhead", "original": [ "KRSNA_Book_Vol.1_1970_ISKCON_Press_edition_SCAN.pdf", "KRSNA_Book_Vol.2_1970_ISKCON_Press_edition_SCAN.pdf", ], "vedabase_start": 419, "vedabase_end": 665, "orig_ranges": [(30, 490), (20, 392)], "type": "multi_volume_prose", }, "cc_adi1": { "name": "Sri Caitanya-caritamrta Adi-lila Vol.1", "original": "adi1.pdf", "vedabase_start": 8605, "vedabase_end": 8820, "orig_start_page": 23, "orig_end_page": 603, "type": "chapter_prose", }, "cc_adi2": { "name": "Sri Caitanya-caritamrta Adi-lila Vol.2", "original": "adi2.pdf", "vedabase_start": 8821, "vedabase_end": 8971, "orig_start_page": 24, "orig_end_page": 413, "type": "chapter_prose", }, "cc_adi3": { "name": "Sri Caitanya-caritamrta Adi-lila Vol.3", "original": "adi3.pdf", "vedabase_start": 8972, "vedabase_end": 9126, "orig_start_page": 48, "orig_end_page": 452, "type": "chapter_prose", }, "sb31": { "name": "Srimad-Bhagavatam Third Canto Part 1", "original": "SB3.1.pdf", "vedabase_start": 11319, "vedabase_end": 11423, "orig_start_page": 20, "orig_end_page": 346, "type": "chapter_prose", }, # --- Srimad-Bhagavatam (all volumes) --- "sb11": { "name": "Srimad-Bhagavatam First Canto Part 1", "original": "SB1.1.pdf", "vedabase_start": 10807, "vedabase_end": 10923, "orig_start_page": 23, "orig_end_page": 407, "type": "chapter_prose", }, "sb12": { "name": "Srimad-Bhagavatam First Canto Part 2", "original": "SB1.2.pdf", "vedabase_start": 10923, "vedabase_end": 11036, "orig_start_page": 24, "orig_end_page": 308, "type": "chapter_prose", }, "sb13": { "name": "Srimad-Bhagavatam First Canto Part 3", "original": "SB1.3.pdf", "vedabase_start": 11036, "vedabase_end": 11127, "orig_start_page": 26, "orig_end_page": 427, "type": "chapter_prose", }, "sb21": { "name": "Srimad-Bhagavatam Second Canto Part 1", "original": "SB2.1.pdf", "vedabase_start": 11127, "vedabase_end": 11211, "orig_start_page": 35, "orig_end_page": 368, "type": "chapter_prose", }, "sb22": { "name": "Srimad-Bhagavatam Second Canto Part 2", "original": "SB2.2.pdf", "vedabase_start": 11211, "vedabase_end": 11319, "orig_start_page": 22, "orig_end_page": 295, "type": "chapter_prose", }, "sb32": { "name": "Srimad-Bhagavatam Third Canto Part 2", "original": "SB3.2.pdf", "vedabase_start": 11424, "vedabase_end": 11533, "orig_start_page": 19, "orig_end_page": 360, "type": "chapter_prose", }, "sb33": { "name": "Srimad-Bhagavatam Third Canto Part 3", "original": "SB3.3.pdf", "vedabase_start": 11533, "vedabase_end": 11633, "orig_start_page": 19, "orig_end_page": 339, "type": "chapter_prose", }, "sb34": { "name": "Srimad-Bhagavatam Third Canto Part 4", "original": "SB3.4.pdf", "vedabase_start": 11633, "vedabase_end": 11776, "orig_start_page": 19, "orig_end_page": 471, "type": "chapter_prose", }, "sb41": { "name": "Srimad-Bhagavatam Fourth Canto Part 1", "original": "sb4.1.pdf", "vedabase_start": 11776, "vedabase_end": 11913, "orig_start_page": 19, "orig_end_page": 372, "type": "chapter_prose", }, "sb42": { "name": "Srimad-Bhagavatam Fourth Canto Part 2", "original": "SB4.2.pdf", "vedabase_start": 11913, "vedabase_end": 12007, "orig_start_page": 18, "orig_end_page": 416, "type": "chapter_prose", }, "sb43": { "name": "Srimad-Bhagavatam Fourth Canto Part 3", "original": "SB4.3.pdf", "vedabase_start": 12007, "vedabase_end": 12143, "orig_start_page": 17, "orig_end_page": 384, "type": "chapter_prose", }, "sb44": { "name": "Srimad-Bhagavatam Fourth Canto Part 4", "original": "SB4.4.pdf", "vedabase_start": 12143, "vedabase_end": 12293, "orig_start_page": 17, "orig_end_page": 475, "type": "chapter_prose", }, "sb51": { "name": "Srimad-Bhagavatam Fifth Canto Part 1", "original": "SB5.1.pdf", "vedabase_start": 12293, "vedabase_end": 12438, "orig_start_page": 9, "orig_end_page": 507, "type": "chapter_prose", }, "sb52": { "name": "Srimad-Bhagavatam Fifth Canto Part 2", "original": "SB5.2.pdf", "vedabase_start": 12438, "vedabase_end": 12547, "orig_start_page": 9, "orig_end_page": 521, "type": "chapter_prose", }, "sb61": { "name": "Srimad-Bhagavatam Sixth Canto Part 1", "original": "SB6.1.pdf", "vedabase_start": 12547, "vedabase_end": 12650, "orig_start_page": 8, "orig_end_page": 353, "type": "chapter_prose", }, "sb62": { "name": "Srimad-Bhagavatam Sixth Canto Part 2", "original": "SB6.2.pdf", "vedabase_start": 12650, "vedabase_end": 12718, "orig_start_page": 7, "orig_end_page": 296, "type": "chapter_prose", }, "sb63": { "name": "Srimad-Bhagavatam Sixth Canto Part 3", "original": "SB6.3.pdf", "vedabase_start": 12718, "vedabase_end": 12804, "orig_start_page": 8, "orig_end_page": 331, "type": "chapter_prose", }, "sb71": { "name": "Srimad-Bhagavatam Seventh Canto Part 1", "original": "SB7.1.pdf", "vedabase_start": 12804, "vedabase_end": 12890, "orig_start_page": 7, "orig_end_page": 325, "type": "chapter_prose", }, "sb72": { "name": "Srimad-Bhagavatam Seventh Canto Part 2", "original": "SB7.2.pdf", "vedabase_start": 12890, "vedabase_end": 12993, "orig_start_page": 7, "orig_end_page": 381, "type": "chapter_prose", }, "sb73": { "name": "Srimad-Bhagavatam Seventh Canto Part 3", "original": "SB7.3.pdf", "vedabase_start": 12993, "vedabase_end": 13074, "orig_start_page": 7, "orig_end_page": 340, "type": "chapter_prose", }, "sb81": { "name": "Srimad-Bhagavatam Eighth Canto Part 1", "original": "sb8.1.pdf", "vedabase_start": 13074, "vedabase_end": 13167, "orig_start_page": 7, "orig_end_page": 366, "type": "chapter_prose", }, "sb82": { "name": "Srimad-Bhagavatam Eighth Canto Part 2", "original": "SB8.2.pdf", "vedabase_start": 13167, "vedabase_end": 13242, "orig_start_page": 7, "orig_end_page": 292, "type": "chapter_prose", }, "sb83": { "name": "Srimad-Bhagavatam Eighth Canto Part 3", "original": "SB8.3.pdf", "vedabase_start": 13242, "vedabase_end": 13320, "orig_start_page": 7, "orig_end_page": 318, "type": "chapter_prose", }, "sb91": { "name": "Srimad-Bhagavatam Ninth Canto Part 1", "original": "SB9.1.pdf", "vedabase_start": 13320, "vedabase_end": 13394, "orig_start_page": 7, "orig_end_page": 305, "type": "chapter_prose", }, "sb92": { "name": "Srimad-Bhagavatam Ninth Canto Part 2", "original": "SB9.2.pdf", "vedabase_start": 13394, "vedabase_end": 13472, "orig_start_page": 8, "orig_end_page": 325, "type": "chapter_prose", }, "sb93": { "name": "Srimad-Bhagavatam Ninth Canto Part 3", "original": "SB9.3.pdf", "vedabase_start": 13472, "vedabase_end": 13545, "orig_start_page": 22, "orig_end_page": 287, "type": "chapter_prose", }, "sb101": { "name": "Srimad-Bhagavatam Tenth Canto Part 1", "original": "SB10.1.pdf", "vedabase_start": 13545, "vedabase_end": 13648, "orig_start_page": 10, "orig_end_page": 394, "type": "chapter_prose", }, "sb102": { "name": "Srimad-Bhagavatam Tenth Canto Part 2", "original": "SB10.2.pdf", "vedabase_start": 13648, "vedabase_end": 13710, "orig_start_page": 9, "orig_end_page": 369, "type": "chapter_prose", }, "sb103": { "name": "Srimad-Bhagavatam Tenth Canto Part 3", "original": "SB10.3.pdf", "vedabase_start": 13710, "vedabase_end": 13770, "orig_start_page": 8, "orig_end_page": 109, "type": "chapter_prose", }, "cc_mad1": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.1", "original": "mad1.pdf", "vedabase_start": 9127, "vedabase_end": 9236, "orig_start_page": 24, "orig_end_page": 382, "type": "chapter_prose", }, "cc_mad2": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.2", "original": "mad2.pdf", "vedabase_start": 9237, "vedabase_end": 9357, "orig_start_page": 33, "orig_end_page": 393, "type": "chapter_prose", }, "cc_mad3": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.3", "original": "mad3.pdf", "vedabase_start": 9358, "vedabase_end": 9489, "orig_start_page": 33, "orig_end_page": 414, "type": "chapter_prose", }, "cc_mad4": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.4", "original": "mad4.pdf", "vedabase_start": 9490, "vedabase_end": 9601, "orig_start_page": 33, "orig_end_page": 363, "type": "chapter_prose", }, "cc_mad5": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.5", "original": "mad5.pdf", "vedabase_start": 9602, "vedabase_end": 9715, "orig_start_page": 33, "orig_end_page": 376, "type": "chapter_prose", }, "cc_mad6": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.6", "original": "mad6.pdf", "vedabase_start": 9716, "vedabase_end": 9817, "orig_start_page": 33, "orig_end_page": 335, "type": "chapter_prose", }, "cc_mad7": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.7", "original": "mad7.pdf", "vedabase_start": 9818, "vedabase_end": 9954, "orig_start_page": 33, "orig_end_page": 437, "type": "chapter_prose", }, "cc_mad8": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.8", "original": "mad8.pdf", "vedabase_start": 9955, "vedabase_end": 10100, "orig_start_page": 33, "orig_end_page": 460, "type": "chapter_prose", }, "cc_mad9": { "name": "Sri Caitanya-caritamrta Madhya-lila Vol.9", "original": "mad9.pdf", "vedabase_start": 10101, "vedabase_end": 10254, "orig_start_page": 30, "orig_end_page": 480, "type": "chapter_prose", }, "cc_ant1": { "name": "Sri Caitanya-caritamrta Antya-lila Vol.1", "original": "ant1.pdf", "vedabase_start": 10255, "vedabase_end": 10369, "orig_start_page": 33, "orig_end_page": 378, "type": "chapter_prose", }, "cc_ant2": { "name": "Sri Caitanya-caritamrta Antya-lila Vol.2", "original": "ant2.pdf", "vedabase_start": 10370, "vedabase_end": 10486, "orig_start_page": 33, "orig_end_page": 385, "type": "chapter_prose", }, "cc_ant3": { "name": "Sri Caitanya-caritamrta Antya-lila Vol.3", "original": "ant3.pdf", "vedabase_start": 10487, "vedabase_end": 10574, "orig_start_page": 33, "orig_end_page": 305, "type": "chapter_prose", }, "cc_ant4": { "name": "Sri Caitanya-caritamrta Antya-lila Vol.4", "original": "ant4.pdf", "vedabase_start": 10575, "vedabase_end": 10671, "orig_start_page": 33, "orig_end_page": 332, "type": "chapter_prose", }, "cc_ant5": { "name": "Sri Caitanya-caritamrta Antya-lila Vol.5", "original": "ant5.pdf", "vedabase_start": 10672, "vedabase_end": 10773, "orig_start_page": 33, "orig_end_page": 350, "type": "chapter_prose", }, # --- Letters --- "letters_v1": { "name": "Letters from Srila Prabhupada Vol.1 (1947-1969)", "original": "Letters_from_Srila_Prabhupada-Vol.1_1947-1969.pdf", "vedabase_start": 6234, "vedabase_end": 6641, "orig_start_page": 11, "orig_end_page": 661, "type": "chapter_prose", "noise_mode": "letters", }, "letters_v2": { "name": "Letters from Srila Prabhupada Vol.2 (1969-1970)", "original": "Letters_from_Srila_Prabhupada-Vol.2_1969-1970.pdf", "vedabase_start": 6620, "vedabase_end": 7060, "orig_start_page": 11, "orig_end_page": 672, "type": "chapter_prose", "noise_mode": "letters", }, "letters_v3": { "name": "Letters from Srila Prabhupada Vol.3 (1970-1972)", "original": "Letters_from_Srila_Prabhupada-Vol.3_1970-1972 (1).pdf", "vedabase_start": 7040, "vedabase_end": 7523, "orig_start_page": 11, "orig_end_page": 673, "type": "chapter_prose", "noise_mode": "letters", }, "letters_v4": { "name": "Letters from Srila Prabhupada Vol.4 (1972-1975)", "original": "Letters_from_Srila_Prabhupada-Vol.4_1972-1975.pdf", "vedabase_start": 7524, "vedabase_end": 8019, "orig_start_page": 11, "orig_end_page": 670, "type": "chapter_prose", "noise_mode": "letters", }, "letters_v5": { "name": "Letters from Srila Prabhupada Vol.5 (1975-1977)", "original": "Letters_from_Srila_Prabhupada-Vol.5_1975-1977.pdf", "vedabase_start": 8020, "vedabase_end": 8488, "orig_start_page": 11, "orig_end_page": 630, "type": "chapter_prose", "noise_mode": "letters", }, } BG_CHAPTER_TITLES = { 1: "Observing the Armies", 2: "Contents of the Gita Summarized", 3: "Karma-yoga", 4: "Transcendental Knowledge", 5: "Karma-yoga-Action in Krsna Consciousness", 6: "Sankhya-yoga", 7: "Knowledge of the Absolute", 8: "Attaining the Supreme", 9: "The Most Confidential Knowledge", 10: "The Opulence of the Absolute", 11: "The Universal Form", 12: "Devotional Service", 13: "Nature, the Enjoyer, and Consciousness", 14: "The Three Modes of Material Nature", 15: "The Yoga of the Supreme Person", 16: "The Divine and Demoniac Natures", 17: "The Divisions of Faith", 18: "Conclusion-The Perfection of Renunciation", } # ============================================================ # TEXT NORMALIZATION # ============================================================ DIACRITICS_MAP = { 'ā': 'a', 'ī': 'i', 'ū': 'u', 'ṛ': 'r', 'ṝ': 'r', 'ḷ': 'l', 'ṁ': 'm', 'ṃ': 'm', 'ḥ': 'h', 'ñ': 'n', 'ṅ': 'n', 'ṇ': 'n', 'ṭ': 't', 'ḍ': 'd', 'ś': 's', 'ṣ': 's', 'č': 'c', 'Ā': 'A', 'Ī': 'I', 'Ū': 'U', 'Ṛ': 'R', 'Ś': 'S', 'Ṣ': 'S', 'Ṇ': 'N', 'Ṭ': 'T', 'Ḍ': 'D', } def strip_diacritics(text: str) -> str: for diac, ascii_char in DIACRITICS_MAP.items(): text = text.replace(diac, ascii_char) text = unicodedata.normalize('NFKD', text) text = ''.join(c for c in text if not unicodedata.combining(c)) return text def normalize_for_comparison(text: str, letters_mode: bool = False) -> str: # Fix hyphenated line breaks text = re.sub(r'(\w)[\-\u00ad]\s*\n\s*(\w)', r'\1\2', text) text = re.sub(r'(\w)[\-\u00ad]\s+(\w)', r'\1\2', text) # Strip diacritics text = strip_diacritics(text) if letters_mode: # Remove letter-specific structural elements from originals: # Running headers: "JANUARY, 1968", "Letters from Srila Prabhupada" text = re.sub(r'(?mi)^\s*(?:JANUARY|FEBRUARY|MARCH|APRIL|MAY|JUNE|JULY|AUGUST|SEPTEMBER|OCTOBER|NOVEMBER|DECEMBER)\s*,?\s*\d{4}\s*$', '', text) text = re.sub(r'(?mi)^\s*Letters?\s+from\s+Sr[iī]la\s+Prabhup[aā]da\s*$', '', text) # Letter codes in original: "68-1-1" or "68- 1 - 1" (OCR spaces) text = re.sub(r'\b\d{2}\s*-\s*\d{1,2}\s*-\s*\d{1,3}\b', '', text) # Vedabase letter markers: "Letter to: Name", bullet "•" text = re.sub(r'Letter to:\s*\S+', '', text) text = re.sub(r'[•·]', '', text) else: # Remove page headers/footers from original scans text = re.sub(r'\d+\s+Bhagavad-g\w*\s+As\s+It\s+Is\s+\[Ch\.\s*\d+', '', text) text = re.sub(r'Text[s]?\s+\d+[\-\d]*\]\s+.*?\d{2,}', '', text) # Sri Isopanisad headers: "Mantra One 13" or "14 Sri Isopanisad" text = re.sub(r'(?m)^\s*Mantra\s+\w+\s+\d{1,3}\s*$', '', text) text = re.sub(r'(?m)^\s*\d{1,3}\s+Sri\s+[Ii]sopani.?ad\s*$', '', text) # Remove standalone page numbers text = re.sub(r'(?m)^\s*\d{1,4}\s*$', '', text) # Remove OCR garbage (non-alpha sequences 2+ chars) text = re.sub(r'(? dict[int, tuple[int, int]]: """Find chapter start pages in original scan PDF. Returns {chapter_num: (start_page, end_page)}.""" doc = fitz.open(pdf_path) total_pages = len(doc) word_map = { 'ONE': 1, 'TWO': 2, 'THREE': 3, 'FOUR': 4, 'FIVE': 5, 'SIX': 6, 'SEVEN': 7, 'EIGHT': 8, 'NINE': 9, 'TEN': 10, 'ELEVEN': 11, 'TWELVE': 12, 'THIRTEEN': 13, 'FOURTEEN': 14, 'FIFTEEN': 15, 'SIXTEEN': 16, 'SEVENTEEN': 17, 'EIGHTEEN': 18, } # Use word boundaries to avoid FOURTEEN matching as FOUR, etc. word_pattern = '|'.join(sorted(word_map.keys(), key=len, reverse=True)) chapter_starts = [] for i in range(30, min(950, total_pages)): # skip TOC and index text = doc[i].get_text() first_400 = text[:400] m = re.search(rf'CHAPTER\s+({word_pattern})\b', first_400, re.IGNORECASE) if m: ch_num = word_map[m.group(1).upper()] # Must have TEXT(S) 1 or TEXT l nearby (within 2 pages) combined = text if i + 1 < total_pages: combined += doc[i + 1].get_text() if re.search(r'TEXTS?\s+[l1]', combined[:4000]): chapter_starts.append((ch_num, i)) doc.close() # Deduplicate: keep first occurrence of each chapter seen = set() unique = [] for num, page in chapter_starts: if num not in seen: seen.add(num) unique.append((num, page)) # Sort by page position unique.sort(key=lambda x: x[1]) # Build page ranges chapters = {} for i, (num, start) in enumerate(unique): end = unique[i + 1][1] if i + 1 < len(unique) else min(950, total_pages) chapters[num] = (start, end) return chapters def find_chapter_pages_vedabase(pdf_path: str, start_page: int, end_page: int) -> dict[int, tuple[int, int]]: """Find chapter start pages in Vedabase PDF. Uses specific title patterns per chapter + TEXT on same page.""" doc = fitz.open(pdf_path) title_patterns = { 1: r'1\.\s+Observing the Armies', 2: r'2\.\s+Contents of the G[iī]t', 3: r'3\.\s+Karma-yoga\b', 4: r'4\.\s+Transcendental Knowledge', 5: r'5\.\s+Karma-yoga.Action', 6: r'6\.\s+S[aā][nṅ]khya', 7: r'7\.\s+Knowledge of the Absolute', 8: r'8\.\s+Attaining the Supreme', 9: r'9\.\s+(?:The\s+)?Most Confidential', 10: r'10\.\s+(?:The\s+)?Opulence', 11: r'11\.\s+(?:The\s+)?Universal Form', 12: r'12\.\s+Devotional Service', 13: r'13\.\s+Nature', 14: r'14\.\s+(?:The\s+)?Three Modes', 15: r'15\.\s+(?:The\s+)?Yoga of the Supreme', 16: r'16\.\s+(?:The\s+)?Divine and Demoniac', 17: r'17\.\s+(?:The\s+)?Divisions of Faith', 18: r'18\.\s+Conclusion', } chapter_map = {} for i in range(start_page, min(end_page, len(doc))): text = doc[i].get_text() for ch_num, pattern in title_patterns.items(): if ch_num not in chapter_map and re.search(pattern, text): if re.search(r'TEXTS?\s+\d', text): chapter_map[ch_num] = i doc.close() # Sort by page and build ranges sorted_chapters = sorted(chapter_map.items(), key=lambda x: x[1]) chapters = {} for i, (num, start) in enumerate(sorted_chapters): end = sorted_chapters[i + 1][1] if i + 1 < len(sorted_chapters) else end_page chapters[num] = (start, end) return chapters def extract_chapter_text(pdf_path: str, start_page: int, end_page: int, strip_headers: bool = False) -> str: """Extract text from a range of pages. If strip_headers=True, removes the first 2 non-empty lines of each page (running headers + page numbers from scanned books).""" doc = fitz.open(pdf_path) text = "" for i in range(start_page, min(end_page, len(doc))): page_text = doc[i].get_text() if strip_headers: lines = page_text.split('\n') removed = 0 cleaned = [] for line in lines: if removed < 2 and line.strip(): removed += 1 continue cleaned.append(line) page_text = '\n'.join(cleaned) text += page_text + "\n" doc.close() return text # ============================================================ # VERSE EXTRACTION # ============================================================ def split_into_mantras(text: str) -> dict[str, dict]: """Split text into mantra sections by MANTRA markers (for Sri Isopanisad etc.).""" word_map = { 'One': '1', 'Two': '2', 'Three': '3', 'Four': '4', 'Five': '5', 'Six': '6', 'Seven': '7', 'Eight': '8', 'Nine': '9', 'Ten': '10', 'Eleven': '11', 'Twelve': '12', 'Thirteen': '13', 'Fourteen': '14', 'Fifteen': '15', 'Sixteen': '16', 'Seventeen': '17', 'Eighteen': '18', } # Match "Mantra One", "MANTRA ONE", "Mantra 1", etc. word_pattern = '|'.join(sorted(word_map.keys(), key=len, reverse=True)) pattern = rf'(?:MANTRA|Mantra)\s+({word_pattern}|\d+)' markers = list(re.finditer(pattern, text, re.IGNORECASE)) if not markers: return {} mantras = {} for i, match in enumerate(markers): num_str = match.group(1) # Convert word to number if needed for word, num in word_map.items(): if num_str.lower() == word.lower(): num_str = num break start = match.end() end = markers[i + 1].start() if i + 1 < len(markers) else len(text) section_text = text[start:end].strip() mantra = {"verse": num_str, "translation": "", "purport": ""} # Extract TRANSLATION trans_match = re.search( r'(?:TRANSLATION|Translation)\s*(.*?)(?=PURPORT|Purport|$)', section_text, re.DOTALL | re.IGNORECASE ) if trans_match: mantra["translation"] = trans_match.group(1).strip() # Extract PURPORT purp_match = re.search( r'(?:PURPORT|Purport)\s*(.*?)$', section_text, re.DOTALL | re.IGNORECASE ) if purp_match: mantra["purport"] = purp_match.group(1).strip() mantras[num_str] = mantra return mantras def split_into_text_words(text: str) -> dict[str, dict]: """Split text by TEXT ONE/TWO/... markers (for Nectar of Instruction etc.).""" word_map = { 'ONE': '1', 'TWO': '2', 'THREE': '3', 'FOUR': '4', 'FIVE': '5', 'SIX': '6', 'SEVEN': '7', 'EIGHT': '8', 'NINE': '9', 'TEN': '10', 'ELEVEN': '11', 'TWELVE': '12', 'THIRTEEN': '13', 'FOURTEEN': '14', 'FIFTEEN': '15', 'SIXTEEN': '16', 'SEVENTEEN': '17', 'EIGHTEEN': '18', } word_pattern = '|'.join(sorted(word_map.keys(), key=len, reverse=True)) pattern = rf'TEXT\s+({word_pattern}|\d+(?:\s*[-\u2013]\s*\d+)?)' markers = list(re.finditer(pattern, text, re.IGNORECASE)) if not markers: return {} verses = {} for i, match in enumerate(markers): num_str = match.group(1).strip().upper() if num_str in word_map: num_str = word_map[num_str] num_str = re.sub(r'\s+', '', num_str) start = match.end() end = markers[i + 1].start() if i + 1 < len(markers) else len(text) section_text = text[start:end].strip() verse = {"verse": num_str, "translation": "", "purport": ""} trans_match = re.search( r'TRANSLATION\s*(.*?)(?=PURPORT|$)', section_text, re.DOTALL | re.IGNORECASE ) if trans_match: verse["translation"] = trans_match.group(1).strip() purp_match = re.search( r'PURPORT\s*(.*?)$', section_text, re.DOTALL | re.IGNORECASE ) if purp_match: verse["purport"] = purp_match.group(1).strip() verses[num_str] = verse return verses def split_chapter_into_verses(chapter_text: str) -> dict[str, dict]: """Split chapter text into verse sections by TEXT markers (numeric only).""" pattern = r'TEXT[S]?\s+(\d+(?:\s*[-\u2013]\s*\d+)?)' markers = list(re.finditer(pattern, chapter_text, re.IGNORECASE)) if not markers: return {} verses = {} for i, match in enumerate(markers): verse_num = re.sub(r'\s+', '', match.group(1)) start = match.end() end = markers[i + 1].start() if i + 1 < len(markers) else len(chapter_text) section_text = chapter_text[start:end].strip() verse = {"verse": verse_num, "translation": "", "purport": ""} # Extract TRANSLATION trans_match = re.search( r'TRANSLATION\s*(.*?)(?=PURPORT|$)', section_text, re.DOTALL | re.IGNORECASE ) if trans_match: verse["translation"] = trans_match.group(1).strip() # Extract PURPORT purp_match = re.search( r'PURPORT\s*(.*?)$', section_text, re.DOTALL | re.IGNORECASE ) if purp_match: verse["purport"] = purp_match.group(1).strip() verses[verse_num] = verse return verses # ============================================================ # COMPARISON # ============================================================ @dataclass class Difference: verse: str chapter: int section: str diff_type: str original: str vedabase: str context_before: str = "" context_after: str = "" def is_sanskrit(text: str) -> bool: """Detect if text is Sanskrit transliteration (not English).""" t = strip_diacritics(text.lower().strip()) if not t: return False # Common Sanskrit word endings and patterns sanskrit_markers = [ r'\b\w+[aā][hḥ]\b', # -ah endings (visarga) r'\b\w+[aā][mṁ]\b', # -am endings (anusvara) r'\b\w*[kc]r[sṣ][nṇ]a\b', # krsna/krishna r'\b\w*yoga\b', r'\b\w*gita\b', r'\b\w*bhagav\w+\b', r'\b\w*arjun\w*\b', r'\b\w*brahm\w+\b', r'\b\w*veda\w*\b', r'\b\w*dharma\w*\b', r'\b\w*karma\w*\b', r'\b\w*atma\w*\b', r'\b\w*maya[v]?\w*\b', r'\b\w*guru\w*\b', r'\b\w*deva\w*\b', r'\b\w*mantra\w*\b', r'\b\w*sutra\w*\b', r'\b\w*tantra\w*\b', r'\b\w*prana\w*\b', r'\b\w*jiva\w*\b', r'\b\w*param\w*\b', r'\b\w*sannyasi\w*\b', r'\b\w*bhakti\w*\b', r'\b\w*mukti\w*\b', r'\b\w*samadhi\w*\b', r'\b\w*nirvana\w*\b', r'\buv[aā]ca\b', r'\btv[aā]m\b', r'\bmah[aā]\b', r'\bca\b', r'\beva\b', r'\btat\b', r'\byat\b', ] words = t.split() if not words: return False # If text contains hyphens connecting short syllables, likely verse transliteration # e.g., "sarva-dharman parityajya mam ekam saranam vraja" hyphen_words = sum(1 for w in words if '-' in w and len(w) > 3) if hyphen_words >= 2: return True # If most words match Sanskrit patterns sanskrit_count = 0 for word in words: clean = re.sub(r'[^a-z]', '', word) if len(clean) < 2: continue for pattern in sanskrit_markers: if re.search(pattern, word): sanskrit_count += 1 break else: # Check for typical Sanskrit phonetics (lots of a's, common consonant clusters) if re.match(r'^[a-z]+$', clean) and len(clean) >= 3: vowel_ratio = sum(1 for c in clean if c in 'aeiou') / len(clean) if vowel_ratio > 0.45 and any(cl in clean for cl in ['bh', 'dh', 'gh', 'kh', 'ph', 'th', 'sh', 'ch']): sanskrit_count += 1 meaningful_words = sum(1 for w in words if len(re.sub(r'[^a-z]', '', w.lower())) >= 2) if meaningful_words > 0 and sanskrit_count / meaningful_words > 0.6: return True return False def is_garbage(text: str) -> bool: """Detect if text is OCR garbage (not real English content).""" t = text.strip() if not t: return True # Count actual English letters vs total chars alpha_chars = sum(1 for c in t if c.isalpha()) total_chars = len(t.replace(' ', '')) if total_chars == 0: return True alpha_ratio = alpha_chars / total_chars # If less than 50% alphabetic, it's garbage if alpha_ratio < 0.5: return True # If mostly single characters separated by spaces/punctuation words = t.split() if words and all(len(w.strip('.,;:!?\'"()-')) <= 1 for w in words): return True # Common OCR garbage patterns if re.match(r'^[\s\W]*[^a-zA-Z]*[\s\W]*$', t): return True # Repeating single character (e.g., "I I I I I" -> alpha "IIIII") alpha_only = re.sub(r'[^a-zA-Z]', '', t) if alpha_only and len(set(alpha_only.lower())) == 1: return True return False COMMON_ENGLISH = { 'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'shall', 'can', 'must', 'not', 'no', 'yes', 'and', 'or', 'but', 'if', 'then', 'so', 'for', 'nor', 'yet', 'both', 'this', 'that', 'these', 'those', 'it', 'its', 'he', 'she', 'they', 'him', 'her', 'them', 'his', 'their', 'my', 'your', 'our', 'who', 'what', 'which', 'where', 'when', 'how', 'why', 'all', 'each', 'we', 'every', 'any', 'some', 'many', 'much', 'more', 'most', 'other', 'such', 'only', 'also', 'very', 'just', 'than', 'too', 'own', 'same', 'with', 'from', 'into', 'upon', 'about', 'between', 'to', 'of', 'through', 'after', 'before', 'above', 'below', 'over', 'under', 'on', 'at', 'by', 'as', 'in', 'up', 'out', 'off', 'down', 'away', 'back', 'one', 'two', 'three', 'first', 'second', 'third', 'last', 'next', 'new', 'old', 'great', 'good', 'bad', 'long', 'high', 'right', 'real', 'man', 'men', 'woman', 'women', 'people', 'world', 'life', 'death', 'time', 'way', 'work', 'part', 'place', 'case', 'point', 'fact', 'said', 'says', 'called', 'known', 'made', 'given', 'done', 'taken', 'god', 'lord', 'king', 'soul', 'body', 'mind', 'self', 'nature', 'therefore', 'however', 'because', 'although', 'since', 'thus', 'never', 'always', 'still', 'even', 'already', 'here', 'there', 'now', 'once', 'again', 'well', 'person', 'material', 'spiritual', 'supreme', 'absolute', 'consciousness', 'knowledge', 'devotional', 'service', 'living', 'entity', 'transcendental', 'conditioned', 'liberated', 'actually', 'certainly', 'directly', 'gradually', 'simply', 'without', 'within', 'according', 'beyond', 'against', 'among', 'means', 'become', 'becomes', 'becoming', 'understand', 'understanding', 'different', 'same', 'another', 'nothing', 'something', 'everything', 'everyone', 'himself', 'itself', 'themselves', 'like', 'take', 'give', 'come', 'go', 'see', 'know', 'think', 'say', 'make', 'find', 'want', 'tell', 'ask', 'use', 'try', 'leave', 'call', 'need', 'keep', 'let', 'begin', 'show', 'hear', 'play', 'run', 'move', 'live', 'believe', 'hold', 'bring', 'happen', 'write', 'provide', 'sit', 'stand', 'lose', 'pay', 'meet', 'include', 'continue', 'set', 'learn', 'change', 'lead', 'follow', 'stop', 'create', 'speak', 'read', 'grow', 'open', 'walk', 'win', 'offer', 'remember', 'consider', 'appear', 'buy', 'serve', 'die', 'send', 'build', 'stay', 'fall', 'cut', 'reach', 'kill', 'remain', 'suggest', 'raise', 'pass', 'sell', 'require', 'report', 'decide', 'pull', 'compare', 'interest', 'yesterday', 'today', 'tomorrow', 'until', 'unto', 'upon', 'during', 'while', 'accept', 'verse', 'chapter', 'previous', 'thing', 'water', 'fire', 'earth', 'air', 'food', 'light', 'night', 'day', 'power', 'hand', 'eye', 'head', 'face', 'side', 'end', 'state', 'form', 'matter', 'energy', 'desire', # Additional common English words for better detection 'human', 'word', 'conclude', 'glossary', 'index', 'cure', 'social', 'genuine', 'name', 'move', 'address', 'declare', 'depend', 'develop', 'nation', 'citizen', 'poet', 'friend', 'mutual', 'religion', 'solution', 'problem', 'crime', 'frontier', 'contain', 'sincere', 'volume', 'teach', 'feel', 'felt', 'sign', 'dance', 'aspect', 'glory', 'discuss', 'explore', 'return', 'present', 'statement', 'fatigue', 'tree', 'important', 'import', 'lecture', 'advise', 'campus', 'hall', 'university', 'nation', 'practice', 'complete', 'full', 'free', 'eternal', 'divine', 'love', 'worship', 'pure', 'perfect', 'control', 'describe', 'perform', 'act', 'help', 'cause', 'effect', 'quality', 'activity', 'reveal', 'realize', 'achieve', 'obtain', 'receive', 'observe', 'express', 'regard', 'connect', 'relate', 'experience', 'fear', 'anger', 'happy', 'peace', 'true', 'false', 'sense', 'age', 'book', 'page', 'line', 'heart', 'number', 'order', 'please', 'thank', 'dear', 'letter', 'write', 'read', 'note', 'yours', 'family', 'home', 'house', 'city', 'country', 'land', 'child', 'father', 'mother', 'son', 'daughter', 'brother', 'sister', 'birth', 'suffer', 'enjoy', 'attach', 'detach', 'surrender', 'animal', 'plant', 'kind', 'type', 'class', 'group', 'system', 'process', 'method', 'rule', 'law', 'principle', 'standard', 'position', 'condition', 'situation', 'circumstance', 'example', 'reason', 'purpose', 'goal', 'result', 'answer', 'question', 'subject', 'object', 'mean', 'explain', 'mention', 'refer', 'support', 'maintain', 'establish', 'introduce', 'apply', 'produce', 'cover', 'direct', 'guide', 'instruct', 'inform', 'branch', 'cow', 'bath', 'bless', 'carve', 'indicate', 'become', 'contain', 'fifty', 'forty', 'thirty', 'twenty', 'hundred', 'pretend', 'faint', 'engrave', 'enumerate', 'code', 'synopsis', } def _is_english_word(w: str) -> bool: """Check if a word is common English, including basic stemming.""" if w in COMMON_ENGLISH: return True # Try removing common suffixes suffixes = ('s', 'es', 'ed', 'ing', 'ly', 'ness', 'ment', 'tion', 'sion', 'able', 'ible', 'ful', 'less', 'ous', 'ive', 'er', 'est', 'ity', 'al', 'ary', 'ence', 'ance', 'ation', 'ment', 'ure', 'ise', 'ize') for suffix in suffixes: if w.endswith(suffix) and len(w) > len(suffix) + 2: stem = w[:-len(suffix)] if stem in COMMON_ENGLISH: return True # Handle 'e' doubling: "likened" -> "liken" -> try "like" if stem.endswith('e') and stem[:-1] in COMMON_ENGLISH: return True # Handle 'e' dropping before suffix: "giver" -> "giv" -> try "give" if len(stem) >= 2 and (stem + 'e') in COMMON_ENGLISH: return True # Try a second suffix strip for compound suffixes # e.g., "followers" -> "follower" -> "follow" for suffix2 in suffixes: if stem.endswith(suffix2) and len(stem) > len(suffix2) + 2: stem2 = stem[:-len(suffix2)] if stem2 in COMMON_ENGLISH: return True if stem2.endswith('e') and stem2[:-1] in COMMON_ENGLISH: return True if len(stem2) >= 2 and (stem2 + 'e') in COMMON_ENGLISH: return True return False def count_english_words(text: str) -> int: """Count how many words in text are common English words.""" words = re.findall(r'[a-zA-Z]+', strip_diacritics(text.lower())) return sum(1 for w in words if _is_english_word(w)) SANSKRIT_WORDS = { 'krsna', 'arjuna', 'bhisma', 'drona', 'brahma', 'visnu', 'narayana', 'hrsikesa', 'canakya', 'pandit', 'sankhya', 'ksatriya', 'brahmana', 'brahmanas', 'ksatriyas', 'lokayatikas', 'vaibhasikas', 'krpana', 'paramatma', 'bhagavan', 'mahabharata', 'vedanta', 'vedas', 'gita', 'yoga', 'karma', 'dharma', 'atma', 'deva', 'guru', 'mantra', 'sutra', 'prana', 'maya', 'loka', 'rasa', 'prema', 'bhakti', 'jnana', 'mukti', 'siddhi', 'sakti', 'caitanya', 'govinda', 'madhava', 'kesava', 'vasudeva', 'pradyumna', 'aniruddha', 'baladeva', 'kuruksetra', 'hastinapura', 'vrndavana', 'mathura', 'dvaraka', 'prahlada', 'hiranyakasipu', 'gandiva', 'pandu', 'kunti', 'draupadi', 'duryodhana', 'dhrtarastra', 'vidura', 'sanjaya', 'vyasa', 'sukadeva', 'narada', 'kapila', 'patanjali', 'sankaracarya', 'ramanujacarya', 'bhagavad', 'upanisad', 'upanisads', 'purana', 'puranas', 'vaisnava', 'pasandi', 'sastra', 'sastras', 'sampradaya', 'acarya', 'acaryas', 'sannyasi', 'sannyasis', 'grhastha', 'vanaprastha', 'brahmacari', 'sudra', 'sudras', 'vaisya', 'vaisyas', 'svami', 'gosvami', 'prabhupada', 'srimad', 'bhagavatam', 'isopanisad', 'mundaka', 'katha', 'svetasvatara', 'aitareya', 'taittiriya', 'chandogya', 'brhadaranyaka', 'manu', 'smrti', 'sruti', 'isvara', 'jiva', 'jivas', 'prakrti', 'purusa', 'guna', 'gunas', 'sattva', 'rajas', 'tamas', 'ahankara', 'buddhi', 'manas', 'citta', 'samadhi', 'nirvana', 'samsara', 'moksa', 'kaivalya', 'vaikuntha', 'goloka', 'kailasa', 'svarga', 'naraka', 'yajna', 'tapas', 'dana', 'satya', 'ahimsa', 'asteya', 'brahmacarya', 'aparigraha', 'pranayama', 'pratyahara', 'dharana', 'dhyana', 'asana', 'niyama', 'yama', 'kriya', 'avatara', 'avatars', 'lila', 'lilas', 'dasa', 'dasya', 'sakhya', 'vatsalya', 'madhurya', 'srngara', 'kirtana', 'bhajana', 'puja', 'arati', 'prasada', 'tilaka', 'tulasi', 'ekadasi', 'dvadasi', 'caturdasi', 'purnima', 'amavasya', 'pandava', 'pandavas', 'kaurava', 'kauravas', 'yudhisthira', 'bhima', 'nakula', 'sahadeva', 'subhadra', 'abhimanyu', 'jayadratha', 'karna', 'salya', 'asvathama', 'krpacarya', 'dronacarya', 'bhismadeva', 'virata', 'drupada', 'dhrstadyumna', 'sikhandin', 'uttara', 'parikshit', 'pariksit', 'janaka', 'dasaratha', 'ramacandra', 'laksmana', 'hanuman', 'ravana', 'vibhisana', 'sugriva', 'sita', 'laksmi', 'sarasvati', 'durga', 'ganesa', 'kartikeya', 'skanda', 'siva', 'rudra', 'indra', 'varuna', 'agni', 'vayu', 'surya', 'candra', 'yama', 'kubera', 'brihaspati', 'sukracarya', 'rahu', 'ketu', 'mangala', 'budha', 'sani', } def is_ocr_of_sanskrit_name(orig: str, veda: str) -> bool: """Check if orig is an OCR-garbled version of a known Sanskrit/English mix on the Vedabase side. The Vedabase text is clean; if every word on the Vedabase side is either a known Sanskrit term OR a common English word, the original side is just OCR garbage of the same text.""" # If vedabase has significantly more words, it's added text, not just OCR o_split = orig.split() v_split = veda.split() if len(v_split) > len(o_split) + 1: return False # If original has significant English content, it's not OCR garbage # but a real English sentence (e.g., "This fact is corroborated in the Sruti Mantras") if count_english_words(orig) >= 3: return False v_clean = strip_diacritics(veda.lower()) v_words = re.findall(r'[a-z]+', v_clean) if not v_words: return False # Count how many Vedabase words are recognized (Sanskrit or English) sanskrit_count = sum(1 for w in v_words if w in SANSKRIT_WORDS) english_count = sum(1 for w in v_words if w in COMMON_ENGLISH) recognized = sanskrit_count + english_count # If ALL Vedabase words are recognized AND at least one is Sanskrit → noise if recognized >= len(v_words) and sanskrit_count >= 1: return True # If most words are recognized and there's Sanskrit if len(v_words) >= 2 and recognized >= len(v_words) - 1 and sanskrit_count >= 1: return True return False def looks_like_ocr_of_same_word(orig: str, veda: str) -> bool: """Check if orig is just an OCR-mangled version of veda (or vice versa). e.g., 'thdse' -> 'those', 'spuitual' -> 'spiritual'.""" # Normalize zero/O confusion before comparison orig_norm = orig.replace('0', 'O') veda_norm = veda.replace('0', 'O') # Strip all non-alpha, compare o = re.sub(r'[^a-zA-Z\s]', '', orig_norm.lower()).split() v = re.sub(r'[^a-zA-Z\s]', '', veda_norm.lower()).split() if not o or not v: return True # Check word-by-word if same number of words if len(o) == len(v): all_ocr = True for ow, vw in zip(o, v): if not _words_are_ocr_similar(ow, vw): all_ocr = False break if all_ocr: return True # Also try flat comparison for merged/split words o_flat = ''.join(o) v_flat = ''.join(v) if not o_flat or not v_flat: return True return _words_are_ocr_similar(o_flat, v_flat) def _words_are_ocr_similar(a: str, b: str) -> bool: """Check if two alpha strings differ by OCR-level noise. Uses SequenceMatcher to handle insertions/deletions properly.""" if not a or not b: return True if a == b: return True longer = max(len(a), len(b)) shorter = min(len(a), len(b)) if shorter < longer * 0.5: return False ratio = difflib.SequenceMatcher(None, a, b).ratio() return ratio >= 0.55 def is_noise(orig: str, veda: str, letters_mode: bool = False) -> bool: """Filter: assume texts SHOULD be identical. Only keep differences that cannot be explained by OCR errors, diacritics, or transliteration. When letters_mode=True, uses stricter criteria so real single-word edits (name spelling changes, word corrections) are not filtered out.""" o = strip_diacritics(orig.lower().strip()) v = strip_diacritics(veda.lower().strip()) # Running header leak: OCR merges page header "[Madhya-lila, Ch. N" # into the text. Strip it and re-compare. header_pat = r'\s*\[\s*(?:M\s*a\s*d\s*h\s*y\s*a|A\s*n\s*t\s*y\s*a|A\s*d\s*i)\s*-\s*l[iī]l[aā],?\s*Ch\.\s*\d+.*$' orig_no_hdr = re.sub(header_pat, '', orig, flags=re.IGNORECASE).strip() if orig_no_hdr != orig.strip() and orig_no_hdr: o_h = re.sub(r'[^a-z]', '', strip_diacritics(orig_no_hdr.lower())) v_h = re.sub(r'[^a-z]', '', strip_diacritics(veda.lower().strip())) if o_h and v_h: if difflib.SequenceMatcher(None, o_h, v_h).ratio() >= 0.8: return True # Either side is OCR garbage if is_garbage(orig) or is_garbage(veda): return True # After diacritics normalization, same? if o == v: return True # OCR zero/O confusion: replace 0 with o before comparing o_zero_fix = o.replace('0', 'o') v_zero_fix = v.replace('0', 'o') if o_zero_fix == v_zero_fix: return True # Only alpha chars — same? o_alpha = re.sub(r'[^a-z]', '', o) v_alpha = re.sub(r'[^a-z]', '', v) if o_alpha == v_alpha: return True # Alpha with zero-to-o fix o_alpha_z = re.sub(r'[^a-z]', '', o_zero_fix) if o_alpha_z == v_alpha: return True # Very short content (< 4 chars alpha) if len(o_alpha) < 4 or len(v_alpha) < 4: return True # --- LETTERS MODE: stricter filtering --- # In letters mode, if both sides are clean text (no OCR garbage chars), # the difference is a real editorial change, not OCR noise. # Still filter: alignment mismatches and OCR-garbled text. if letters_mode: _garbage_chars = re.compile(r'[!?(){}\[\]|@#$%^&*ࢺ<>=~`]') orig_clean = not _garbage_chars.search(orig) veda_clean = not _garbage_chars.search(veda) if orig_clean and veda_clean: # Still filter alignment false positives (very low similarity) if len(o_alpha) > 25 and len(v_alpha) > 25: ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() if ratio < 0.25: return True if max(len(o_alpha), len(v_alpha)) > 40: shorter, longer = sorted([len(o_alpha), len(v_alpha)]) if shorter > 0 and longer / shorter > 3: ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() if ratio < 0.3: return True # Clean text, not alignment noise → real edit return False # Alignment false positive: both sides are long but completely different content # This indicates a page boundary misalignment, not an editorial change # Alignment false positive: both sides are long but completely different content if len(o_alpha) > 25 and len(v_alpha) > 25: ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() if ratio < 0.25: return True # One side much longer than the other with low similarity — misaligned pages if max(len(o_alpha), len(v_alpha)) > 40: shorter, longer = sorted([len(o_alpha), len(v_alpha)]) if shorter > 0 and longer / shorter > 3: ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() if ratio < 0.3: return True # Consecutive uppercase in middle of text = severe OCR garble (any length) # E.g., "fNOILOA30iO11VLJN" or "VUSJJauJpUVJSJopUfl" alpha_only_orig = re.sub(r'[^a-zA-Z]', '', orig) if alpha_only_orig and re.search(r'[a-z][A-Z]{2,}', alpha_only_orig): return True # OCR garble: special characters embedded within words # E.g., "Kr!?I).a" (garbled "Krsna"), "Sri/a" (garbled "Srila") # Pattern: a letter followed by 2+ special chars then another letter if re.search(r'[a-zA-Z][!?(){}\[\]|@#$%^&*]{2,}[a-zA-Z]', orig): return True # Abbreviation expansion: "Bs." -> "Brahma-samhita", "Bg." -> "Bhagavad-gita" # Original has abbreviated reference, vedabase has full name if re.search(r'\(Bs\.|\(Bg\.|\(SB\.|\(Cc\.|\(NOI\b|\(ISO\b', orig, re.IGNORECASE): return True # Dialogue speaker label misalignment (PQPA-style conversation books) # E.g., "? SRILA PRABHUPADA: The" vs "of life. Bob: Is the" if re.search(r'(PRABHUP[AĀ]DA|Prabhup[aā]da|Bob|Syamasundara|Hayagriva)\s*:', orig) or \ re.search(r'(PRABHUP[AĀ]DA|Prabhup[aā]da|Bob|Syamasundara|Hayagriva)\s*:', veda): # Check if the surrounding context suggests these are the same passage o_words = set(re.sub(r'[^a-z\s]', '', o).split()) v_words = set(re.sub(r'[^a-z\s]', '', v).split()) overlap = len(o_words & v_words) if overlap >= 1 and (overlap / max(len(o_words), len(v_words), 1)) >= 0.2: return True # Structural headings (PURPORT, SYNONYMS, TRANSLATION) merged with body text # E.g., "mind • .. PURPORT" vs "mind." — strip the heading and OCR noise, compare # Also handles: "tathii-as" vs "SYNONYMS tatha-as" (heading on vedabase side) for side_orig, side_veda in [(orig, veda), (veda, orig)]: # Strip PURPORT/SYNONYMS/TRANSLATION anywhere, but TEXT/TEXTS only at # start of text (not mid-reference like "Texts 8-9" in citations) structural = re.sub(r'\b(PURPORT|SYNONYMS|TRANSLATION)\b', '', side_orig, flags=re.IGNORECASE) structural = re.sub(r'^[\s\d.]*\bTEXTS?\b', '', structural, flags=re.IGNORECASE) structural = re.sub(r'[•·\.\s]+$', '', structural).strip() # Also strip leading page numbers like "11377" structural = re.sub(r'^\d{4,}\s*', '', structural).strip() if structural and structural != side_orig.strip(): s_alpha = re.sub(r'[^a-z]', '', strip_diacritics(structural.lower())) other_alpha = re.sub(r'[^a-z]', '', strip_diacritics(side_veda.lower())) if s_alpha and other_alpha: if s_alpha == other_alpha: return True # OCR similarity after stripping heading struct_ratio = difflib.SequenceMatcher(None, s_alpha, other_alpha).ratio() if struct_ratio >= 0.66: return True # Section heading/number appended after sentence end in original # E.g., "gopis. 14. Artistic" vs "gopis." or "ksepana. Dancing" vs "ksepana." # Strip everything after the last sentence-ending punctuation and compare o_trimmed = re.sub(r'([."\u201d])\s+\d*\.?\s*[A-Z].*$', r'\1', orig) if o_trimmed != orig: t_alpha = re.sub(r'[^a-z]', '', strip_diacritics(o_trimmed.lower())) if t_alpha and t_alpha == v_alpha: return True # Also check OCR similarity after trim (e.g., "ksepara" vs "ksepana") if t_alpha and v_alpha and len(t_alpha) >= 3: trim_ratio = difflib.SequenceMatcher(None, t_alpha, v_alpha).ratio() if trim_ratio >= 0.6: return True # Strip bracketed scripture citations from original, then re-compare # E.g., "[Bhagavad-gita 18.66] aeporter:" vs "Reporter:" o_no_cite = re.sub(r'\[.*?\]', '', orig).strip() if o_no_cite != orig.strip() and o_no_cite: nc_alpha = re.sub(r'[^a-z]', '', strip_diacritics(o_no_cite.lower())) if nc_alpha and nc_alpha == v_alpha: return True if nc_alpha and v_alpha and len(nc_alpha) >= 3: cite_ratio = difflib.SequenceMatcher(None, nc_alpha, v_alpha).ratio() if cite_ratio >= 0.55: return True # Synonym entry with same meaning: "snuUiiyiifl-of" vs "snusayah-of" # Both sides are Sanskrit synonym entries (word-meaning format) # If the English meaning part is identical, it's OCR of the same entry # Use last hyphen to handle multi-hyphenated Sanskrit compounds # e.g., "tii-sobiiro-of" vs "ta-sabara-of" (meaning is "of" after last hyphen) o_syn_str = orig.strip().rstrip(',.') v_syn_str = re.sub(r'^(?:SYNONYMS|\d+)\s+', '', veda.strip().rstrip(',.')) o_last_hyp = o_syn_str.rfind('-') v_last_hyp = v_syn_str.rfind('-') if o_last_hyp > 0 and v_last_hyp > 0: o_meaning = strip_diacritics(o_syn_str[o_last_hyp+1:].strip().lower()) v_meaning = strip_diacritics(v_syn_str[v_last_hyp+1:].strip().lower()) if o_meaning and v_meaning and len(o_meaning) <= 30 and o_meaning == v_meaning: # Verify pre-hyphen parts are not English (avoid matching "self-realized") o_pre_words = re.findall(r'[a-zA-Z]{3,}', o_syn_str[:o_last_hyp]) v_pre_words = re.findall(r'[a-zA-Z]{3,}', v_syn_str[:v_last_hyp]) o_pre_eng = any(_is_english_word(w.lower()) for w in o_pre_words) v_pre_eng = any(_is_english_word(w.lower()) for w in v_pre_words) if not o_pre_eng and not v_pre_eng: # Guard: if pre-hyphen contains "; " (multiple synonym entries), # verify the pre-hyphen parts are similar (OCR of same words, # not different Sanskrit terms like "sarsti; sarupya" vs "samipya; sarsti") o_pre = o_syn_str[:o_last_hyp] v_pre = v_syn_str[:v_last_hyp] if '; ' in o_pre or '; ' in v_pre: o_pre_alpha = re.sub(r'[^a-z]', '', strip_diacritics(o_pre.lower())) v_pre_alpha = re.sub(r'[^a-z]', '', strip_diacritics(v_pre.lower())) pre_ratio = difflib.SequenceMatcher(None, o_pre_alpha, v_pre_alpha).ratio() if pre_ratio < 0.4: pass # Pre-hyphen parts too different — likely real edit else: return True else: return True # Section marker: [Canto X, Ch. Y ... if re.search(r'\[Canto\s+\d', orig): o_no_canto = re.sub(r'\[Canto\s+\d+,?\s*Ch\.\s*\d+\s*', '', orig).strip() if not o_no_canto: return True nc_alpha = re.sub(r'[^a-z]', '', strip_diacritics(o_no_canto.lower())) if nc_alpha and nc_alpha == v_alpha: return True if nc_alpha and v_alpha: canto_ratio = difflib.SequenceMatcher(None, nc_alpha, v_alpha).ratio() if canto_ratio >= 0.55: return True # Standalone numbers if re.match(r'^[\d\s.]+$', orig.strip()) or re.match(r'^[\d\s.]+$', veda.strip()): return True # --- KEY FILTER: count English words on each side --- o_eng = count_english_words(orig) v_eng = count_english_words(veda) # If NEITHER side has English words, it's Sanskrit/transliteration — noise if o_eng == 0 and v_eng == 0: return True # Non-English OCR garble: short English function words ("of", "is", "no", "to") # can appear coincidentally in OCR'd Sanskrit/Bengali text, inflating eng count. # Use substantive English words (≥3 chars) for a stricter check. # Guards: require both sides have ≥3 alpha words (avoids short real edits), # both eng ≤1 (avoids text with multiple English words), and not substring. o_subst_eng = sum(1 for w in re.findall(r'[a-zA-Z]{3,}', strip_diacritics(orig.lower())) if _is_english_word(w)) v_subst_eng = sum(1 for w in re.findall(r'[a-zA-Z]{3,}', strip_diacritics(veda.lower())) if _is_english_word(w)) o_alpha_words = re.findall(r'[a-zA-Z]{2,}', orig) v_alpha_words_all = re.findall(r'[a-zA-Z]{2,}', veda) if (o_subst_eng == 0 and v_subst_eng == 0 and o_eng <= 1 and v_eng <= 1 and len(o_alpha_words) >= 3 and len(v_alpha_words_all) >= 3): shorter_a, longer_a = (o_alpha, v_alpha) if len(o_alpha) <= len(v_alpha) else (v_alpha, o_alpha) if shorter_a not in longer_a: ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() if ratio >= 0.35: return True # Single-word OCR garble: original has embedded special chars in a single word, # vedabase is a clean single word (e.g., "piiqo(l(/is" vs "pasandis") o_stripped_punc = orig.strip().rstrip(',.') v_stripped_punc = veda.strip().rstrip(',.') if ' ' not in o_stripped_punc and ' ' not in v_stripped_punc: if re.search(r'[a-zA-Z][^a-zA-Z\-\'][a-zA-Z]', o_stripped_punc): if re.match(r'^[a-zA-Z\-]+$', v_stripped_punc): # Guard: synonym entries with different meanings are real edits # e.g., "jiita{l-be" vs "jatah-is" (meaning changed from "be" to "is") o_hyp = o_stripped_punc.rfind('-') v_hyp = v_stripped_punc.rfind('-') if o_hyp > 0 and v_hyp > 0: o_m = o_stripped_punc[o_hyp+1:].lower() v_m = v_stripped_punc[v_hyp+1:].lower() if o_m != v_m and _is_english_word(o_m) and _is_english_word(v_m): pass # Different English meanings = real edit else: return True else: return True # Control characters in original = severe OCR corruption # But only if they're embedded in the text (not just leading/trailing) orig_stripped_ctrl = orig.strip() while orig_stripped_ctrl and ord(orig_stripped_ctrl[0]) < 32: orig_stripped_ctrl = orig_stripped_ctrl[1:] while orig_stripped_ctrl and ord(orig_stripped_ctrl[-1]) < 32: orig_stripped_ctrl = orig_stripped_ctrl[:-1] if any(ord(c) < 32 and c not in '\t\n\r' for c in orig_stripped_ctrl): return True # Digit-letter mixing in original tokens with no English on vedabase side # e.g., "3to 3to po{luyo" (Bengali garble with digits) if v_eng == 0: has_digit_letter = any( re.search(r'\d[a-zA-Z]|[a-zA-Z]\d', token.strip(',.:;\'"')) for token in orig.split() ) if has_digit_letter: ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() if ratio >= 0.35: return True # If both sides have very few English words (1) and it's short, likely noise if o_eng <= 1 and v_eng <= 1 and len(orig.split()) <= 3 and len(veda.split()) <= 3: # Check if it's an OCR version of the same word if looks_like_ocr_of_same_word(orig, veda): return True # Consecutive uppercase in middle of text = severe OCR garble # E.g., "proI'IIHH it!" for "process is" alpha_only = re.sub(r'[^a-zA-Z]', '', orig) if re.search(r'[a-z][A-Z]{2,}', alpha_only): return True # Single hyphenated Sanskrit terms: OCR garble of same compound word # e.g., "upiisanii-kiirzl)a" vs "upasana-kanda" if v_eng == 0 and len(orig.split()) == 1 and len(veda.split()) == 1: ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() if ratio >= 0.4: return True # OCR-garbled Sanskrit name? (Vedabase has clean text, original is garbled) if is_ocr_of_sanskrit_name(orig, veda): return True # Single/short: if OCR can explain the difference, it's noise if len(orig.split()) <= 3 and len(veda.split()) <= 3: if looks_like_ocr_of_same_word(orig, veda): return True # If neither has English words, it's Sanskrit names/terms if o_eng == 0 and v_eng == 0: return True # If original has NO English words but vedabase does, AND the original # looks like OCR garble, it's severe OCR noise if o_eng == 0 and v_eng >= 1: alpha_orig = re.sub(r'[^a-zA-Z]', '', orig).lower() if alpha_orig: vowels = sum(1 for c in alpha_orig if c in 'aeiou') vowel_ratio = vowels / len(alpha_orig) # Real English has ~40% vowels; OCR garble often has less if vowel_ratio < 0.25: return True # High ratio of repeated chars (e.g., "iiillrtl") = garble unique_ratio = len(set(alpha_orig)) / len(alpha_orig) if unique_ratio < 0.5: return True # Very low similarity to vedabase = garble not real substitute ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() if ratio < 0.35: return True # Multi-word but no English on either side — Sanskrit quote if o_eng == 0 and v_eng == 0: return True # Both sides are Sanskrit transliteration if is_sanskrit(orig) and is_sanskrit(veda): return True # Contains hyphenated Sanskrit compounds and no English o_hyphens = len(re.findall(r'\w+-\w+', orig)) v_hyphens = len(re.findall(r'\w+-\w+', veda)) if (o_hyphens >= 2 or v_hyphens >= 2) and o_eng == 0 and v_eng == 0: return True # --- BROAD OCR CHECK for any length --- # Compare full alpha-only strings. If SequenceMatcher shows high similarity, # it's the same text garbled by OCR (handles mixed Sanskrit+English). if len(o_alpha) >= 4 and len(v_alpha) >= 4: ratio = difflib.SequenceMatcher(None, o_alpha, v_alpha).ratio() # For very short text, OCR garble is more severe so use lower threshold max_alpha = max(len(o_alpha), len(v_alpha)) if max_alpha <= 6: threshold = 0.4 elif max_alpha <= 15 and o_eng == 0 and v_eng >= 1: # Short phrase with no English in original = likely garble threshold = 0.45 else: threshold = 0.6 if ratio >= threshold: return True # --- PAGE-BOUNDARY TRUNCATION --- # Original is a tiny fragment (OCR page break) while Vedabase has full text. o_words_clean = re.findall(r'[a-zA-Z]{2,}', orig) v_words_clean = re.findall(r'[a-zA-Z]{2,}', veda) if len(o_words_clean) <= 5 and len(v_words_clean) >= 5: # Check word overlap first: if original words appear in vedabase, # it's likely a real edit (text added), not page boundary noise o_word_set = set(re.sub(r'[^a-z\s]', '', strip_diacritics(orig.lower())).split()) v_word_set = set(re.sub(r'[^a-z\s]', '', strip_diacritics(veda.lower())).split()) o_word_set.discard('') v_word_set.discard('') overlap = o_word_set & v_word_set has_word_overlap = overlap and len(overlap) / max(len(o_word_set), 1) >= 0.5 if not has_word_overlap: non_alpha_ratio = 1 - len(o_alpha) / max(len(orig.strip()), 1) ends_with_hyphen = orig.rstrip().endswith('-') has_garbage = non_alpha_ratio > 0.2 or ends_with_hyphen if has_garbage: return True if len(o_words_clean) <= 3: return True # --- OCR GARBAGE in original --- # If original has non-ASCII bytes or very high non-alpha ratio, it's OCR noise has_non_ascii = any(ord(c) > 127 and ord(c) < 256 for c in orig) if has_non_ascii: non_alpha_ratio = 1 - len(o_alpha) / max(len(orig.strip()), 1) if non_alpha_ratio > 0.25: return True # Even with lower ratio, if it's a short fragment vs long vedabase, truncation if len(o_words_clean) <= 8 and len(v_words_clean) >= 10: return True # --- PAGE HEADERS and CHAPTER ENDINGS --- # "Bhagavad-gita As It Is" is a page header in the original scanned book if re.search(r'Bhagavad.{0,3}g[iy]t[aiiA]?\s+As\s+It\s+Is', orig): return True # "Thus end the Bhaktivedanta Purports to the Nth Chapter" # Also catches fragments like "s to the Fourteenth" from OCR splitting if re.search(r'(Thus\s+end|Purports?\s+to\s+the|to\s+the\s+(First|Second|Third|Fourth|Fifth|Sixth|Seventh|Eighth|Ninth|Tenth|Eleventh|Twelfth|Thirteenth|Fourteenth|Fifteenth|Sixteenth|Seventeenth|Eighteenth))', orig): return True # Chapter/section markers like "[Ch. 4" or "TEXT 10" in middle of diff if re.search(r'\[Ch\.\s*\d', orig): return True # --- RUNNING HEADER CONTAMINATION --- # Scanned pages have running headers (book/chapter titles) that OCR embeds # in the middle of words, e.g. "inChoosing Human and Animal Lives fallible" # -> "infallible". Strip title-like phrases from original and check match. title_pattern = re.compile( r'(?:\d+\s+)?' # optional page number r'(?:Raja-vidya:\s+)?' # optional book prefix r'(?:The\s+)?' # optional "The" r'(?:[A-Z][a-z]{2,}(?:\s+(?:of|to|the|in|at|and|from|for|is|as|by|on|' r'[A-Z][a-z]{2,}|Kr[^\s]*a))+)' # title-like phrase (min 3 char words) ) # Also try stripping known book-title-like phrases with colons # E.g., "Consciousness: the Topmost Yoga System" embedded in text book_title_pattern = re.compile( r'(?:Kr[^\s]*a\s+)?Consciousness:?\s+(?:the\s+)?(?:[A-Z][a-z]+\s+)*[A-Z][a-z]+' ) for pat in [title_pattern, book_title_pattern]: m = pat.search(orig) if m: # Remove the title and rejoin the fragments cleaned = orig[:m.start()] + orig[m.end():] cleaned = re.sub(r'\s+', '', cleaned) # join word fragments v_compact = re.sub(r'\s+', '', veda) c_alpha = re.sub(r'[^a-z]', '', strip_diacritics(cleaned.lower())) v_alpha2 = re.sub(r'[^a-z]', '', strip_diacritics(v_compact.lower())) if c_alpha and v_alpha2: ratio = difflib.SequenceMatcher(None, c_alpha, v_alpha2).ratio() if ratio >= 0.6: return True # Vedabase section markers like "9.10 7 / Salvation of Trnavarta" (KRSNA Book style) if re.search(r'\d+\.\d+\s+\d+\s*/\s*\w', veda): v_clean = re.sub(r'\d+\.\d+\s+\d+\s*/\s*.+', '', veda).strip() if not v_clean: return True o_alpha_clean = re.sub(r'[^a-z]', '', strip_diacritics(orig.lower())) v_alpha_clean = re.sub(r'[^a-z]', '', strip_diacritics(v_clean.lower())) if o_alpha_clean and v_alpha_clean: ratio = difflib.SequenceMatcher(None, o_alpha_clean, v_alpha_clean).ratio() if ratio >= 0.5: return True elif not v_alpha_clean: return True # Vedabase section markers like "24.4 Introduction" or "28.5 Chapter Two" # or with page number prefix "14200 28.4 Chapter One" # Chapter names may be hyphenated like "Thirty-seven" if re.search(r'(?:\d+\s+)?\d+\.\d+\s+(Chapter\s+[\w-]+|Introduction|Preface|Conclusion|Appendix)', veda): v_clean = re.sub(r'(?:\d+\s+)?\d+\.\d+\s+(?:Chapter\s+[\w-]+(?:\s+[\w-]+){0,6}|Introduction|Preface|Conclusion|Appendix)', '', veda).strip() if not v_clean: return True o_alpha_clean = re.sub(r'[^a-z]', '', strip_diacritics(orig.lower())) v_alpha_clean = re.sub(r'[^a-z]', '', strip_diacritics(v_clean.lower())) if o_alpha_clean and v_alpha_clean: ratio = difflib.SequenceMatcher(None, o_alpha_clean, v_alpha_clean).ratio() if ratio >= 0.5: return True elif not v_alpha_clean: return True # Front matter: publisher info, addresses, book listings front_matter_phrases = [ r'International Society', r'Watseka\s+Aven', r'Readers?\s+interested\s+in', r'Secretary', r'Founder-Ac[aā]rya\s+of\s+the', r'Back to Godhead Magazine', r'invited\s+by\s+the\s+International', r'Concluding\s+Words', r'Glossary\b.*\bIndex\b', r'Table\s+of\s+Contents', ] for phrase in front_matter_phrases: if re.search(phrase, orig, re.IGNORECASE): return True # Vedabase chapter markers like "17.4 Chapter Three" or "3.6 Chapter Five" if re.search(r'\d+\.\d+\s+Chapter\s+(One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten|\w+)', veda): # Strip the chapter marker and check if remaining text matches v_clean = re.sub(r'\d+\.\d+\s+Chapter\s+\w+(\s+\w+){0,6}', '', veda).strip() if not v_clean: return True o_alpha_clean = re.sub(r'[^a-z]', '', strip_diacritics(orig.lower())) v_alpha_clean = re.sub(r'[^a-z]', '', strip_diacritics(v_clean.lower())) if o_alpha_clean and v_alpha_clean: ratio = difflib.SequenceMatcher(None, o_alpha_clean, v_alpha_clean).ratio() if ratio >= 0.5: return True # If remaining vedabase is just a Sanskrit word, use OCR check v_words = v_clean.split() if len(v_words) <= 2 and all( strip_diacritics(w.strip('.,\'"()').lower()) in SANSKRIT_WORDS or len(w.strip('.,\'"()')) <= 3 for w in v_words if w.strip('.,\'"()') ): return True elif not v_alpha_clean: return True # Vedabase book-abbreviation section markers like "NoD 11:" or "SSR 7:" if re.search(r'\b(NoD|SSR|KB|TLC|TLK|BBD|POY|EKC|PQPA|KCTY|TTP|EJOP)\s+\d+\s*:', veda): v_clean = re.sub(r'\b(NoD|SSR|KB|TLC|TLK|BBD|POY|EKC|PQPA|KCTY|TTP|EJOP)\s+\d+\s*:\s*[^A-Z]*', '', veda).strip() if not v_clean: return True o_alpha_clean = re.sub(r'[^a-z]', '', strip_diacritics(orig.lower())) v_alpha_clean = re.sub(r'[^a-z]', '', strip_diacritics(v_clean.lower())) if o_alpha_clean and v_alpha_clean: ratio = difflib.SequenceMatcher(None, o_alpha_clean, v_alpha_clean).ratio() if ratio >= 0.5: return True elif not v_alpha_clean: return True # Reverse page boundary: vedabase is tiny fragment, original is longer if len(v_words_clean) <= 3 and len(o_words_clean) >= 5: return True return False def compare_texts(orig: str, veda: str, verse: str, chapter: int, section: str, letters_mode: bool = False) -> list[Difference]: orig_n = normalize_for_comparison(orig, letters_mode=letters_mode) veda_n = normalize_for_comparison(veda, letters_mode=letters_mode) orig_words = orig_n.split() veda_words = veda_n.split() if not orig_words or not veda_words: return [] sm = difflib.SequenceMatcher(None, orig_words, veda_words) diffs = [] for tag, i1, i2, j1, j2 in sm.get_opcodes(): if tag == 'equal': continue orig_chunk = ' '.join(orig_words[i1:i2]) veda_chunk = ' '.join(veda_words[j1:j2]) if is_noise(orig_chunk, veda_chunk, letters_mode=letters_mode): continue ctx_before = ' '.join(orig_words[max(0, i1 - 40):i1]) ctx_after = ' '.join(orig_words[i2:i2 + 40]) diffs.append(Difference( verse=verse, chapter=chapter, section=section, diff_type=tag, original=orig_chunk, vedabase=veda_chunk, context_before=ctx_before, context_after=ctx_after, )) return diffs # ============================================================ # HTML REPORT # ============================================================ def generate_html_report(book_name: str, all_diffs: list[Difference], stats: dict, output_path: Path): html = f""" Text Comparison: {book_name}

{book_name} — Text Comparison Report

Original scanned edition vs Vedabase digital text.
Filtered: diacritics, transliteration, OCR artifacts, and page numbers excluded.

Summary

Chapters analyzed{stats.get('chapters_compared', 0)}
Verses compared{stats.get('verses_compared', 0)}
Verses with differences{stats.get('verses_with_diffs', 0)}
Total editorial differences{stats.get('total_diffs', 0)}
In translations{stats.get('translation_diffs', 0)}
In purports{stats.get('purport_diffs', 0)}
""" current_chapter = None for diff in all_diffs: if diff.chapter != current_chapter: current_chapter = diff.chapter title = BG_CHAPTER_TITLES.get(current_chapter, "") ch_diffs = sum(1 for d in all_diffs if d.chapter == current_chapter) html += f'

Chapter {current_chapter}: {title}

\n' html += f'
{ch_diffs} differences in this chapter
\n' html += f"""
{diff.chapter}.{diff.verse} {diff.section} {diff.diff_type}
{diff.context_before}""" if diff.original: html += f' {diff.original}' if diff.vedabase: html += f' {diff.vedabase}' html += f' {diff.context_after}
\n' html += """

Generated by Vedabase Text Comparison Pipeline v2.
Legend: struck-through red = original 1972 text, green = Vedabase text.

""" output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(html, encoding='utf-8') print(f"Report saved to: {output_path}") # ============================================================ # MAIN PIPELINE # ============================================================ def compare_book(book_key: str): book = BOOKS[book_key] print(f"\n{'='*60}") print(f"Comparing: {book['name']}") print(f"{'='*60}") book_type = book.get("type", "verse_by_chapter") if book_type == "mantra": return compare_book_mantra(book_key) if book_type == "text_flat": return compare_book_flat(book_key, split_into_text_words) if book_type == "chapter_prose": return compare_book_prose(book_key) if book_type == "multi_volume_prose": return compare_book_multi_volume(book_key) orig_path = resolve_orig(book["original"]) # === CHAPTER-BASED COMPARISON (BG etc.) === # 1. Find chapter pages in both PDFs print("Finding chapters in original...") orig_ch_pages = find_chapter_pages_original(orig_path) print(f" Found: {sorted(orig_ch_pages.keys())}") for ch, (s, e) in sorted(orig_ch_pages.items()): print(f" Ch.{ch}: pages {s}-{e}") print("Finding chapters in Vedabase...") veda_ch_pages = find_chapter_pages_vedabase( str(VEDABASE_PDF), book['vedabase_start'], book['vedabase_end'] ) print(f" Found: {sorted(veda_ch_pages.keys())}") for ch, (s, e) in sorted(veda_ch_pages.items()): print(f" Ch.{ch}: pages {s}-{e}") # 2. Compare chapter by chapter all_diffs = [] total_verses = 0 total_with_diffs = 0 chapters_compared = 0 for ch_num in sorted(orig_ch_pages.keys()): if ch_num not in veda_ch_pages: print(f" WARNING: Chapter {ch_num} not found in Vedabase") continue chapters_compared += 1 o_start, o_end = orig_ch_pages[ch_num] v_start, v_end = veda_ch_pages[ch_num] orig_text = extract_chapter_text(orig_path, o_start, o_end) veda_text = extract_chapter_text(str(VEDABASE_PDF), v_start, v_end) orig_verses = split_chapter_into_verses(orig_text) veda_verses = split_chapter_into_verses(veda_text) # Build full chapter text for cross-reference (misalignment detection) full_veda_chapter = normalize_for_comparison(veda_text) full_orig_chapter = normalize_for_comparison(orig_text) matched = 0 ch_diffs = 0 for v_num, orig_v in orig_verses.items(): veda_v = veda_verses.get(v_num) if not veda_v: continue matched += 1 total_verses += 1 verse_diffs = [] # Before comparing, check if this verse is misaligned. # Compare the matched-number verse texts. If they're very different, # check if the original verse matches a DIFFERENT vedabase verse better. orig_full = (orig_v.get('translation', '') + ' ' + orig_v.get('purport', '')).strip() veda_full = (veda_v.get('translation', '') + ' ' + veda_v.get('purport', '')).strip() orig_fn = normalize_for_comparison(orig_full) veda_fn = normalize_for_comparison(veda_full) if orig_fn and veda_fn: direct_ratio = difflib.SequenceMatcher( None, orig_fn.split()[:50], veda_fn.split()[:50]).ratio() if direct_ratio < 0.4: # Very different — check if orig matches another vedabase verse best_match = direct_ratio for other_vnum, other_vv in veda_verses.items(): if other_vnum == v_num: continue other_full = (other_vv.get('translation', '') + ' ' + other_vv.get('purport', '')).strip() other_fn = normalize_for_comparison(other_full) if other_fn: r = difflib.SequenceMatcher( None, orig_fn.split()[:50], other_fn.split()[:50]).ratio() if r > best_match: best_match = r if best_match > direct_ratio: # Original matches a different verse better → misalignment continue # Skip this verse entirely if orig_v['translation'] and veda_v['translation']: diffs = compare_texts( orig_v['translation'], veda_v['translation'], v_num, ch_num, "translation" ) verse_diffs.extend(diffs) if orig_v['purport'] and veda_v['purport']: diffs = compare_texts( orig_v['purport'], veda_v['purport'], v_num, ch_num, "purport" ) verse_diffs.extend(diffs) # Cross-reference check: filter out misalignment false positives. # If original diff text appears in full Vedabase chapter (or vice versa), # it's a verse boundary misalignment, not a real editorial change. real_diffs = [] for d in verse_diffs: # Extract English-only words (3+ chars) for cross-ref search # This avoids OCR garble and Sanskrit mismatch issues o_eng_words = [w for w in re.findall(r'[a-zA-Z]{3,}', strip_diacritics(d.original.lower())) if w in COMMON_ENGLISH or len(w) >= 5] v_eng_words = [w for w in re.findall(r'[a-zA-Z]{3,}', strip_diacritics(d.vedabase.lower())) if w in COMMON_ENGLISH or len(w) >= 5] o_found = False v_found = False # Search for a sequence of English words in the other chapter if len(o_eng_words) >= 3: snippet = ' '.join(o_eng_words[:6]) if snippet in full_veda_chapter: o_found = True else: # Try partial: any consecutive 3-word sequence for i in range(len(o_eng_words) - 2): sub = ' '.join(o_eng_words[i:i+3]) if len(sub) >= 12 and sub in full_veda_chapter: o_found = True break if len(v_eng_words) >= 3: snippet = ' '.join(v_eng_words[:6]) if snippet in full_orig_chapter: v_found = True else: for i in range(len(v_eng_words) - 2): sub = ' '.join(v_eng_words[i:i+3]) if len(sub) >= 12 and sub in full_orig_chapter: v_found = True break if o_found or v_found: continue # Misalignment — text exists in both, just at different verses real_diffs.append(d) if real_diffs: total_with_diffs += 1 ch_diffs += len(real_diffs) all_diffs.extend(real_diffs) print(f" Ch.{ch_num}: {len(orig_verses)} orig / {len(veda_verses)} veda verses, {matched} matched, {ch_diffs} diffs") # 3. Stats and reports stats = { 'chapters_compared': chapters_compared, 'verses_compared': total_verses, 'verses_with_diffs': total_with_diffs, 'total_diffs': len(all_diffs), 'translation_diffs': sum(1 for d in all_diffs if d.section == 'translation'), 'purport_diffs': sum(1 for d in all_diffs if d.section == 'purport'), } print(f"\n{'='*40}") print(f"RESULTS:") for k, v in stats.items(): print(f" {k}: {v}") print(f"{'='*40}") report_path = OUTPUT_DIR / f"{book_key}_comparison.html" generate_html_report(book['name'], all_diffs, stats, report_path) json_path = OUTPUT_DIR / f"{book_key}_diffs.json" json_data = { 'book': book['name'], 'stats': stats, 'differences': [asdict(d) for d in all_diffs], } json_path.write_text(json.dumps(json_data, indent=2, ensure_ascii=False), encoding='utf-8') print(f"JSON saved to: {json_path}") return all_diffs, stats def compare_book_flat(book_key: str, splitter_func): """Compare a single-level book with TEXT/MANTRA markers, no chapters.""" book = BOOKS[book_key] orig_path = resolve_orig(book["original"]) print("Extracting original text...") orig_text = extract_chapter_text( orig_path, book['orig_start_page'], book['orig_end_page'], strip_headers=True ) print("Extracting Vedabase text...") veda_text = extract_chapter_text( str(VEDABASE_PDF), book['vedabase_start'], book['vedabase_end'] ) print("Splitting into sections...") orig_sections = splitter_func(orig_text) veda_sections = splitter_func(veda_text) print(f" Original: {len(orig_sections)} sections") print(f" Vedabase: {len(veda_sections)} sections") full_veda = normalize_for_comparison(veda_text) full_orig = normalize_for_comparison(orig_text) all_diffs = [] total_verses = 0 total_with_diffs = 0 for s_num in sorted(orig_sections.keys(), key=lambda x: int(x.split('-')[0])): veda_s = veda_sections.get(s_num) if not veda_s: print(f" WARNING: Section {s_num} not found in Vedabase") continue orig_s = orig_sections[s_num] total_verses += 1 verse_diffs = [] # Misalignment check orig_full = (orig_s.get('translation', '') + ' ' + orig_s.get('purport', '')).strip() veda_full = (veda_s.get('translation', '') + ' ' + veda_s.get('purport', '')).strip() orig_fn = normalize_for_comparison(orig_full) veda_fn = normalize_for_comparison(veda_full) if orig_fn and veda_fn: direct_ratio = difflib.SequenceMatcher( None, orig_fn.split()[:50], veda_fn.split()[:50]).ratio() if direct_ratio < 0.4: best_match = direct_ratio for other_num, other_s in veda_sections.items(): if other_num == s_num: continue other_full = (other_s.get('translation', '') + ' ' + other_s.get('purport', '')).strip() other_fn = normalize_for_comparison(other_full) if other_fn: r = difflib.SequenceMatcher(None, orig_fn.split()[:50], other_fn.split()[:50]).ratio() if r > best_match: best_match = r if best_match > direct_ratio: print(f" Text {s_num}: SKIPPED (misalignment)") continue ch_num = 1 if orig_s['translation'] and veda_s['translation']: verse_diffs.extend(compare_texts(orig_s['translation'], veda_s['translation'], s_num, ch_num, "translation")) if orig_s['purport'] and veda_s['purport']: verse_diffs.extend(compare_texts(orig_s['purport'], veda_s['purport'], s_num, ch_num, "purport")) # Cross-reference check real_diffs = _cross_ref_filter(verse_diffs, full_veda, full_orig) n = len(real_diffs) if real_diffs: total_with_diffs += 1 all_diffs.extend(real_diffs) print(f" Text {s_num}: {n} diffs") return _finalize_report(book_key, book['name'], all_diffs, total_verses, total_with_diffs) def compare_book_prose(book_key: str): """Compare a chapter-based prose book (no verse markers within chapters). Compares full chapter text directly.""" book = BOOKS[book_key] orig_path = resolve_orig(book["original"]) letters_mode = book.get("noise_mode") == "letters" print("Extracting full original text...") orig_text = extract_chapter_text( orig_path, book['orig_start_page'], book['orig_end_page'], strip_headers=True ) print("Extracting full Vedabase text...") veda_text = extract_chapter_text( str(VEDABASE_PDF), book['vedabase_start'], book['vedabase_end'] ) # For prose books, just compare full text directly full_orig = normalize_for_comparison(orig_text, letters_mode=letters_mode) full_veda = normalize_for_comparison(veda_text, letters_mode=letters_mode) print("Comparing full text...") all_diffs = compare_texts(full_orig, full_veda, "full", 1, "prose", letters_mode=letters_mode) # Cross-reference filter (orig text in veda and vice versa) real_diffs = _cross_ref_filter(all_diffs, full_veda, full_orig) print(f" Raw diffs: {len(all_diffs)}, After cross-ref: {len(real_diffs)}") return _finalize_report(book_key, book['name'], real_diffs, 1, 1 if real_diffs else 0) def compare_book_multi_volume(book_key: str): """Compare a multi-volume prose book (e.g., KRSNA Book Vol.1+2).""" book = BOOKS[book_key] originals = book["original"] # list of filenames orig_ranges = book["orig_ranges"] # list of (start, end) tuples print("Extracting original text from multiple volumes...") orig_text = "" for filename, (start, end) in zip(originals, orig_ranges): path = resolve_orig(filename) vol_text = extract_chapter_text(path, start, end, strip_headers=True) orig_text += vol_text + "\n" print(f" {filename}: pages {start}-{end}") print("Extracting Vedabase text...") veda_text = extract_chapter_text( str(VEDABASE_PDF), book['vedabase_start'], book['vedabase_end'] ) full_orig = normalize_for_comparison(orig_text) full_veda = normalize_for_comparison(veda_text) print("Comparing full text...") all_diffs = compare_texts(full_orig, full_veda, "full", 1, "prose") real_diffs = _cross_ref_filter(all_diffs, full_veda, full_orig) print(f" Raw diffs: {len(all_diffs)}, After cross-ref: {len(real_diffs)}") return _finalize_report(book_key, book['name'], real_diffs, 1, 1 if real_diffs else 0) def _cross_ref_filter(verse_diffs, full_veda, full_orig): """Filter diffs by cross-referencing text in full chapter texts.""" real_diffs = [] for d in verse_diffs: o_eng_words = [w for w in re.findall(r'[a-zA-Z]{3,}', strip_diacritics(d.original.lower())) if w in COMMON_ENGLISH or len(w) >= 5] v_eng_words = [w for w in re.findall(r'[a-zA-Z]{3,}', strip_diacritics(d.vedabase.lower())) if w in COMMON_ENGLISH or len(w) >= 5] o_found = v_found = False if len(o_eng_words) >= 3: snippet = ' '.join(o_eng_words[:6]) if snippet in full_veda: o_found = True else: for i in range(len(o_eng_words) - 2): sub = ' '.join(o_eng_words[i:i+3]) if len(sub) >= 12 and sub in full_veda: o_found = True break if len(v_eng_words) >= 3: snippet = ' '.join(v_eng_words[:6]) if snippet in full_orig: v_found = True else: for i in range(len(v_eng_words) - 2): sub = ' '.join(v_eng_words[i:i+3]) if len(sub) >= 12 and sub in full_orig: v_found = True break if not (o_found or v_found): real_diffs.append(d) return real_diffs def _finalize_report(book_key, book_name, all_diffs, total_verses, total_with_diffs): """Generate stats, HTML report, and JSON output.""" stats = { 'verses_compared': total_verses, 'verses_with_diffs': total_with_diffs, 'total_diffs': len(all_diffs), 'translation_diffs': sum(1 for d in all_diffs if d.section == 'translation'), 'purport_diffs': sum(1 for d in all_diffs if d.section == 'purport'), } print(f"\n{'='*40}") print(f"RESULTS:") for k, v in stats.items(): print(f" {k}: {v}") print(f"{'='*40}") report_path = OUTPUT_DIR / f"{book_key}_comparison.html" generate_html_report(book_name, all_diffs, stats, report_path) json_path = OUTPUT_DIR / f"{book_key}_diffs.json" json_data = { 'book': book_name, 'stats': stats, 'differences': [asdict(d) for d in all_diffs], } json_path.write_text(json.dumps(json_data, indent=2, ensure_ascii=False), encoding='utf-8') print(f"JSON saved to: {json_path}") return all_diffs, stats def compare_book_mantra(book_key: str): """Compare a single-level book (mantras, no chapters) like Sri Isopanisad.""" book = BOOKS[book_key] orig_path = resolve_orig(book["original"]) # Extract full text from both PDFs print("Extracting original text...") orig_text = extract_chapter_text( orig_path, book['orig_start_page'], book['orig_end_page'] ) print("Extracting Vedabase text...") veda_text = extract_chapter_text( str(VEDABASE_PDF), book['vedabase_start'], book['vedabase_end'] ) # Split into mantras print("Splitting into mantras...") orig_mantras = split_into_mantras(orig_text) veda_mantras = split_into_mantras(veda_text) print(f" Original: {len(orig_mantras)} mantras ({sorted(orig_mantras.keys(), key=int)})") print(f" Vedabase: {len(veda_mantras)} mantras ({sorted(veda_mantras.keys(), key=int)})") # Build full text for cross-reference full_veda = normalize_for_comparison(veda_text) full_orig = normalize_for_comparison(orig_text) all_diffs = [] total_verses = 0 total_with_diffs = 0 matched = 0 for m_num in sorted(orig_mantras.keys(), key=int): veda_m = veda_mantras.get(m_num) if not veda_m: print(f" WARNING: Mantra {m_num} not found in Vedabase") continue orig_m = orig_mantras[m_num] matched += 1 total_verses += 1 verse_diffs = [] # Misalignment check orig_full = (orig_m.get('translation', '') + ' ' + orig_m.get('purport', '')).strip() veda_full = (veda_m.get('translation', '') + ' ' + veda_m.get('purport', '')).strip() orig_fn = normalize_for_comparison(orig_full) veda_fn = normalize_for_comparison(veda_full) if orig_fn and veda_fn: direct_ratio = difflib.SequenceMatcher( None, orig_fn.split()[:50], veda_fn.split()[:50]).ratio() if direct_ratio < 0.4: best_match = direct_ratio for other_num, other_m in veda_mantras.items(): if other_num == m_num: continue other_full = (other_m.get('translation', '') + ' ' + other_m.get('purport', '')).strip() other_fn = normalize_for_comparison(other_full) if other_fn: r = difflib.SequenceMatcher( None, orig_fn.split()[:50], other_fn.split()[:50]).ratio() if r > best_match: best_match = r if best_match > direct_ratio: print(f" Mantra {m_num}: SKIPPED (misalignment detected)") continue ch_num = 1 # Single "chapter" for mantra books if orig_m['translation'] and veda_m['translation']: diffs = compare_texts( orig_m['translation'], veda_m['translation'], m_num, ch_num, "translation" ) verse_diffs.extend(diffs) if orig_m['purport'] and veda_m['purport']: diffs = compare_texts( orig_m['purport'], veda_m['purport'], m_num, ch_num, "purport" ) verse_diffs.extend(diffs) # Cross-reference check real_diffs = [] for d in verse_diffs: o_eng_words = [w for w in re.findall(r'[a-zA-Z]{3,}', strip_diacritics(d.original.lower())) if w in COMMON_ENGLISH or len(w) >= 5] v_eng_words = [w for w in re.findall(r'[a-zA-Z]{3,}', strip_diacritics(d.vedabase.lower())) if w in COMMON_ENGLISH or len(w) >= 5] o_found = False v_found = False if len(o_eng_words) >= 3: snippet = ' '.join(o_eng_words[:6]) if snippet in full_veda: o_found = True else: for i in range(len(o_eng_words) - 2): sub = ' '.join(o_eng_words[i:i+3]) if len(sub) >= 12 and sub in full_veda: o_found = True break if len(v_eng_words) >= 3: snippet = ' '.join(v_eng_words[:6]) if snippet in full_orig: v_found = True else: for i in range(len(v_eng_words) - 2): sub = ' '.join(v_eng_words[i:i+3]) if len(sub) >= 12 and sub in full_orig: v_found = True break if o_found or v_found: continue real_diffs.append(d) n_diffs = len(real_diffs) if real_diffs: total_with_diffs += 1 all_diffs.extend(real_diffs) print(f" Mantra {m_num}: {n_diffs} diffs") # Stats and reports stats = { 'mantras_compared': matched, 'verses_compared': total_verses, 'verses_with_diffs': total_with_diffs, 'total_diffs': len(all_diffs), 'translation_diffs': sum(1 for d in all_diffs if d.section == 'translation'), 'purport_diffs': sum(1 for d in all_diffs if d.section == 'purport'), } print(f"\n{'='*40}") print(f"RESULTS:") for k, v in stats.items(): print(f" {k}: {v}") print(f"{'='*40}") report_path = OUTPUT_DIR / f"{book_key}_comparison.html" generate_html_report(book['name'], all_diffs, stats, report_path) json_path = OUTPUT_DIR / f"{book_key}_diffs.json" json_data = { 'book': book['name'], 'stats': stats, 'differences': [asdict(d) for d in all_diffs], } json_path.write_text(json.dumps(json_data, indent=2, ensure_ascii=False), encoding='utf-8') print(f"JSON saved to: {json_path}") return all_diffs, stats if __name__ == '__main__': import sys book = sys.argv[1] if len(sys.argv) > 1 else 'bg' if book not in BOOKS: print(f"Available books: {', '.join(BOOKS.keys())}") sys.exit(1) compare_book(book)