#!/usr/bin/env python3
"""Fetch all chapters of 'In Search of the Ultimate Goal of Life' from Vanisource."""

import urllib.request
import html
import re
import time
import os

BASE_URL = "https://vanisource.org/wiki/"

CHAPTERS = [
    ("Varna_and_Asrama_Jeopardized", "Varna and Asrama Jeopardized"),
    ("Mother_Nature%27s_Atom_Bomb", "Mother Nature's Atom Bomb"),
    ("The_Problem_Remains_the_Same", "The Problem Remains the Same"),
    ("Perverted_Castes", "Perverted Castes"),
    ("Real_Varna_and_Asrama", "Real Varna and Asrama"),
    ("Varna_and_Asrama_Rejected", "Varna and Asrama Rejected"),
    ("Hodge-podge_Impersonalism", "Hodge-podge Impersonalism"),
    ("Educated_Means_Atheist", "Educated Means Atheist"),
    ("Sankara_Astonished", "Sankara Astonished"),
    ("Absolute_Truth_is_Personal", "Absolute Truth is Personal"),
    ("No_Clear_Idea", "No Clear Idea"),
    ("Accept_Sannyasa", "Accept Sannyasa"),
    ("Go_Higher", "Go Higher"),
    ("Mixed_Devotion", "Mixed Devotion"),
    ("Knowledge-free_Devotion", "Knowledge-free Devotion"),
    ("Hear_From_Self-realized_Souls", "Hear From Self-realized Souls"),
    ("Spontaneous_Love_of_Godhead", "Spontaneous Love of Godhead"),
    ("Dasya-Prema", "Dasya-Prema"),
]


def strip_html_tags(text):
    """Remove HTML tags and decode entities."""
    # Replace <br> variants with newlines
    text = re.sub(r'<br\s*/?>', '\n', text)
    # Replace </p> and </div> with newlines
    text = re.sub(r'</p>', '\n', text)
    text = re.sub(r'</div>', '\n', text)
    # Replace <dd> (used for verse indentation)
    text = re.sub(r'<dd>', '    ', text)
    text = re.sub(r'</dd>', '\n', text)
    # Replace <dt>
    text = re.sub(r'<dt>', '', text)
    text = re.sub(r'</dt>', '\n', text)
    # Remove all remaining tags
    text = re.sub(r'<[^>]+>', '', text)
    # Decode HTML entities
    text = html.unescape(text)
    return text


def extract_content(page_html):
    """Extract the main article content from the wiki page HTML."""
    # Find the mw-parser-output div content
    match = re.search(r'<div class="mw-parser-output">(.*?)<div class="printfooter">',
                       page_html, re.DOTALL)
    if not match:
        # Try alternative end marker
        match = re.search(r'<div class="mw-parser-output">(.*?)<!--\s*NewPP',
                           page_html, re.DOTALL)
    if not match:
        match = re.search(r'<div class="mw-parser-output">(.*?)<div id="catlinks"',
                           page_html, re.DOTALL)
    if not match:
        print("  WARNING: Could not find main content area")
        return ""

    content_html = match.group(1)

    # Remove navigation boxes, edit links, TOC, etc.
    content_html = re.sub(r'<div id="toc".*?</div>\s*</div>', '', content_html, flags=re.DOTALL)
    content_html = re.sub(r'<span class="mw-editsection">.*?</span>', '', content_html, flags=re.DOTALL)
    content_html = re.sub(r'<div class="smw-factbox.*?</div>\s*</div>', '', content_html, flags=re.DOTALL)
    # Remove figure/image elements
    content_html = re.sub(r'<figure[^>]*>.*?</figure>', '', content_html, flags=re.DOTALL)
    # Remove style tags
    content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL)

    text = strip_html_tags(content_html)

    # Clean up whitespace
    # Replace multiple blank lines with double newline
    text = re.sub(r'\n\s*\n\s*\n+', '\n\n', text)
    # Strip leading/trailing whitespace from each line
    lines = [line.rstrip() for line in text.split('\n')]
    text = '\n'.join(lines)
    # Strip leading/trailing whitespace from the whole text
    text = text.strip()

    return text


def fetch_page(url_slug):
    """Fetch a wiki page and return the HTML content."""
    url = BASE_URL + url_slug
    print(f"  Fetching: {url}")
    req = urllib.request.Request(url, headers={
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
    })
    with urllib.request.urlopen(req, timeout=30) as response:
        return response.read().decode('utf-8')


def main():
    output_path = "/Users/jaganat/.emacs.d/git_projects/veda_pdf_scan/vedabase_texts/in_search_ultimate_goal.txt"

    # First, get the intro text from the main page
    print("Fetching main page for introduction...")
    main_html = fetch_page("In_Search_of_the_Ultimate_Goal_of_Life")

    # Extract intro from the main page (before the chapter links)
    match = re.search(r'<div class="mw-parser-output">(.*?)<div style="clear:both;">',
                       main_html, re.DOTALL)
    intro_text = ""
    if match:
        intro_html = match.group(1)
        # Remove images
        intro_html = re.sub(r'<figure[^>]*>.*?</figure>', '', intro_html, flags=re.DOTALL)
        intro_text = strip_html_tags(intro_html)
        intro_text = re.sub(r'\n\s*\n\s*\n+', '\n\n', intro_text)
        intro_text = intro_text.strip()

    all_text = []
    all_text.append("IN SEARCH OF THE ULTIMATE GOAL OF LIFE")
    all_text.append("by His Divine Grace A.C. Bhaktivedanta Swami Prabhupada")
    all_text.append("=" * 60)
    all_text.append("")

    if intro_text:
        all_text.append("INTRODUCTION")
        all_text.append("-" * 40)
        all_text.append(intro_text)
        all_text.append("")
        all_text.append("")

    # Fetch each chapter
    for i, (slug, title) in enumerate(CHAPTERS, 1):
        print(f"\nFetching chapter {i}/{len(CHAPTERS)}: {title}")
        try:
            page_html = fetch_page(slug)
            content = extract_content(page_html)

            if content:
                all_text.append(f"CHAPTER {i}: {title.upper()}")
                all_text.append("=" * 60)
                all_text.append("")
                all_text.append(content)
                all_text.append("")
                all_text.append("")
                print(f"  Got {len(content)} characters")
            else:
                all_text.append(f"CHAPTER {i}: {title.upper()}")
                all_text.append("=" * 60)
                all_text.append("")
                all_text.append("[Content could not be extracted]")
                all_text.append("")
                print(f"  WARNING: No content extracted!")

            # Be polite to the server
            time.sleep(1)

        except Exception as e:
            print(f"  ERROR: {e}")
            all_text.append(f"CHAPTER {i}: {title.upper()}")
            all_text.append("=" * 60)
            all_text.append("")
            all_text.append(f"[Error fetching chapter: {e}]")
            all_text.append("")

    # Write the combined text
    final_text = '\n'.join(all_text)
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    with open(output_path, 'w', encoding='utf-8') as f:
        f.write(final_text)

    print(f"\n{'=' * 60}")
    print(f"Done! Written {len(final_text)} characters to:")
    print(f"  {output_path}")
    print(f"Total chapters: {len(CHAPTERS)}")


if __name__ == "__main__":
    main()