#!/usr/bin/env python3
import asyncio
import json
import logging
import random
import urllib.parse
import sys
import signal
import re
from playwright.async_api import async_playwright
from sqlalchemy import text
from app.database import AsyncSessionLocal

# R2.3 - SENTINEL (Hardened & Obedient Edition)
logging.basicConfig(level=logging.INFO, format='%(asctime)s [R2.3-SENTINEL] %(message)s')
logger = logging.getLogger("R2.3")

# --- 1. SZŰRÉSEK ÉS TILTÓLISTÁK ---
# Csak olyan típusokat keresünk, amik nem utánfutók vagy munkagépek
JUNK_LIST = [
    'SARIS', 'ANSSEMS', 'HAPERT', 'HUMBAUR', 'EDUARD', 'IFOR WILLIAMS', 'FENDT', 
    'HOBBY', 'ADRIA', 'PEECON', 'JAKO', 'KAWECO', 'POTTINGER', 'BOCKMANN',
    'JOHN DEERE', 'CLAAS', 'IVECO', 'SCANIA', 'MAN', 'DAF', 'KNAUS', 'PÖSSL', 'HYMER', 'WESTFALIA'
]

# --- 2. FORDÍTÁSOK (DE/NL -> EN) ---
TRANSLATIONS = {
    "3ER REIHE": "3 Series", "5ER REIHE": "5 Series", "1ER REIHE": "1 Series", "7ER REIHE": "7 Series",
    "E-KLASSE": "E Class", "C-KLASSE": "C Class", "S-KLASSE": "S Class", "A-KLASSE": "A Class",
    "REIHE": "Series", "KLASSE": "Class", "BESTELWAGEN": "Van"
}

class RobotScout:
    def __init__(self):
        self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
        self.running = True

    def clean_name(self, make, model):
        """Standardizált angol név előállítása."""
        m = model.upper()
        for de, en in TRANSLATIONS.items():
            m = m.replace(de, en)
        # Márkanév duplázódás törlése (pl. VOLVO VOLVO V60 -> VOLVO V60)
        m = m.replace(make.upper(), "").strip()
        return f"{make} {m}"

    # --- COLUMN MAPPING for scraping ---
    COLUMN_MAPPING = {
        "horsepower": "power_kw",
        "engine displacement": "engine_capacity",
        "maximum torque": "torque_nm",
        "top speed": "max_speed",
        "curb weight": "curb_weight",
        "wheelbase": "wheelbase",
        "num. of seats": "seats"
    }

    def clean_number(self, val: str, key: str = "") -> int:
        if not val or val == "-": return 0
        try:
            if "hp" in val.lower() or "kw" in val.lower():
                kw_match = re.search(r'(\d+)\s*kw', val.lower())
                if kw_match: return int(kw_match.group(1))
            nums = re.findall(r'\d+', val.replace(' ', '').replace(',', '').replace('.', ''))
            return int(nums[0]) if nums else 0
        except: return 0

    async def get_car_links(self, page, make, model, year, use_year=True):
        """Minden autós link kigyűjtése fallback mechanizmussal retry logikával."""
        clean_model = self.clean_name(make, model)
        search_query = f"{clean_model} {year}" if use_year else clean_model
        url = f"https://www.ultimatespecs.com/index.php?q={urllib.parse.quote(search_query)}"
        
        logger.info(f"🔎 KERESÉS: {search_query}")
        
        async def _fetch_links():
            await page.goto(url, wait_until="domcontentloaded", timeout=25000)
            
            # 1. Ha direkt az adatlapon vagyunk
            if any(x in page.url for x in ['/car-specs/', '/motorcycles-specs/']):
                logger.info("🎯 Direkt találat!")
                return [{"name": await page.title(), "url": page.url}]

            # 2. Várakozás és linkek kigyűjtése
            await asyncio.sleep(2)
            variants = await page.evaluate("""
                () => {
                    let results = [];
                    document.querySelectorAll('a').forEach(a => {
                        let href = a.getAttribute('href') || '';
                        let text = a.innerText.trim();
                        // Csak technikai adatlapokat gyűjtünk, reklámokat/kategóriákat nem
                        if ((href.includes('/car-specs/') || href.includes('/motorcycles-specs/'))
                            && href.includes('.html') && text.length > 3) {
                            results.push({ name: text, url: href });
                        }
                    });
                    return results;
                }
            """)

            # 3. Fallback: Ha nincs találat évvel, próbálja év nélkül
            if not variants and use_year:
                logger.info("   ↳ Nincs találat évszámmal, próbálkozom évszám nélkül...")
                return await self.get_car_links(page, make, model, year, use_year=False)

            return variants
        
        try:
            variants = await self._retry_with_backoff(
                _fetch_links,
                max_attempts=3,
                base_delay=2,
                exception_message=f"❌ Hálózati hiba a(z) {url} oldalon"
            )
            return variants if variants is not None else []
        except Exception as e:
            logger.error(f"❌ Hálózati hiba (végleges): {str(e)[:50]}")
            return []

    async def _retry_with_backoff(self, func, max_attempts=3, base_delay=2,
                                  exception_message="Retry failed", retry_exceptions=True):
        """Helper function for retry logic with exponential backoff."""
        for attempt in range(max_attempts):
            try:
                return await func()
            except Exception as e:
                if attempt == max_attempts - 1:
                    logger.error(f"{exception_message} after {max_attempts} attempts: {str(e)[:100]}")
                    raise
                else:
                    delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
                    logger.warning(f"⚠️ Attempt {attempt + 1} failed: {str(e)[:50]}. Retrying in {delay:.1f}s...")
                    await asyncio.sleep(delay)
        return None

    async def scrape_car_details(self, page, url):
        """Scrape car specifications from a given Ultimate Specs URL with comprehensive data extraction and retry logic."""
        async def _scrape():
            await page.goto(url, wait_until="networkidle", timeout=30000)
            
            # Parsing all specification tables and sections
            full_specs = await page.evaluate("""
                () => {
                    let results = {};
                    
                    // 1. Collect all specification tables (existing logic)
                    document.querySelectorAll('table.table_specs, table.responsive').forEach(table => {
                        table.querySelectorAll('tr').forEach(row => {
                            let t = row.querySelector('.table_specs_title, .td_title, td:first-child');
                            let v = row.querySelector('.table_specs_value, .td_value, td:last-child');
                            if(t && v) {
                                let k = t.innerText.replace(':','').trim().toLowerCase();
                                let val = v.innerText.trim();
                                if(k && val && val !== "-") results[k] = val;
                            }
                        });
                    });
                    
                    // 2. Collect section headers and their content for additional technical data
                    // Look for h2, h3, h4 elements that might contain section titles
                    const sections = {};
                    const headers = document.querySelectorAll('h2, h3, h4, .section-title, .specs-header');
                    
                    headers.forEach(header => {
                        const title = header.innerText.trim();
                        if (title && title.length > 0) {
                            // Find the next table or div with specs after this header
                            let nextElement = header.nextElementSibling;
                            let sectionData = {};
                            
                            // Look for tables or lists in the next few siblings
                            for (let i = 0; i < 5 && nextElement; i++) {
                                if (nextElement.tagName === 'TABLE') {
                                    nextElement.querySelectorAll('tr').forEach(row => {
                                        let t = row.querySelector('td:first-child');
                                        let v = row.querySelector('td:last-child');
                                        if(t && v) {
                                            let k = t.innerText.replace(':','').trim().toLowerCase();
                                            let val = v.innerText.trim();
                                            if(k && val && val !== "-") {
                                                sectionData[k] = val;
                                                // Also add to main results with section prefix
                                                results[`${title.toLowerCase().replace(/ /g, '_')}_${k}`] = val;
                                            }
                                        }
                                    });
                                }
                                nextElement = nextElement.nextElementSibling;
                            }
                            
                            sections[title.toLowerCase().replace(/ /g, '_')] = sectionData;
                        }
                    });
                    
                    // 3. Extract specific known sections by looking for text patterns
                    const pageText = document.body.innerText.toLowerCase();
                    
                    // Check for electric/hybrid sections
                    if (pageText.includes('electric engine') || pageText.includes('battery')) {
                        // Try to find battery voltage, capacity, etc.
                        const batteryRegex = /battery\s*voltage[:\s]*([\d\.]+)\s*v/gi;
                        const match = batteryRegex.exec(document.body.innerText);
                        if (match) results['battery_voltage_v'] = match[1];
                    }
                    
                    // 4. Extract dimensions data
                    const dimensionPatterns = {
                        'wheelbase': /wheelbase[:\s]*([\d\.]+)\s*cm/gi,
                        'length': /length[:\s]*([\d\.]+)\s*cm/gi,
                        'width': /width[:\s]*([\d\.]+)\s*cm/gi,
                        'height': /height[:\s]*([\d\.]+)\s*cm/gi,
                        'curb_weight': /curb\s*weight[:\s]*([\d\.]+)\s*kg/gi,
                        'towing_capacity': /towing\s*capacity[:\s]*([\d\.]+)\s*kg/gi
                    };
                    
                    for (const [key, regex] of Object.entries(dimensionPatterns)) {
                        const match = regex.exec(document.body.innerText);
                        if (match) results[key] = match[1];
                    }
                    
                    // 5. Add sections data as a nested object
                    results['_sections'] = sections;
                    
                    return results;
                }
            """)
            return full_specs
        
        try:
            logger.info(f"🌐 Scraping: {url}")
            full_specs = await self._retry_with_backoff(
                _scrape,
                max_attempts=3,
                base_delay=2,
                exception_message=f"❌ Scrape hiba a(z) {url} oldalon"
            )
            return full_specs
        except Exception as e:
            logger.error(f"❌ Scrape hiba (végleges): {str(e)[:100]}...")
            return None

    async def run(self):
        async with async_playwright() as p:
            browser = await p.chromium.launch(headless=True)
            context = await browser.new_context(user_agent=self.user_agent)
            page = await context.new_page()

            while self.running:
                # --- A FÉK: 3-6 mp szigorú pihenő minden kör elején ---
                wait = random.uniform(3, 6)
                logger.info(f"💤 Várakozás {wait:.1f} mp...")
                await asyncio.sleep(wait)

                async with AsyncSessionLocal() as db:
                    # Következő feldolgozatlan autó (John Deere, Iveco, stb. kizárva)
                    target = (await db.execute(text("""
                        SELECT id, make, marketing_name, year_from FROM vehicle.vehicle_model_definitions 
                        WHERE status IN ('pending', 'manual_review_needed') 
                        AND NOT (make = ANY(:junks))
                        ORDER BY priority_score DESC LIMIT 1
                    """), {"junks": JUNK_LIST})).fetchone()

                    if not target:
                        logger.info("✨ Minden tétel feldolgozva.")
                        break

                    t_id, make, model, year = target
                    logger.info(f"🚀 CÉLPONT: {make} {model} ({year}) [ID: {t_id}]")

                    try:
                        links = await self.get_car_links(page, make, model, year)
                    except Exception as e:
                        logger.error(f"❌ Hálózati hiba linkek lekérésekor: {str(e)[:100]}")
                        await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_network' WHERE id=:id"), {"id": t_id})
                        await db.commit()
                        continue
                    
                    if not links:
                        logger.warning(f"❌ Nem található adatlap. research_failed_empty rögzítése.")
                        await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_empty' WHERE id=:id"), {"id": t_id})
                        await db.commit()
                        continue

                    # --- 1. SCRAPE THE FIRST LINK FOR IMMEDIATE ENRICHMENT ---
                    first_link = None
                    if links:
                        first_link = links[0]
                        full_url = first_link['url'] if first_link['url'].startswith('http') else f"https://www.ultimatespecs.com{first_link['url']}"
                        logger.info(f"⚡ Azonnali adatgyűjtés: {full_url}")
                        web_data = await self.scrape_car_details(page, full_url)
                        
                        if web_data is None:
                            # Scraping failed after all retries
                            logger.error(f"❌ Scraping sikertelen minden próbálkozás után. research_failed_parsing rögzítése.")
                            await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_parsing' WHERE id=:id"), {"id": t_id})
                            await db.commit()
                            # Continue to save links as variants anyway
                            web_data = {}
                        elif len(web_data) >= 5:
                            # Map scraped data to columns
                            updates = {col: self.clean_number(web_data.get(k)) for k, col in self.COLUMN_MAPPING.items()}
                            # Also extract fuel_type, transmission, etc. if possible
                            fuel_type = web_data.get('fuel type', 'Unknown')
                            transmission_type = web_data.get('transmission', 'Unknown')
                            drive_type = web_data.get('drive type', 'Unknown')
                            body_type = web_data.get('body type', 'Unknown')
                            engine_capacity = updates.get('engine_capacity', 0)
                            power_kw = updates.get('power_kw', 0)
                            
                            # Update the original record with scraped data
                            await db.execute(text("""
                                UPDATE vehicle.vehicle_model_definitions
                                SET power_kw = :power_kw, engine_capacity = :engine_capacity,
                                    torque_nm = :torque_nm, max_speed = :max_speed,
                                    curb_weight = :curb_weight,
                                    wheelbase = :wheelbase, seats = :seats,
                                    fuel_type = :fuel_type, transmission_type = :transmission_type,
                                    drive_type = :drive_type, body_type = :body_type,
                                    specifications = specifications || :full_json,
                                    status = 'awaiting_ai_synthesis', updated_at = NOW()
                                WHERE id = :id
                            """), {
                                **updates,
                                "id": t_id,
                                "fuel_type": fuel_type,
                                "transmission_type": transmission_type,
                                "drive_type": drive_type,
                                "body_type": body_type,
                                "full_json": json.dumps(web_data)
                            })
                            logger.info(f"✅ AZONNALI PUBLIKÁLÁS: {make} {model} ({power_kw} kW)")
                        else:
                            logger.warning("⚠️ Scraping kevés adatot talált, csak linkek mentve.")
                    
                    # --- 2. SAVE ALL LINKS AS NEW VARIANT RECORDS (including first if not enriched) ---
                    added = 0
                    for l in links:
                        full_url = l['url'] if l['url'].startswith('http') else f"https://www.ultimatespecs.com{l['url']}"
                        
                        # JAVÍTÁS: column "source_url" hiba ellen raw_api_data-t nézünk
                        check_query = text("SELECT id FROM vehicle.vehicle_model_definitions WHERE raw_api_data->>'url' = :u")
                        exists = (await db.execute(check_query, {"u": full_url})).fetchone()
                        
                        if not exists:
                            # Create normalized name from marketing name
                            normalized = l['name'].lower().replace(' ', '_').replace('-', '_').replace('.', '').replace(',', '')[:200]
                            
                            await db.execute(text("""
                                INSERT INTO vehicle.vehicle_model_definitions
                                (make, marketing_name, normalized_name, year_from, status,
                                 raw_api_data, priority_score, source, market,
                                 technical_code, variant_code, version_code,
                                 specifications, marketing_name_aliases, raw_search_context)
                                VALUES (:make, :name, :normalized, :year, 'awaiting_ai_synthesis',
                                        :raw, 30, 'ultimatespecs', 'EU',
                                        'UNKNOWN', 'UNKNOWN', 'UNKNOWN',
                                        '{}'::jsonb, '[]'::jsonb, '')
                            """), {
                                "make": make, "name": l['name'], "normalized": normalized,
                                "year": year, "raw": json.dumps({"url": full_url}), "priority": 30
                            })
                            added += 1
                    
                    # Eredeti rekord archiválása (ha még nem publikáltuk)
                    if not web_data:
                        await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='expanded_to_variants', updated_at=NOW() WHERE id=:id"), {"id": t_id})
                    
                    await db.commit()
                    logger.info(f"✅ SIKER: {added} új variáció mentve. R4-R5 robotok értesítve.")

            await browser.close()

if __name__ == "__main__":
    scout = RobotScout()
    # Handle CTRL+C
    def stop_signal(sig, frame):
        logger.info("🛑 LEÁLLÍTÁS (Kérés érzékelve)...")
        scout.running = False
        sys.exit(0)
    
    signal.signal(signal.SIGINT, stop_signal)
    
    try:
        asyncio.run(scout.run())
    except KeyboardInterrupt:
        pass