# /opt/docker/dev/service_finder/backend/app/workers/vehicle/ultimatespecs/vehicle_ultimate_r0_spider.py
#!/usr/bin/env python3
"""
Worker: vehicle_ultimate_r0_spider
Producer-Consumer lánc első eleme. Kivesz egy autót a vehicle.vehicle_model_definitions táblából,
keres az UltimateSpecs oldalán, és a talált .html linkeket beszúrja a vehicle.auto_data_crawler_queue táblába.
"""

import asyncio
import logging
import random
import sys
import signal
import urllib.parse
from datetime import datetime
from typing import Optional, Dict, Any, List

from playwright.async_api import async_playwright, Page, Browser, BrowserContext
from sqlalchemy import text, select, and_, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession

from app.database import AsyncSessionLocal
from app.models.vehicle.external_reference_queue import ExternalReferenceQueue
from app.models.vehicle.vehicle_definitions import VehicleModelDefinition

# Logging konfiguráció
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [R0-SPIDER] %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S'
)
logger = logging.getLogger("R0-SPIDER")

# Konfiguráció
SLEEP_INTERVAL = random.uniform(1, 2)  # 1-2 mp között várakozás
MAX_RETRIES = 3
BASE_URL = "https://www.ultimatespecs.com/index.php?q={query}"


class UltimateSpecsSpider:
    def __init__(self):
        self.running = True
        self.playwright = None
        self.browser: Optional[Browser] = None
        self.context: Optional[BrowserContext] = None
        self.user_agent = (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
            "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
        )

    async def init_browser(self):
        """Playwright böngésző inicializálása"""
        try:
            self.playwright = await async_playwright().start()
            self.browser = await self.playwright.chromium.launch(
                headless=True,
                args=[
                    '--disable-blink-features=AutomationControlled',
                    '--disable-dev-shm-usage',
                    '--no-sandbox',
                ]
            )
            self.context = await self.browser.new_context(
                user_agent=self.user_agent,
                viewport={'width': 1920, 'height': 1080},
                java_script_enabled=True
            )
            logger.info("Playwright böngésző inicializálva")
        except Exception as e:
            logger.error(f"Hiba a böngésző inicializálásakor: {e}")
            raise

    async def close_browser(self):
        """Playwright böngésző lezárása"""
        if self.context:
            await self.context.close()
        if self.browser:
            await self.browser.close()
        if self.playwright:
            await self.playwright.stop()
        logger.info("Playwright böngésző lezárva")

    async def fetch_next_vehicle(self, session: AsyncSession) -> Optional[Dict[str, Any]]:
        """
        Kivesz egy feldolgozandó járművet a vehicle_model_definitions táblából.
        """
        query = text("""
            SELECT id, make, marketing_name, year_from, vehicle_class 
            FROM vehicle.vehicle_model_definitions 
            WHERE status IN ('pending', 'manual_review_needed') 
            AND vehicle_class IN ('car', 'motorcycle')
            ORDER BY priority_score DESC, updated_at ASC 
            LIMIT 1
            FOR UPDATE SKIP LOCKED
        """)
        
        try:
            result = await session.execute(query)
            row = result.fetchone()
            if row:
                return {
                    'id': row[0],
                    'make': row[1],
                    'marketing_name': row[2],
                    'year_from': row[3],
                    'vehicle_class': row[4]
                }
            return None
        except Exception as e:
            logger.error(f"Hiba a következő jármű lekérdezésekor: {e}")
            return None

    def build_search_query(self, make: str, marketing_name: str, year_from: Optional[int]) -> str:
        """
        Build search query for UltimateSpecs.
        """
        # Clean and prepare the query
        make_clean = make.lower().replace(' ', '-').replace('.', '')
        model_clean = marketing_name.lower().replace(' ', '-').replace('.', '')
        
        # Remove common suffixes
        for suffix in ['-', 'series', 'class', 'model']:
            if model_clean.endswith(suffix):
                model_clean = model_clean[:-len(suffix)].rstrip('-')
        
        query_parts = [make_clean, model_clean]
        if year_from:
            query_parts.append(str(year_from))
        
        return ' '.join(query_parts)

    async def extract_links_with_js(self, page: Page, make_url: str, model_word: str) -> List[Dict[str, str]]:
        """
        Extract .html links from the page using the provided JavaScript filter.
        """
        js_code = """
        (args) => {
            let targetMakeUrl = args.makeUrl; // pl. 'honda' vagy 'alfa-romeo'
            let targetModel = args.modelWord; // pl. 'civic'
            let specs = [];
            document.querySelectorAll('a').forEach(a => {
                let href = a.getAttribute('href') || '';
                let text = a.innerText.trim();
                let hrefLow = href.toLowerCase();
                let textLow = text.toLowerCase();
                if (hrefLow.includes('/car-specs/') || hrefLow.includes('/motorcycles-specs/')) {
                    // SZIGORÚ MÁRKA SZŰRŐ AZ URL-BEN (Reklámok ellen)
                    if (hrefLow.includes('/' + targetMakeUrl + '/') || hrefLow.includes(targetMakeUrl + '-models')) {
                        // MODELL SZŰRŐ A SZÖVEGBEN VAGY URL-BEN
                        if (targetModel === '' || textLow.includes(targetModel) || hrefLow.includes(targetModel)) {
                            if (hrefLow.endsWith('.html') && text.length > 1) {
                                specs.push({ name: text, url: href });
                            }
                        }
                    }
                }
            });
            return specs;
        }
        """
        
        try:
            # Prepare arguments for the JS function
            args = {
                'makeUrl': make_url.lower(),
                'modelWord': model_word.lower()
            }
            
            # Execute the JavaScript
            specs = await page.evaluate(js_code, args)
            return specs
        except Exception as e:
            logger.error(f"Hiba a JS szűrő futtatásakor: {e}")
            return []

    async def search_and_extract_links(self, vehicle: Dict[str, Any]) -> List[Dict[str, str]]:
        """
        Search on UltimateSpecs and extract links using two-step drill-down.
        """
        search_query = self.build_search_query(
            vehicle['make'], 
            vehicle['marketing_name'], 
            vehicle['year_from']
        )
        
        # Prepare make URL part
        make_url = vehicle['make'].lower().replace(' ', '-').replace('.', '')
        model_word = vehicle['marketing_name'].lower().split()[0] if vehicle['marketing_name'] else ''
        
        encoded_query = urllib.parse.quote(search_query)
        search_url = BASE_URL.format(query=encoded_query)
        
        logger.info(f"Keresés: {search_query} | URL: {search_url}")
        
        page = None
        try:
            page = await self.context.new_page()
            
            # 1. Step: Go to search page
            await page.goto(search_url, wait_until='networkidle', timeout=30000)
            
            # Check if we're on a category page or search results
            current_url = page.url
            
            # 2. Step: Extract links with JS filter
            all_links = await self.extract_links_with_js(page, make_url, model_word)
            
            # If no links found on first page, try to click on first result
            if not all_links and 'index.php' in current_url:
                # Try to find and click on first relevant link
                first_link = await page.query_selector('a[href*="/car-specs/"], a[href*="/motorcycles-specs/"]')
                if first_link:
                    await first_link.click()
                    await page.wait_for_load_state('networkidle')
                    
                    # Extract links from the new page
                    all_links = await self.extract_links_with_js(page, make_url, model_word)
            
            # Ensure URLs are absolute
            for link in all_links:
                if not link['url'].startswith('http'):
                    link['url'] = f"https://www.ultimatespecs.com{link['url']}"
            
            logger.info(f"{len(all_links)} link találva")
            return all_links
            
        except Exception as e:
            logger.error(f"Hiba a keresés során: {e}")
            return []
        finally:
            if page:
                await page.close()

    async def save_links_to_queue(self, session: AsyncSession, links: List[Dict[str, str]], 
                                 vehicle: Dict[str, Any]) -> int:
        """
        Save extracted links to the external reference queue.
        """
        saved_count = 0
        
        for link in links:
            try:
                # Check if URL already exists
                existing_query = select(ExternalReferenceQueue).where(
                    ExternalReferenceQueue.url == link['url']
                )
                existing_result = await session.execute(existing_query)
                if existing_result.scalar_one_or_none():
                    logger.debug(f"URL már létezik: {link['url']}")
                    continue
                
                # Create new queue entry
                queue_entry = ExternalReferenceQueue(
                    url=link['url'],
                    level='engine',
                    category=vehicle['vehicle_class'] or 'car',
                    name=link['name'][:255],
                    parent_id=vehicle['id'],
                    status='pending'
                )
                
                session.add(queue_entry)
                await session.commit()
                saved_count += 1
                logger.debug(f"URL mentve: {link['url']}")
                
            except IntegrityError:
                await session.rollback()
                logger.debug(f"URL már létezik (integrity): {link['url']}")
            except Exception as e:
                await session.rollback()
                logger.error(f"Hiba a URL mentésekor: {e}")
        
        return saved_count

    async def update_vehicle_status(self, session: AsyncSession, vehicle_id: int, 
                                   status: str, error_msg: str = None):
        """
        Update the vehicle's status in the database.
        """
        try:
            query = text("""
                UPDATE vehicle.vehicle_model_definitions 
                SET status = :status, 
                    last_error = :error_msg,
                    updated_at = NOW(),
                    attempts = attempts + 1
                WHERE id = :id
            """)
            
            await session.execute(
                query, 
                {'status': status, 'error_msg': error_msg, 'id': vehicle_id}
            )
            await session.commit()
            logger.info(f"Jármű státusz frissítve: {vehicle_id} -> {status}")
            
        except Exception as e:
            await session.rollback()
            logger.error(f"Hiba a státusz frissítésekor: {e}")

    async def process_single_vehicle(self):
        """
        Process a single vehicle: fetch, search, extract links, save to queue.
        """
        async with AsyncSessionLocal() as session:
            try:
                # 1. Fetch next vehicle
                vehicle = await self.fetch_next_vehicle(session)
                if not vehicle:
                    logger.info("Nincs feldolgozandó jármű")
                    return False
                
                logger.info(f"Feldolgozás: {vehicle['make']} {vehicle['marketing_name']} "
                          f"(ID: {vehicle['id']})")
                
                # 2. Search and extract links
                links = await self.search_and_extract_links(vehicle)
                
                if not links:
                    # No links found
                    await self.update_vehicle_status(
                        session, vehicle['id'], 
                        'research_failed_empty',
                        'No links found on UltimateSpecs'
                    )
                    logger.warning(f"Nem található link: {vehicle['make']} {vehicle['marketing_name']}")
                    return True
                
                # 3. Save links to queue
                saved_count = await self.save_links_to_queue(session, links, vehicle)
                
                # 4. Update vehicle status
                if saved_count > 0:
                    await self.update_vehicle_status(
                        session, vehicle['id'], 
                        'spider_dispatched',
                        f'{saved_count} links added to queue'
                    )
                    logger.info(f"{saved_count} link mentve a queue-ba")
                else:
                    # All links already existed
                    await self.update_vehicle_status(
                        session, vehicle['id'], 
                        'spider_dispatched',
                        'All links already in queue'
                    )
                    logger.info("Minden link már szerepel a queue-ban")
                
                return True
                
            except Exception as e:
                logger.error(f"Hiba a jármű feldolgozása során: {e}")
                # Try to update status with error
                try:
                    if 'vehicle' in locals():
                        await self.update_vehicle_status(
                            session, vehicle['id'],
                            'research_failed_network',
                            str(e)[:500]
                        )
                except:
                    pass
                return True

    async def run(self):
        """
        Main loop of the spider.
        """
        logger.info("UltimateSpecs R0 Spider indítása...")
        
        try:
            await self.init_browser()
            
            while self.running:
                try:
                    # Process a single vehicle
                    processed = await self.process_single_vehicle()
                    
                    if not processed:
                        # No vehicles to process, wait longer
                        await asyncio.sleep(SLEEP_INTERVAL * 2)
                    else:
                        # Wait before next iteration
                        await asyncio.sleep(SLEEP_INTERVAL)
                        
                except KeyboardInterrupt:
                    logger.info("Keyboard interrupt, leállítás...")
                    self.running = False
                    break
                except Exception as e:
                    logger.error(f"Hiba a fő ciklusban: {e}")
                    await asyncio.sleep(SLEEP_INTERVAL)
                    
        finally:
            await self.close_browser()
            logger.info("UltimateSpecs R0 Spider leállt")

    def stop(self):
        """Stop the spider gracefully."""
        self.running = False
        logger.info("Leállítás kérése érkezett")


async def main():
    """Main entry point."""
    spider = UltimateSpecsSpider()
    
    # Signal handling for graceful shutdown
    def signal_handler(signum, frame):
        logger.info(f"Signal {signum} received, stopping...")
        spider.stop()
    
    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)
    
    try:
        await spider.run()
    except Exception as e:
        logger.error(f"Váratlan hiba: {e}")
        sys.exit(1)


if __name__ == "__main__":
    asyncio.run(main())