teljes backend_mentés

2026-03-22 18:59:27 +00:00
parent 5d44339f21
commit 5d96b00f81
34 changed files with 2575 additions and 977 deletions
--- a/backend/app/workers/monitor_dashboard.py.old
+++ b/backend/app/workers/monitor_dashboard.py.old
@@ -0,0 +1,221 @@
+# /opt/docker/dev/service_finder/backend/app/workers/monitor_dashboard.py
+import asyncio
+import os
+import httpx
+import pynvml
+import psutil
+import subprocess
+from datetime import datetime, timedelta
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.live import Live
+from rich.layout import Layout
+from app.core.config import settings
+
+console = Console()
+
+STATUS_TRANSLATIONS = {
+    'published': 'Véglegesítve (Publikált)',
+    'awaiting_ai_synthesis': 'AI Szintézisre Vár',
+    'manual_review_needed': 'Kézi Javítás Szükséges',
+    'unverified': 'Ellenőrizetlen (Nyers)',
+    'research_in_progress': 'Kutatás Folyamatban',
+    'ai_synthesis_in_progress': 'AI Szintézis Alatt',
+    'gold_enriched': 'Aranyosított (Végleges)',
+    'pending': 'Függőben',
+    'processing': 'Feldolgozás alatt'
+}
+
+try:
+    pynvml.nvmlInit()
+    gpu_available = True
+except Exception:
+    gpu_available = False
+
+def get_gpu_content():
+    try:
+        gpu_raw = subprocess.check_output(
+            ['nvidia-smi', '--query-gpu=name,utilization.gpu,memory.used,memory.total,temperature.gpu', '--format=csv,noheader,nounits'],
+            encoding='utf-8'
+        ).strip().split(', ')
+        gpu_name = gpu_raw[0].replace("NVIDIA ", "")
+        gpu_content = f"GPU: [bold bright_white]NVIDIA {gpu_name}[/]\nTerhelés: [bold orange3]{gpu_raw[1]}%[/]\nVRAM: [bold cyan]{gpu_raw[2]} MB[/] / {gpu_raw[3]} MB\nHőmérséklet: [bold red]{gpu_raw[4]} °C[/]"
+    except Exception as e:
+        gpu_content = f"GPU adatok olvasása sikertelen: {str(e)}"
+    return gpu_content
+
+async def get_hardware_stats():
+    stats = {
+        "cpu_usage": psutil.cpu_percent(interval=None),
+        "ram_total": psutil.virtual_memory().total // 1024**2,
+        "ram_used": psutil.virtual_memory().used // 1024**2,
+        "ram_perc": psutil.virtual_memory().percent,
+        "gpu_content": get_gpu_content()
+    }
+    return stats
+
+async def get_ollama_models():
+    try:
+        async with httpx.AsyncClient(timeout=2.0) as client:
+            resp = await client.get("http://ollama:11434/api/ps")
+            if resp.status_code == 200:
+                return [m['name'] for m in resp.json().get("models", [])]
+    except: return ["Ollama API Offline"]
+    return []
+
+async def get_stats(engine):
+    async with engine.connect() as conn:
+        # JAVÍTVA: Hogy valós sebességet lássunk, a 'gold_enriched' (épp elkészült) autókat is beleszámoljuk az órás rate-be!
+        hr_rate = (await conn.execute(text("SELECT COALESCE(count(*), 0) FROM vehicle.vehicle_model_definitions WHERE status IN ('published', 'gold_enriched') AND updated_at > NOW() - INTERVAL '1 hour'"))).scalar()
+        day_rate = (await conn.execute(text("SELECT COALESCE(count(*), 0) FROM vehicle.vehicle_model_definitions WHERE status IN ('published', 'gold_enriched') AND updated_at > NOW() - INTERVAL '24 hours'"))).scalar()
+
+        r1 = (await conn.execute(text("SELECT count(*) FROM vehicle.catalog_discovery WHERE status = 'pending'"))).scalar()
+        r2 = (await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'unverified'"))).scalar()
+        r3 = (await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'awaiting_ai_synthesis'"))).scalar()
+        r4 = (await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched'"))).scalar()
+        r_counts = (r1, r2, r3, r4)
+
+        top_makes = (await conn.execute(text("SELECT make, count(*) as qty FROM vehicle.vehicle_model_definitions GROUP BY make ORDER BY qty DESC LIMIT 7"))).fetchall()
+
+        res_r4 = (await conn.execute(text("SELECT make, marketing_name FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' ORDER BY updated_at DESC LIMIT 5"))).fetchall()
+        res_r3 = (await conn.execute(text("SELECT make, marketing_name FROM vehicle.vehicle_model_definitions WHERE status = 'ai_synthesis_in_progress' ORDER BY updated_at DESC LIMIT 5"))).fetchall()
+        res_r12 = (await conn.execute(text("SELECT make, model FROM vehicle.catalog_discovery WHERE status = 'processing' ORDER BY id DESC LIMIT 5"))).fetchall()
+
+        published_count = (await conn.execute(text("SELECT COUNT(*) FROM vehicle.vehicle_model_definitions WHERE status = 'published'"))).scalar()
+        manual_review_needed_count = (await conn.execute(text("SELECT COUNT(*) FROM vehicle.vehicle_model_definitions WHERE status = 'manual_review_needed'"))).scalar()
+        
+        status_distribution = (await conn.execute(text("SELECT status, COUNT(*) as count FROM vehicle.vehicle_model_definitions GROUP BY status ORDER BY count DESC"))).fetchall()
+        make_distribution = (await conn.execute(text("SELECT make, COUNT(*) as count FROM vehicle.vehicle_model_definitions WHERE status = 'published' GROUP BY make ORDER BY count DESC LIMIT 15"))).fetchall()
+
+        manual_review_list = (await conn.execute(text(
+            "SELECT make, marketing_name, COUNT(*) as count FROM vehicle.vehicle_model_definitions WHERE status = 'manual_review_needed' GROUP BY make, marketing_name ORDER BY count DESC LIMIT 15"
+        ))).fetchall()
+
+        hw = await get_hardware_stats()
+        ai = await get_ollama_models()
+
+        return (hr_rate, day_rate), r_counts, top_makes, (res_r4, res_r3, res_r12), hw, ai, (published_count, manual_review_needed_count, status_distribution, make_distribution, manual_review_list)
+
+def make_layout() -> Layout:
+    layout = Layout()
+    layout.split_column(
+        Layout(name="header", size=3),
+        Layout(name="main", ratio=1),
+        Layout(name="hardware", size=8),
+        Layout(name="footer", size=3)
+    )
+    layout["main"].split_row(
+        Layout(name="left", ratio=1),
+        Layout(name="middle", ratio=1),
+        Layout(name="right", ratio=2)
+    )
+    layout["left"].split_column(Layout(name="robot_stats", ratio=1), Layout(name="inventory", ratio=2))
+    layout["middle"].split_column(Layout(name="db_left", ratio=1), Layout(name="db_right", ratio=2))
+    layout["right"].split_column(
+        Layout(name="live_ops", ratio=1),
+        Layout(name="manual_review", ratio=2)
+    )
+    return layout
+
+def translate_status(status):
+    return STATUS_TRANSLATIONS.get(status, status)
+
+def update_dashboard(layout, data, error_msg=""):
+    rates, r_counts, top_makes, live_data, hw, ai_models, db_stats = data
+    r4_list, r3_list, r12_list = live_data
+    published_count, manual_review_needed_count, status_distribution, make_distribution, manual_review_list = db_stats
+
+    local_time = datetime.now() + timedelta(hours=1)
+    
+    layout["header"].update(Panel(
+        f"🛰️  SENTINEL IRÁNYÍTÓKÖZPONT | [bold yellow]{local_time.strftime('%Y-%m-%d %H:%M:%S')}[/] | AI Teljesítmény: [green]{rates[0]:,}[/] /óra — [cyan]{rates[1]:,}[/] /nap | Összes publikált: [bold green]{published_count:,}[/]",
+        style="bold white on blue"
+    ))
+
+    robot_table = Table(title="🤖 Robot Pipeline Állapot", expand=True, border_style="cyan")
+    robot_table.add_column("Robot", style="bold")
+    robot_table.add_column("Várakozik", justify="right")
+    robot_table.add_row("R1-Hunter (Nyers gyűjtés)", f"{r_counts[0]:,} db")
+    robot_table.add_row("R2-Researcher (Webes kutatás)", f"{r_counts[1]:,} db")
+    robot_table.add_row("R3-Alchemist (AI Szintézis)", f"{r_counts[2]:,} db")
+    robot_table.add_row("R4-Validator (Várakozó Arany)", f"[green]{r_counts[3]:,}[/] db")
+    layout["robot_stats"].update(robot_table)
+
+    brand_table = Table(title="🚜 Bányászott Márkák (Top 7)", expand=True, border_style="magenta")
+    brand_table.add_column("Márka", style="yellow")
+    brand_table.add_column("Darabszám", justify="right")
+    for m, q in top_makes: brand_table.add_row(str(m), str(q))
+    layout["inventory"].update(brand_table)
+ 
+    ops_table = Table(title="⚡ Aktuális Folyamatok", expand=True, border_style="green")
+    ops_table.add_column("Robot", width=15)
+    ops_table.add_column("Márka / Típus")
+    
+    for r in r4_list: ops_table.add_row("[gold1]R4-ARANY[/]", f"{r[0]} {r[1] or ''}")
+    if r4_list: ops_table.add_section()
+    for r in r3_list: ops_table.add_row("[medium_purple1]R3-AI[/]", f"{r[0]} {r[1] or ''}")
+    if r3_list: ops_table.add_section()
+    for r in r12_list: ops_table.add_row("[sky_blue1]R1-HUNTER[/]", f"{r[0]} {r[1] or ''}")
+    layout["live_ops"].update(ops_table)
+
+    hw_layout = Layout()
+    hw_layout.split_row(
+        Layout(name="sys", ratio=1),
+        Layout(name="gpu_combined", ratio=2)
+    )
+    
+    sys_info = f"[bold]CPU:[/]\t[bright_blue]{hw['cpu_usage']}%[/]\n[bold]RAM:[/]\t[bright_magenta]{hw['ram_perc']}%[/] ({hw['ram_used']}/{hw['ram_total']}MB)"
+    hw_layout["sys"].update(Panel(sys_info, title="💻 Rendszer", border_style="bright_blue"))
+
+    gpu_info = hw.get("gpu_content", "GPU adatok nem elérhetők")
+    ai_info = " | ".join([f"🧠 [plum1]{m}[/]" for m in ai_models]) if ai_models else "Nincs betöltve modell."
+    combined_gpu_text = f"{gpu_info}\n[bold bright_white]🤖 Ollama Modellek:[/] {ai_info}"
+    hw_layout["gpu_combined"].update(Panel(combined_gpu_text, title="🔌 GPU & AI Központ", border_style="orange3"))
+    layout["hardware"].update(hw_layout)
+
+    status_table = Table(title="📈 Státusz eloszlás", expand=True, border_style="magenta")
+    status_table.add_column("Státusz", style="bold")
+    status_table.add_column("Mennyiség", justify="right")
+    for status, count in status_distribution:
+        status_table.add_row(translate_status(status), f"{count:,}")
+    layout["db_left"].update(Panel(status_table, title="📊 Státuszok", border_style="magenta"))
+    
+    # ÚJ: Bekerült a végösszesítő mező a lista aljára!
+    make_table = Table(title="🚗 Márkák (véglegesített)", expand=True, border_style="green")
+    make_table.add_column("Márka", style="yellow")
+    make_table.add_column("Darab", justify="right")
+    for make, count in make_distribution:
+        make_table.add_row(str(make), f"{count:,}")
+    make_table.add_section()
+    make_table.add_row("[bold bright_white]ÖSSZES PUBLIKÁLT[/]", f"[bold green]{published_count:,}[/]")
+    layout["db_right"].update(Panel(make_table, title="🏆 Top Márkák", border_style="green"))
+    
+    manual_table = Table(title="🛠️ Kézi Javításra Várók (Top 15)", expand=True, border_style="yellow")
+    manual_table.add_column("Márka", style="bold")
+    manual_table.add_column("Modell", style="cyan")
+    manual_table.add_column("Darabszám", justify="right")
+    for make, model, count in manual_review_list:
+        manual_table.add_row(str(make), str(model) if model else "N/A", f"{count:,}")
+    layout["manual_review"].update(Panel(manual_table, title="🛠️ Kézi Javításra Várók", border_style="yellow"))
+    
+    footer_text = f"Sentinel v2.6 | Kernel: Stabil | R1 Pörög: {r_counts[0]:,} várakozik"
+    if error_msg: footer_text = f"[red bold]HIBA: {error_msg}[/]"
+    layout["footer"].update(Panel(footer_text, style="italic grey50"))
+
+async def main():
+    engine = create_async_engine(settings.DATABASE_URL)
+    layout = make_layout()
+    with Live(layout, refresh_per_second=2, screen=True):
+        while True:
+            try:
+                data = await get_stats(engine)
+                update_dashboard(layout, data)
+            except Exception as e:
+                update_dashboard(layout, ((0,0), (0,0,0,0), [], ([],[],[]), {"cpu_usage":0,"ram_perc":0,"ram_used":0,"ram_total":0,"gpu_content":""}, [], (0, 0, [], [], [])), str(e))
+            await asyncio.sleep(0.5)
+
+if __name__ == "__main__":
+    asyncio.run(main())