átlagos kiegészítséek jó sok
This commit is contained in:
Roo
@@ -1,238 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
import asyncio
|
||||
import logging
|
||||
import warnings
|
||||
import os
|
||||
import json
|
||||
from datetime import datetime
|
||||
from sqlalchemy import text, update, func
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning, module='duckduckgo_search')
|
||||
import httpx
|
||||
import re
|
||||
from bs4 import BeautifulSoup
|
||||
from duckduckgo_search import DDGS
|
||||
from playwright.async_api import async_playwright
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
# MB 2.0 Szabvány naplózás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-2-Researcher: %(message)s')
|
||||
logger = logging.getLogger("Vehicle-Robot-2-Researcher")
|
||||
# Figyelmeztetések némítása (a csomag átnevezése miatti zaj elkerülésére)
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning, module='duckduckgo_search')
|
||||
|
||||
class QuotaManager:
|
||||
""" Szigorú napi limit figyelő a fizetős/hatósági API-khoz """
|
||||
def __init__(self, service_name: str, daily_limit: int):
|
||||
self.service_name = service_name
|
||||
self.daily_limit = daily_limit
|
||||
self.state_file = f"/app/temp/.quota_{service_name}.json"
|
||||
self._ensure_file()
|
||||
|
||||
def _ensure_file(self):
|
||||
os.makedirs(os.path.dirname(self.state_file), exist_ok=True)
|
||||
if not os.path.exists(self.state_file):
|
||||
with open(self.state_file, 'w') as f:
|
||||
json.dump({"date": datetime.now().strftime("%Y-%m-%d"), "count": 0}, f)
|
||||
|
||||
def can_make_request(self) -> bool:
|
||||
with open(self.state_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
if data["date"] != today:
|
||||
data = {"date": today, "count": 0} # Új nap, kvóta nullázása
|
||||
|
||||
if data["count"] >= self.daily_limit:
|
||||
return False
|
||||
|
||||
# Növeljük a számlálót
|
||||
data["count"] += 1
|
||||
with open(self.state_file, 'w') as f:
|
||||
json.dump(data, f)
|
||||
return True
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [R2-MASTER-EDITION] %(message)s'
|
||||
)
|
||||
logger = logging.getLogger("R2-Researcher")
|
||||
|
||||
class VehicleResearcher:
|
||||
"""
|
||||
Vehicle Robot 2.5: Sniper Researcher (Mesterlövész Adatgyűjtő)
|
||||
Célzott keresésekkel és strukturált aktakészítéssel dolgozik az AI kímélése érdekében.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.max_attempts = 5
|
||||
self.search_timeout = 15.0
|
||||
def __init__(self, concurrency=5):
|
||||
# Egyszerre 5 böngésző fület kezelünk a sebesség érdekében
|
||||
self.semaphore = asyncio.Semaphore(concurrency)
|
||||
self.ollama_url = "http://sf_ollama:11434/api/generate"
|
||||
|
||||
# Kvóta menedzserek beállítása (.env-ből olvasva)
|
||||
dvla_limit = int(os.getenv("DVLA_DAILY_LIMIT", "1000"))
|
||||
self.dvla_quota = QuotaManager("dvla", dvla_limit)
|
||||
self.dvla_token = os.getenv("DVLA_API_KEY")
|
||||
# FORDÍTÓ SZÓTÁR: Holland RDW -> Nemzetközi keresési nevek
|
||||
self.translation_map = {
|
||||
"ER REIHE": "Series",
|
||||
"T-MODELL": "Estate",
|
||||
"KLASSE": "Class",
|
||||
"PERSONENAUTO": "Car",
|
||||
"STATIONWAGEN": "Estate",
|
||||
"MERCEDES-BENZ": "Mercedes",
|
||||
"Vrachtwagen": "Truck",
|
||||
"Oplegger": "Trailer"
|
||||
}
|
||||
|
||||
async def fetch_ddg_targeted(self, label: str, query: str) -> str:
|
||||
""" Célzott keresés szálbiztosan a DuckDuckGo-n. """
|
||||
def clean_name(self, make, model):
|
||||
"""Lefordítja a holland modellneveket, hogy a Google/Bing megtalálja őket."""
|
||||
name = f"{make} {model}".upper()
|
||||
for dutch, eng in self.translation_map.items():
|
||||
name = name.replace(dutch, eng)
|
||||
return name.title()
|
||||
|
||||
async def get_url(self, make, model, year, kw):
|
||||
"""Keresés a DuckDuckGo-val. JAVÍTVA: 0kW fix és több találat."""
|
||||
clean_n = self.clean_name(make, model)
|
||||
|
||||
# Ha a kW 0, None vagy érvénytelen, kihagyjuk a keresésből a találati arány javítására
|
||||
kw_val = 0
|
||||
try:
|
||||
def search():
|
||||
if kw and str(kw).replace('.','').isdigit():
|
||||
kw_val = int(float(kw))
|
||||
except: pass
|
||||
|
||||
kw_part = f"{kw_val}kW" if kw_val > 0 else ""
|
||||
query = f"site:auto-data.net {clean_n} {year} {kw_part} specifications"
|
||||
|
||||
try:
|
||||
def _search():
|
||||
with DDGS() as ddgs:
|
||||
# max_results=2: Nem kell sok zaj, csak a legrelevánsabb 2 találat
|
||||
results = ddgs.text(query, max_results=2)
|
||||
return [f"- {r.get('body', '')}" for r in results] if results else []
|
||||
|
||||
results = await asyncio.wait_for(asyncio.to_thread(search), timeout=self.search_timeout)
|
||||
|
||||
if not results:
|
||||
return f"[SOURCE: {label}]\nNincs érdemi találat.\n"
|
||||
|
||||
content = f"[SOURCE: {label} | KERESÉS: {query}]\n"
|
||||
content += "\n".join(results) + "\n"
|
||||
return content
|
||||
# Megnézzük az első 3 találatot, hátha az első nem direkt link
|
||||
res = ddgs.search(query, max_results=3)
|
||||
return [r.get('link', r.get('href', '')) for r in res if 'auto-data.net' in r.get('link', r.get('href', ''))]
|
||||
|
||||
links = await asyncio.to_thread(_search)
|
||||
return links[0] if links else None
|
||||
except Exception as e:
|
||||
logger.debug(f"Keresési hiba ({label}): {e}")
|
||||
return f"[SOURCE: {label}]\nKERESÉSI HIBA.\n"
|
||||
logger.warning(f"Keresési hiba ({query}): {e}")
|
||||
return None
|
||||
|
||||
def extract_specs_from_text(self, text: str) -> dict:
|
||||
""" Regex alapú kinyerés a nyers szövegből: ccm, kW, motoradatok. """
|
||||
import re
|
||||
async def scrape_auto_data(self, url, browser):
|
||||
"""Letölti az oldalt és kinyeri az összes technikai adatot."""
|
||||
specs = {}
|
||||
|
||||
# CCM (köbcentiméter) minta: 1998 cc, 2.0 L, 2000 cm³
|
||||
ccm_pattern = r'(\d{3,4})\s*(?:cc|ccm|cm³|cm3|cc\.)'
|
||||
match = re.search(ccm_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['ccm'] = int(match.group(1))
|
||||
else:
|
||||
# Alternatív minta: 2.0 liter -> 2000 cc
|
||||
liter_pattern = r'(\d+\.?\d*)\s*(?:L|liter|ℓ)'
|
||||
match = re.search(liter_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
liters = float(match.group(1))
|
||||
specs['ccm'] = int(liters * 1000)
|
||||
|
||||
# KW (kilowatt) minta: 150 kW, 150kW, 150 KW
|
||||
kw_pattern = r'(\d{2,4})\s*(?:kW|kw|KW)'
|
||||
match = re.search(kw_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['kw'] = int(match.group(1))
|
||||
else:
|
||||
# Le (lóerő) átváltás: 150 LE -> 110 kW (kb)
|
||||
hp_pattern = r'(\d{2,4})\s*(?:HP|hp|LE|le|Ps)'
|
||||
match = re.search(hp_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
hp = int(match.group(1))
|
||||
specs['kw'] = int(hp * 0.7355) # hozzávetőleges átváltás
|
||||
|
||||
# Motor kód minta: motor kód: 1.8 TSI, engine code: N47
|
||||
engine_pattern = r'(?:motor\s*kód|engine\s*code|motor\s*code)[:\s]+([A-Z0-9\.\- ]+)'
|
||||
match = re.search(engine_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['engine_code'] = match.group(1).strip()
|
||||
|
||||
return specs
|
||||
|
||||
async def research_vehicle(self, db, vehicle_id: int, make: str, model: str, engine: str, year: str, current_attempts: int):
|
||||
""" Egy jármű átvilágítása és a strukturált 'Akta' elkészítése a GPU számára. """
|
||||
engine_safe = engine or ""
|
||||
year_safe = str(year) if year else ""
|
||||
|
||||
logger.info(f"🔎 Mesterlövész Kutatás: {make} {model} (Motor: {engine_safe})")
|
||||
|
||||
# 1. TIER: Ingyenes, Célzott Keresések (A legmegbízhatóbb források)
|
||||
queries = [
|
||||
("ULTIMATE_SPECS", f"{make} {model} {engine_safe} {year_safe} site:ultimatespecs.com"),
|
||||
("AUTO_DATA", f"{make} {model} {engine_safe} {year_safe} site:auto-data.net"),
|
||||
("COMMON_ISSUES", f"{make} {model} {engine_safe} reliability common problems")
|
||||
]
|
||||
|
||||
tasks = [self.fetch_ddg_targeted(label, q) for label, q in queries]
|
||||
search_results = await asyncio.gather(*tasks)
|
||||
|
||||
# 2. TIER: Fizetős / Kvótás API-k (Példa a DVLA helyére)
|
||||
# Ha a jövőben bejön brit rendszám, itt hívjuk meg a DVLA-t:
|
||||
# if has_uk_plate and self.dvla_quota.can_make_request():
|
||||
# uk_data = await self.fetch_dvla_data(plate)
|
||||
# search_results.append(uk_data)
|
||||
|
||||
# 3. ÖSSZESÍTÉS (Az Akta összeállítása)
|
||||
# Maximalizáljuk a szöveg hosszát, hogy az AI GPU ne fulladjon le!
|
||||
full_context = "\n".join(search_results)
|
||||
if len(full_context) > 2500:
|
||||
full_context = full_context[:2500] + "\n...[TRUNCATED TO SAVE GPU TOKENS]"
|
||||
|
||||
# Regex alapú specifikáció kinyerés
|
||||
extracted_specs = self.extract_specs_from_text(full_context)
|
||||
|
||||
full_text = ""
|
||||
try:
|
||||
if len(full_context.strip()) > 150: # Csökkentettük az elvárást, mert a célzott keresés tömörebb
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == vehicle_id)
|
||||
.values(
|
||||
raw_search_context=full_context,
|
||||
research_metadata=extracted_specs,
|
||||
status='awaiting_ai_synthesis', # Kész az Akta, mehet az Alkimistának!
|
||||
last_research_at=func.now(),
|
||||
attempts=current_attempts + 1
|
||||
)
|
||||
)
|
||||
logger.info(f"✅ Akta rögzítve ({len(full_context)} karakter): {make} {model}")
|
||||
else:
|
||||
new_status = 'suspended_research' if current_attempts + 1 >= self.max_attempts else 'unverified'
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == vehicle_id)
|
||||
.values(
|
||||
status=new_status,
|
||||
attempts=current_attempts + 1,
|
||||
last_research_at=func.now()
|
||||
)
|
||||
)
|
||||
if new_status == 'suspended_research':
|
||||
logger.warning(f"🛑 Felfüggesztve (Nincs nyom a weben): {make} {model}")
|
||||
else:
|
||||
logger.warning(f"⚠️ Kevés adat: {make} {model}, visszatéve a sorba.")
|
||||
page = await browser.new_page()
|
||||
# Gyorsítás: képek, videók és stíluslapok tiltása
|
||||
await page.route("**/*.{png,jpg,jpeg,gif,css,woff2}", lambda r: r.abort())
|
||||
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=20000)
|
||||
html = await page.content()
|
||||
# Kimentjük a tiszta szöveget is, ha az AI-nak kellene később
|
||||
full_text = await page.evaluate("() => document.body.innerText")
|
||||
await page.close()
|
||||
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
# Végigfutunk minden táblázat soron
|
||||
for row in soup.find_all('tr'):
|
||||
th = row.find('th')
|
||||
td = row.find('td')
|
||||
if th and td:
|
||||
k, v = th.get_text(strip=True).lower(), td.get_text(strip=True)
|
||||
|
||||
await db.commit()
|
||||
# Minden fontos mező kinyerése
|
||||
if "engine model/code" in k: specs["engine_code"] = v
|
||||
elif "engine oil capacity" in k: specs["oil_l"] = v
|
||||
elif "acceleration 0 - 100" in k: specs["acc_0_100"] = v
|
||||
elif "maximum speed" in k: specs["max_speed"] = v
|
||||
elif "fuel consumption" in k and "combined" in k: specs["cons_avg"] = v
|
||||
elif "co2 emissions" in k: specs["co2"] = v
|
||||
elif "generation" in k: specs["generation"] = v
|
||||
elif "tires size" in k: specs["tires"] = v
|
||||
elif "trunk (boot) space" in k: specs["trunk_l"] = v
|
||||
elif "kerb weight" in k: specs["weight_kg"] = v
|
||||
elif "drivetrain" in k: specs["drivetrain"] = v
|
||||
elif "number of gears" in k: specs["transmission"] = v
|
||||
|
||||
return specs, full_text
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(f"🚨 Adatbázis hiba az eredmény mentésénél ({vehicle_id}): {e}")
|
||||
logger.error(f"Scraping hiba az oldalon ({url}): {e}")
|
||||
return {}, ""
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
self_instance = cls()
|
||||
logger.info("🚀 Vehicle Researcher 2.5 ONLINE (Sniper & Quota Manager)")
|
||||
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# ATOMI ZÁROLÁS
|
||||
query = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = 'research_in_progress'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.vehicle_model_definitions
|
||||
WHERE status IN ('unverified', 'awaiting_research', 'ACTIVE')
|
||||
AND attempts < :max_attempts
|
||||
AND is_manual = FALSE
|
||||
ORDER BY
|
||||
CASE WHEN make = 'TOYOTA' THEN 1 ELSE 2 END,
|
||||
attempts ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, make, marketing_name, engine_code, year_from, attempts;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {"max_attempts": self_instance.max_attempts})
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
async def ask_ai_fallback(self, raw_text):
|
||||
"""Ha a BeautifulSoup nem talál táblázatot, megkérjük az Ollamát."""
|
||||
if not raw_text or len(raw_text) < 200: return {}
|
||||
prompt = f"Extract vehicle specs (engine_code, oil_capacity, tires, generation) as JSON from this text: {raw_text[:2500]}"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
r = await client.post(self.ollama_url, json={
|
||||
"model": "qwen2.5-coder:14b",
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"format": "json"
|
||||
})
|
||||
return json.loads(r.json().get("response", "{}"))
|
||||
except: return {}
|
||||
|
||||
if task:
|
||||
v_id, v_make, v_model, v_engine, v_year, v_attempts = task
|
||||
async with AsyncSessionLocal() as process_db:
|
||||
await self_instance.research_vehicle(process_db, v_id, v_make, v_model, v_engine, v_year, v_attempts)
|
||||
|
||||
await asyncio.sleep(2) # Rate limit védelem a DDG felé
|
||||
async def process_vehicle(self, v_id, make, model, year, kw, browser):
|
||||
"""Egy jármű dúsításának teljes folyamata."""
|
||||
async with self.semaphore:
|
||||
logger.info(f"🔍 Kutatás: {make} {model} ({year}) | kW: {kw}")
|
||||
url = await self.get_url(make, model, year, kw)
|
||||
|
||||
specs = {}
|
||||
if url:
|
||||
logger.info(f"🔗 Találat: {url}")
|
||||
specs, raw_text = await self.scrape_auto_data(url, browser)
|
||||
|
||||
# Ha a táblázatból nem jött ki elég adat, jöhet az AI fallback
|
||||
if len(specs) < 3:
|
||||
ai_specs = await self.ask_ai_fallback(raw_text)
|
||||
specs.update(ai_specs)
|
||||
|
||||
# MENTÉS: Minden szál saját adatbázis kapcsolatot használ a biztonság érdekében
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Csak akkor validation_ready, ha találtunk adatot. Ha nem, külön státuszba tesszük.
|
||||
new_status = 'validation_ready' if len(specs) > 0 else 'research_failed_empty'
|
||||
|
||||
update_query = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET specifications = specifications || CAST(:specs AS JSONB),
|
||||
status = :status,
|
||||
last_research_at = now()
|
||||
WHERE id = :id
|
||||
""")
|
||||
await db.execute(update_query, {
|
||||
"specs": json.dumps(specs),
|
||||
"status": new_status,
|
||||
"id": v_id
|
||||
})
|
||||
await db.commit()
|
||||
|
||||
if len(specs) > 0:
|
||||
logger.info(f"✅ SIKER: {make} {model} ({len(specs)} adat kinyerve)")
|
||||
else:
|
||||
await asyncio.sleep(30)
|
||||
logger.warning(f"❌ SIKERTELEN: {make} {model} (nem találtunk adatot a neten)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||||
await asyncio.sleep(10)
|
||||
async def run(self):
|
||||
logger.info("🚀 R2-Kutató MASTER-EDITION (0kW fix + AI Fallback) ONLINE")
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 10 autó bekérése párhuzamos feldolgozásra
|
||||
res = await db.execute(text("""
|
||||
UPDATE vehicle.vehicle_model_definitions SET status = 'research_in_progress'
|
||||
WHERE id IN (
|
||||
SELECT id FROM vehicle.vehicle_model_definitions
|
||||
WHERE status = 'enrich_ready'
|
||||
LIMIT 10
|
||||
) RETURNING id, make, marketing_name, year_from, power_kw
|
||||
"""))
|
||||
rows = res.fetchall()
|
||||
await db.commit()
|
||||
|
||||
if not rows:
|
||||
await asyncio.sleep(15)
|
||||
continue
|
||||
|
||||
tasks = [self.process_vehicle(r[0], r[1], r[2], r[3], r[4], browser) for r in rows]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(VehicleResearcher.run())
|
||||
except KeyboardInterrupt:
|
||||
logger.info("🛑 Kutató robot leállítva.")
|
||||
asyncio.run(VehicleResearcher().run())
|
||||
Reference in New Issue
Block a user