238 lines
10 KiB
Python
238 lines
10 KiB
Python
import asyncio
|
||
import logging
|
||
import warnings
|
||
import os
|
||
import json
|
||
from datetime import datetime
|
||
from sqlalchemy import text, update, func
|
||
from app.database import AsyncSessionLocal
|
||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||
|
||
warnings.filterwarnings("ignore", category=RuntimeWarning, module='duckduckgo_search')
|
||
from duckduckgo_search import DDGS
|
||
|
||
# MB 2.0 Szabvány naplózás
|
||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-2-Researcher: %(message)s')
|
||
logger = logging.getLogger("Vehicle-Robot-2-Researcher")
|
||
|
||
class QuotaManager:
|
||
""" Szigorú napi limit figyelő a fizetős/hatósági API-khoz """
|
||
def __init__(self, service_name: str, daily_limit: int):
|
||
self.service_name = service_name
|
||
self.daily_limit = daily_limit
|
||
self.state_file = f"/app/temp/.quota_{service_name}.json"
|
||
self._ensure_file()
|
||
|
||
def _ensure_file(self):
|
||
os.makedirs(os.path.dirname(self.state_file), exist_ok=True)
|
||
if not os.path.exists(self.state_file):
|
||
with open(self.state_file, 'w') as f:
|
||
json.dump({"date": datetime.now().strftime("%Y-%m-%d"), "count": 0}, f)
|
||
|
||
def can_make_request(self) -> bool:
|
||
with open(self.state_file, 'r') as f:
|
||
data = json.load(f)
|
||
|
||
today = datetime.now().strftime("%Y-%m-%d")
|
||
if data["date"] != today:
|
||
data = {"date": today, "count": 0} # Új nap, kvóta nullázása
|
||
|
||
if data["count"] >= self.daily_limit:
|
||
return False
|
||
|
||
# Növeljük a számlálót
|
||
data["count"] += 1
|
||
with open(self.state_file, 'w') as f:
|
||
json.dump(data, f)
|
||
return True
|
||
|
||
class VehicleResearcher:
|
||
"""
|
||
Vehicle Robot 2.5: Sniper Researcher (Mesterlövész Adatgyűjtő)
|
||
Célzott keresésekkel és strukturált aktakészítéssel dolgozik az AI kímélése érdekében.
|
||
"""
|
||
def __init__(self):
|
||
self.max_attempts = 5
|
||
self.search_timeout = 15.0
|
||
|
||
# Kvóta menedzserek beállítása (.env-ből olvasva)
|
||
dvla_limit = int(os.getenv("DVLA_DAILY_LIMIT", "1000"))
|
||
self.dvla_quota = QuotaManager("dvla", dvla_limit)
|
||
self.dvla_token = os.getenv("DVLA_API_KEY")
|
||
|
||
async def fetch_ddg_targeted(self, label: str, query: str) -> str:
|
||
""" Célzott keresés szálbiztosan a DuckDuckGo-n. """
|
||
try:
|
||
def search():
|
||
with DDGS() as ddgs:
|
||
# max_results=2: Nem kell sok zaj, csak a legrelevánsabb 2 találat
|
||
results = ddgs.text(query, max_results=2)
|
||
return [f"- {r.get('body', '')}" for r in results] if results else []
|
||
|
||
results = await asyncio.wait_for(asyncio.to_thread(search), timeout=self.search_timeout)
|
||
|
||
if not results:
|
||
return f"[SOURCE: {label}]\nNincs érdemi találat.\n"
|
||
|
||
content = f"[SOURCE: {label} | KERESÉS: {query}]\n"
|
||
content += "\n".join(results) + "\n"
|
||
return content
|
||
|
||
except Exception as e:
|
||
logger.debug(f"Keresési hiba ({label}): {e}")
|
||
return f"[SOURCE: {label}]\nKERESÉSI HIBA.\n"
|
||
|
||
def extract_specs_from_text(self, text: str) -> dict:
|
||
""" Regex alapú kinyerés a nyers szövegből: ccm, kW, motoradatok. """
|
||
import re
|
||
specs = {}
|
||
|
||
# CCM (köbcentiméter) minta: 1998 cc, 2.0 L, 2000 cm³
|
||
ccm_pattern = r'(\d{3,4})\s*(?:cc|ccm|cm³|cm3|cc\.)'
|
||
match = re.search(ccm_pattern, text, re.IGNORECASE)
|
||
if match:
|
||
specs['ccm'] = int(match.group(1))
|
||
else:
|
||
# Alternatív minta: 2.0 liter -> 2000 cc
|
||
liter_pattern = r'(\d+\.?\d*)\s*(?:L|liter|ℓ)'
|
||
match = re.search(liter_pattern, text, re.IGNORECASE)
|
||
if match:
|
||
liters = float(match.group(1))
|
||
specs['ccm'] = int(liters * 1000)
|
||
|
||
# KW (kilowatt) minta: 150 kW, 150kW, 150 KW
|
||
kw_pattern = r'(\d{2,4})\s*(?:kW|kw|KW)'
|
||
match = re.search(kw_pattern, text, re.IGNORECASE)
|
||
if match:
|
||
specs['kw'] = int(match.group(1))
|
||
else:
|
||
# Le (lóerő) átváltás: 150 LE -> 110 kW (kb)
|
||
hp_pattern = r'(\d{2,4})\s*(?:HP|hp|LE|le|Ps)'
|
||
match = re.search(hp_pattern, text, re.IGNORECASE)
|
||
if match:
|
||
hp = int(match.group(1))
|
||
specs['kw'] = int(hp * 0.7355) # hozzávetőleges átváltás
|
||
|
||
# Motor kód minta: motor kód: 1.8 TSI, engine code: N47
|
||
engine_pattern = r'(?:motor\s*kód|engine\s*code|motor\s*code)[:\s]+([A-Z0-9\.\- ]+)'
|
||
match = re.search(engine_pattern, text, re.IGNORECASE)
|
||
if match:
|
||
specs['engine_code'] = match.group(1).strip()
|
||
|
||
return specs
|
||
|
||
async def research_vehicle(self, db, vehicle_id: int, make: str, model: str, engine: str, year: str, current_attempts: int):
|
||
""" Egy jármű átvilágítása és a strukturált 'Akta' elkészítése a GPU számára. """
|
||
engine_safe = engine or ""
|
||
year_safe = str(year) if year else ""
|
||
|
||
logger.info(f"🔎 Mesterlövész Kutatás: {make} {model} (Motor: {engine_safe})")
|
||
|
||
# 1. TIER: Ingyenes, Célzott Keresések (A legmegbízhatóbb források)
|
||
queries = [
|
||
("ULTIMATE_SPECS", f"{make} {model} {engine_safe} {year_safe} site:ultimatespecs.com"),
|
||
("AUTO_DATA", f"{make} {model} {engine_safe} {year_safe} site:auto-data.net"),
|
||
("COMMON_ISSUES", f"{make} {model} {engine_safe} reliability common problems")
|
||
]
|
||
|
||
tasks = [self.fetch_ddg_targeted(label, q) for label, q in queries]
|
||
search_results = await asyncio.gather(*tasks)
|
||
|
||
# 2. TIER: Fizetős / Kvótás API-k (Példa a DVLA helyére)
|
||
# Ha a jövőben bejön brit rendszám, itt hívjuk meg a DVLA-t:
|
||
# if has_uk_plate and self.dvla_quota.can_make_request():
|
||
# uk_data = await self.fetch_dvla_data(plate)
|
||
# search_results.append(uk_data)
|
||
|
||
# 3. ÖSSZESÍTÉS (Az Akta összeállítása)
|
||
# Maximalizáljuk a szöveg hosszát, hogy az AI GPU ne fulladjon le!
|
||
full_context = "\n".join(search_results)
|
||
if len(full_context) > 2500:
|
||
full_context = full_context[:2500] + "\n...[TRUNCATED TO SAVE GPU TOKENS]"
|
||
|
||
# Regex alapú specifikáció kinyerés
|
||
extracted_specs = self.extract_specs_from_text(full_context)
|
||
|
||
try:
|
||
if len(full_context.strip()) > 150: # Csökkentettük az elvárást, mert a célzott keresés tömörebb
|
||
await db.execute(
|
||
update(VehicleModelDefinition)
|
||
.where(VehicleModelDefinition.id == vehicle_id)
|
||
.values(
|
||
raw_search_context=full_context,
|
||
research_metadata=extracted_specs,
|
||
status='awaiting_ai_synthesis', # Kész az Akta, mehet az Alkimistának!
|
||
last_research_at=func.now(),
|
||
attempts=current_attempts + 1
|
||
)
|
||
)
|
||
logger.info(f"✅ Akta rögzítve ({len(full_context)} karakter): {make} {model}")
|
||
else:
|
||
new_status = 'suspended_research' if current_attempts + 1 >= self.max_attempts else 'unverified'
|
||
await db.execute(
|
||
update(VehicleModelDefinition)
|
||
.where(VehicleModelDefinition.id == vehicle_id)
|
||
.values(
|
||
status=new_status,
|
||
attempts=current_attempts + 1,
|
||
last_research_at=func.now()
|
||
)
|
||
)
|
||
if new_status == 'suspended_research':
|
||
logger.warning(f"🛑 Felfüggesztve (Nincs nyom a weben): {make} {model}")
|
||
else:
|
||
logger.warning(f"⚠️ Kevés adat: {make} {model}, visszatéve a sorba.")
|
||
|
||
await db.commit()
|
||
except Exception as e:
|
||
await db.rollback()
|
||
logger.error(f"🚨 Adatbázis hiba az eredmény mentésénél ({vehicle_id}): {e}")
|
||
|
||
@classmethod
|
||
async def run(cls):
|
||
self_instance = cls()
|
||
logger.info("🚀 Vehicle Researcher 2.5 ONLINE (Sniper & Quota Manager)")
|
||
|
||
while True:
|
||
try:
|
||
async with AsyncSessionLocal() as db:
|
||
# ATOMI ZÁROLÁS
|
||
query = text("""
|
||
UPDATE vehicle.vehicle_model_definitions
|
||
SET status = 'research_in_progress'
|
||
WHERE id = (
|
||
SELECT id FROM vehicle.vehicle_model_definitions
|
||
WHERE status IN ('unverified', 'awaiting_research', 'ACTIVE')
|
||
AND attempts < :max_attempts
|
||
AND is_manual = FALSE
|
||
ORDER BY
|
||
CASE WHEN make = 'TOYOTA' THEN 1 ELSE 2 END,
|
||
attempts ASC
|
||
FOR UPDATE SKIP LOCKED
|
||
LIMIT 1
|
||
)
|
||
RETURNING id, make, marketing_name, engine_code, year_from, attempts;
|
||
""")
|
||
|
||
result = await db.execute(query, {"max_attempts": self_instance.max_attempts})
|
||
task = result.fetchone()
|
||
await db.commit()
|
||
|
||
if task:
|
||
v_id, v_make, v_model, v_engine, v_year, v_attempts = task
|
||
async with AsyncSessionLocal() as process_db:
|
||
await self_instance.research_vehicle(process_db, v_id, v_make, v_model, v_engine, v_year, v_attempts)
|
||
|
||
await asyncio.sleep(2) # Rate limit védelem a DDG felé
|
||
else:
|
||
await asyncio.sleep(30)
|
||
|
||
except Exception as e:
|
||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||
await asyncio.sleep(10)
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
asyncio.run(VehicleResearcher.run())
|
||
except KeyboardInterrupt:
|
||
logger.info("🛑 Kutató robot leállítva.") |