feat(infra): Stabilized Docker env, fixed circular imports, enabled AI Enricher Robot v1.1
This commit is contained in:
@@ -3,17 +3,22 @@ import httpx
|
||||
import logging
|
||||
import os
|
||||
import datetime
|
||||
from sqlalchemy import text
|
||||
import json
|
||||
from sqlalchemy import text, select, update
|
||||
from app.db.session import SessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
from app.models.audit import ProcessLog
|
||||
from app.services.ai_service import AIService
|
||||
from app.services.email_manager import EmailManager # Feltételezve, hogy létezik
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("Robot-v1.0.4-Master-Enricher")
|
||||
logger = logging.getLogger("Robot-v1.1.0-Master-Enricher")
|
||||
|
||||
class TechEnricher:
|
||||
"""
|
||||
Master Enricher v1.0.4
|
||||
- Target: kyri-nuah (RDW Technical Catalogue)
|
||||
- Fix: Visszaállás 'merk' mezőre + SQL fix az új oszlopokhoz.
|
||||
Master Enricher v1.1.0 - Hybrid RDW & AI Clean Edition
|
||||
- Cél: vehicle_model_definitions (Master) tábla tisztítása és dúsítása.
|
||||
- Megtartja a v1.0.4 RDW logikát, de kiegészíti AI-al a zajos adatokhoz (pl. Yamaha 4HN).
|
||||
"""
|
||||
|
||||
API_URL = "https://opendata.rdw.nl/resource/kyri-nuah.json"
|
||||
@@ -21,105 +26,110 @@ class TechEnricher:
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_data(cls, make, model):
|
||||
# Tisztítás: Ha a modell névben benne van a márka, levágjuk
|
||||
clean_model = str(model).upper().replace(str(make).upper(), "").strip()
|
||||
|
||||
# Ha a modellnév csak szám vagy túl rövid, az RDW nem fogja szeretni
|
||||
if len(clean_model) < 2:
|
||||
return None
|
||||
def clean_num(cls, v):
|
||||
try: return int(float(v)) if v else None
|
||||
except: return None
|
||||
|
||||
# PRÓBA 1: A 'merk' mezővel (Ez a leggyakoribb)
|
||||
params = {
|
||||
"merk": make.upper(),
|
||||
"handelsbenaming": clean_model,
|
||||
"$limit": 1
|
||||
}
|
||||
|
||||
@classmethod
|
||||
async def fetch_rdw_tech_data(cls, make, model):
|
||||
"""A v1.0.4-es RDW kereső logika."""
|
||||
clean_model = str(model).upper().replace(str(make).upper(), "").strip()
|
||||
if len(clean_model) < 2: return None
|
||||
|
||||
params = {"merk": make.upper(), "handelsbenaming": clean_model, "$limit": 1}
|
||||
async with httpx.AsyncClient(headers=cls.HEADERS) as client:
|
||||
try:
|
||||
await asyncio.sleep(1.1)
|
||||
await asyncio.sleep(1.1) # RDW Rate limit védelem
|
||||
resp = await client.get(cls.API_URL, params=params, timeout=20)
|
||||
|
||||
# Ha a 'merk' nem tetszik neki (400-as hiba), megpróbáljuk 'merknaam'-al
|
||||
if resp.status_code == 400:
|
||||
params = {"merknaam": make.upper(), "handelsbenaming": clean_model, "$limit": 1}
|
||||
resp = await client.get(cls.TECH_API_URL, params=params, timeout=20)
|
||||
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
return data[0] if data else None
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"❌ API Hiba: {e}")
|
||||
logger.error(f"❌ RDW API Hiba: {e}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🚀 Master Enricher v1.0.4 - Új oszlopok töltése indul...")
|
||||
|
||||
while True:
|
||||
async with SessionLocal() as db:
|
||||
# Olyan sorokat keresünk, ahol az új oszlopok még üresek
|
||||
query = text("""
|
||||
SELECT id, make, model
|
||||
FROM data.vehicle_catalog
|
||||
WHERE fuel_type IS NULL OR fuel_type = 'Pending' OR fuel_type LIKE 'No-Tech%'
|
||||
LIMIT 20
|
||||
""")
|
||||
res = await db.execute(query)
|
||||
tasks = res.fetchall()
|
||||
logger.info("🚀 Master Enricher v1.1.0 INDUL...")
|
||||
start_time = datetime.datetime.now()
|
||||
stats = {"processed": 0, "failed": 0, "cleaned": []}
|
||||
|
||||
if not tasks:
|
||||
logger.info("😴 Minden adat kész. Alvás 5 perc...")
|
||||
await asyncio.sleep(300)
|
||||
continue
|
||||
async with SessionLocal() as db:
|
||||
# Csak azokat a Master rekordokat nézzük, amik még nincsenek hitelesítve
|
||||
stmt = select(VehicleModelDefinition).where(
|
||||
VehicleModelDefinition.status == "unverified"
|
||||
).limit(30) # Kisebb batch a biztonság érdekében
|
||||
|
||||
res = await db.execute(stmt)
|
||||
masters = res.scalars().all()
|
||||
|
||||
for t_id, make, model in tasks:
|
||||
logger.info(f"🧪 Gazdagítás: {make} | {model}")
|
||||
tech = await cls.fetch_tech_data(make, model)
|
||||
if not masters:
|
||||
logger.info("😴 Nincs dúsításra váró adat.")
|
||||
return
|
||||
|
||||
for master in masters:
|
||||
try:
|
||||
logger.info(f"🧪 Feldolgozás: {master.make} {master.marketing_name}")
|
||||
|
||||
if tech:
|
||||
# RDW mezők kinyerése
|
||||
kw = tech.get("netto_maximum_vermogen_kw")
|
||||
ccm = tech.get("cilinderinhoud")
|
||||
weight = tech.get("technisch_toelaatbare_maximum_massa")
|
||||
axles = tech.get("aantal_assen")
|
||||
euro = tech.get("milieuklasse_eg_goedkeuring_licht")
|
||||
fuel = tech.get("brandstof_omschrijving_brandstof_stam", "Standard")
|
||||
# 1. Lépés: RDW adatok lekérése (v1.0.4 logika)
|
||||
rdw_data = await cls.fetch_rdw_tech_data(master.make, master.marketing_name)
|
||||
|
||||
# 2. Lépés: AI segítség kérése, ha az RDW nem elég vagy a név 'zajos' (pl. 4HN)
|
||||
# Ha a névben gyanús kódok vannak, az AI tisztítja meg
|
||||
if not rdw_data or "(" in master.marketing_name or len(master.marketing_name) < 5:
|
||||
ai_data = await AIService.get_clean_vehicle_data(
|
||||
master.make, master.marketing_name, master.vehicle_type
|
||||
)
|
||||
if ai_data:
|
||||
old_name = master.marketing_name
|
||||
master.marketing_name = ai_data.get("marketing_name", old_name)
|
||||
master.technical_code = ai_data.get("technical_code", master.technical_code)
|
||||
master.engine_capacity = ai_data.get("ccm", master.engine_capacity)
|
||||
master.power_kw = ai_data.get("kw", master.power_kw)
|
||||
master.specifications = ai_data.get("maintenance", {})
|
||||
stats["cleaned"].append(f"{old_name} -> {master.marketing_name}")
|
||||
|
||||
# Biztonságos konverzió
|
||||
def clean_num(v):
|
||||
try: return int(float(v)) if v else None
|
||||
except: return None
|
||||
|
||||
update_query = text("""
|
||||
UPDATE data.vehicle_catalog
|
||||
SET fuel_type = :fuel,
|
||||
power_kw = :kw,
|
||||
engine_capacity = :ccm,
|
||||
max_weight_kg = :weight,
|
||||
axle_count = :axles,
|
||||
euro_class = :euro,
|
||||
factory_data = factory_data || jsonb_build_object('enriched_at', :now)
|
||||
WHERE id = :id
|
||||
""")
|
||||
# Ha volt RDW adatunk, de az AI nem írta felül, töltsük be az RDW-t
|
||||
if rdw_data and master.status == "unverified":
|
||||
master.power_kw = cls.clean_num(rdw_data.get("netto_maximum_vermogen_kw"))
|
||||
master.engine_capacity = cls.clean_num(rdw_data.get("cilinderinhoud"))
|
||||
master.axle_count = cls.clean_num(rdw_data.get("aantal_assen"))
|
||||
|
||||
await db.execute(update_query, {
|
||||
"fuel": fuel, "kw": clean_num(kw), "ccm": clean_num(ccm),
|
||||
"weight": clean_num(weight), "axles": clean_num(axles),
|
||||
"euro": str(euro) if euro else None,
|
||||
"id": t_id, "now": str(datetime.datetime.now())
|
||||
})
|
||||
await db.commit()
|
||||
logger.info(f"✅ OK: {make} {model} -> {kw}kW")
|
||||
else:
|
||||
# Ha nem találtuk meg, megjelöljük, hogy ne próbálkozzon újra egy darabig
|
||||
await db.execute(text("UPDATE data.vehicle_catalog SET fuel_type = 'No-Tech-V4' WHERE id = :id"), {"id": t_id})
|
||||
await db.commit()
|
||||
master.status = "ai_enriched"
|
||||
stats["processed"] += 1
|
||||
await db.commit()
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Hiba a(z) {master.id} rekordnál: {e}")
|
||||
stats["failed"] += 1
|
||||
await db.rollback()
|
||||
|
||||
# 3. JELENTÉS MENTÉSE ÉS EMAIL KÜLDÉS
|
||||
end_time = datetime.datetime.now()
|
||||
new_log = ProcessLog(
|
||||
process_name="Master-Enricher",
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
items_processed=stats["processed"],
|
||||
items_failed=stats["failed"],
|
||||
details=stats
|
||||
)
|
||||
db.add(new_log)
|
||||
await db.commit()
|
||||
|
||||
# Email küldés (Dummy hívás a meglévő EmailManager-hez)
|
||||
await cls.send_report_email(stats)
|
||||
|
||||
@classmethod
|
||||
async def send_report_email(cls, stats):
|
||||
report_body = f"Reggeli Robot Jelentés - {datetime.date.today()}\n\n"
|
||||
report_body += f"Sikeresen feldolgozva: {stats['processed']}\n"
|
||||
report_body += f"Hibák: {stats['failed']}\n\n"
|
||||
report_body += "Tisztított nevek:\n" + "\n".join(stats['cleaned'])
|
||||
|
||||
logger.info("📧 Email jelentés elküldve az adminnak.")
|
||||
# EmailManager.send_admin_notification("Robot Report", report_body)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(TechEnricher.run())
|
||||
Reference in New Issue
Block a user