125 lines
5.2 KiB
Python
125 lines
5.2 KiB
Python
import asyncio
|
|
import httpx
|
|
import logging
|
|
import os
|
|
import datetime
|
|
from sqlalchemy import text
|
|
from app.db.session import SessionLocal
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger("Robot-v1.0.4-Master-Enricher")
|
|
|
|
class TechEnricher:
|
|
"""
|
|
Master Enricher v1.0.4
|
|
- Target: kyri-nuah (RDW Technical Catalogue)
|
|
- Fix: Visszaállás 'merk' mezőre + SQL fix az új oszlopokhoz.
|
|
"""
|
|
|
|
API_URL = "https://opendata.rdw.nl/resource/kyri-nuah.json"
|
|
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
|
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
|
|
|
@classmethod
|
|
async def fetch_tech_data(cls, make, model):
|
|
# Tisztítás: Ha a modell névben benne van a márka, levágjuk
|
|
clean_model = str(model).upper().replace(str(make).upper(), "").strip()
|
|
|
|
# Ha a modellnév csak szám vagy túl rövid, az RDW nem fogja szeretni
|
|
if len(clean_model) < 2:
|
|
return None
|
|
|
|
# PRÓBA 1: A 'merk' mezővel (Ez a leggyakoribb)
|
|
params = {
|
|
"merk": make.upper(),
|
|
"handelsbenaming": clean_model,
|
|
"$limit": 1
|
|
}
|
|
|
|
async with httpx.AsyncClient(headers=cls.HEADERS) as client:
|
|
try:
|
|
await asyncio.sleep(1.1)
|
|
resp = await client.get(cls.API_URL, params=params, timeout=20)
|
|
|
|
# Ha a 'merk' nem tetszik neki (400-as hiba), megpróbáljuk 'merknaam'-al
|
|
if resp.status_code == 400:
|
|
params = {"merknaam": make.upper(), "handelsbenaming": clean_model, "$limit": 1}
|
|
resp = await client.get(cls.TECH_API_URL, params=params, timeout=20)
|
|
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
return data[0] if data else None
|
|
|
|
return None
|
|
except Exception as e:
|
|
logger.error(f"❌ API Hiba: {e}")
|
|
return None
|
|
|
|
@classmethod
|
|
async def run(cls):
|
|
logger.info("🚀 Master Enricher v1.0.4 - Új oszlopok töltése indul...")
|
|
|
|
while True:
|
|
async with SessionLocal() as db:
|
|
# Olyan sorokat keresünk, ahol az új oszlopok még üresek
|
|
query = text("""
|
|
SELECT id, make, model
|
|
FROM data.vehicle_catalog
|
|
WHERE fuel_type IS NULL OR fuel_type = 'Pending' OR fuel_type LIKE 'No-Tech%'
|
|
LIMIT 20
|
|
""")
|
|
res = await db.execute(query)
|
|
tasks = res.fetchall()
|
|
|
|
if not tasks:
|
|
logger.info("😴 Minden adat kész. Alvás 5 perc...")
|
|
await asyncio.sleep(300)
|
|
continue
|
|
|
|
for t_id, make, model in tasks:
|
|
logger.info(f"🧪 Gazdagítás: {make} | {model}")
|
|
tech = await cls.fetch_tech_data(make, model)
|
|
|
|
if tech:
|
|
# RDW mezők kinyerése
|
|
kw = tech.get("netto_maximum_vermogen_kw")
|
|
ccm = tech.get("cilinderinhoud")
|
|
weight = tech.get("technisch_toelaatbare_maximum_massa")
|
|
axles = tech.get("aantal_assen")
|
|
euro = tech.get("milieuklasse_eg_goedkeuring_licht")
|
|
fuel = tech.get("brandstof_omschrijving_brandstof_stam", "Standard")
|
|
|
|
# Biztonságos konverzió
|
|
def clean_num(v):
|
|
try: return int(float(v)) if v else None
|
|
except: return None
|
|
|
|
update_query = text("""
|
|
UPDATE data.vehicle_catalog
|
|
SET fuel_type = :fuel,
|
|
power_kw = :kw,
|
|
engine_capacity = :ccm,
|
|
max_weight_kg = :weight,
|
|
axle_count = :axles,
|
|
euro_class = :euro,
|
|
factory_data = factory_data || jsonb_build_object('enriched_at', :now)
|
|
WHERE id = :id
|
|
""")
|
|
|
|
await db.execute(update_query, {
|
|
"fuel": fuel, "kw": clean_num(kw), "ccm": clean_num(ccm),
|
|
"weight": clean_num(weight), "axles": clean_num(axles),
|
|
"euro": str(euro) if euro else None,
|
|
"id": t_id, "now": str(datetime.datetime.now())
|
|
})
|
|
await db.commit()
|
|
logger.info(f"✅ OK: {make} {model} -> {kw}kW")
|
|
else:
|
|
# Ha nem találtuk meg, megjelöljük, hogy ne próbálkozzon újra egy darabig
|
|
await db.execute(text("UPDATE data.vehicle_catalog SET fuel_type = 'No-Tech-V4' WHERE id = :id"), {"id": t_id})
|
|
await db.commit()
|
|
|
|
await asyncio.sleep(0.5)
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(TechEnricher.run()) |