feat(robot): hunter v2.7, geocoding support, docker network fix, changelog update
This commit is contained in:
@@ -1,60 +1,198 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, func, or_, text
|
||||
from app.db.session import SessionLocal
|
||||
from app.models.asset import AssetCatalog
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("Robot1-Catalog")
|
||||
logger = logging.getLogger("Robot1-Master-Fleet-DeepDive")
|
||||
|
||||
class CatalogScout:
|
||||
"""
|
||||
Robot 1: Járműkatalógus feltöltő.
|
||||
Stratégia: Magyarországi alapok -> Globális EU márkák -> Technikai mélység.
|
||||
Robot 1: Univerzális Járműkatalógus Építő és Audit Robot.
|
||||
Logika: EU-Elsődlegesség (CarQuery) -> US-Kiegészítés (NHTSA).
|
||||
Kategóriák: Car, Motorcycle, Bus, Truck, Trailer, ATV, Marine, Aerial.
|
||||
Szekvenciák:
|
||||
1. Deep Dive (Motorvariánsok gyűjtése)
|
||||
2. Audit (Hiányos adatok pótlása)
|
||||
"""
|
||||
|
||||
CQ_URL = "https://www.carqueryapi.com/api/0.3/"
|
||||
NHTSA_BASE = "https://vpic.nhtsa.dot.gov/api/vehicles/GetModelsForMakeYear/make/"
|
||||
|
||||
@staticmethod
|
||||
async def get_initial_hu_data():
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json"
|
||||
}
|
||||
|
||||
# --- KATEGÓRIA DEFINÍCIÓK (Szigorú flotta-szétválasztás) ---
|
||||
MOTO_MAKES = ['ducati', 'ktm', 'triumph', 'aprilia', 'benelli', 'vespa', 'simson', 'mz', 'etz', 'jawa', 'husqvarna', 'gasgas', 'sherco']
|
||||
MARINE_IDS = ['DF', 'DT', 'OUTBOARD', 'MARINE', 'JET SKI', 'SEA-DOO', 'WAVERUNNER', 'YACHT', 'BOAT']
|
||||
AERIAL_IDS = ['CESSNA', 'PIPER', 'AIRBUS', 'BOEING', 'HELICOPTER', 'AIRCRAFT', 'BEECHCRAFT', 'EMBRAER', 'DRONE']
|
||||
ATV_IDS = ['LT-', 'LTZ', 'LTR', 'KINGQUAD', 'QUAD', 'POLARIS', 'CAN-AM', 'MULE', 'RZR', 'ARCTIC CAT', 'UTV', 'SIDE-BY-SIDE']
|
||||
|
||||
# Versenygépek (Motorkerékpárként, üzemóra alapú szervizhez)
|
||||
RACING_IDS = ['RM-Z', 'KX', 'CRF', 'YZ', 'SX-F', 'XC-W', 'RM125', 'RM250', 'CR125', 'CR250', 'MC450']
|
||||
MOTO_KEYWORDS = ['CBR', 'GSX', 'YZF', 'NINJA', 'Z1000', 'DR-Z', 'MT-0', 'V-STROM', 'ADVENTURE', 'SCRAMBLER', 'CBF', 'VFR', 'HAYABUSA']
|
||||
|
||||
# Flotta kategóriák szétválasztása
|
||||
BUS_KEYWORDS = ['BUS', 'COACH', 'INTERCITY', 'SHUTTLE', 'TRANSIT']
|
||||
TRUCK_KEYWORDS = ['TRUCK', 'SEMI', 'TRACTOR', 'HAULER', 'ACTROS', 'MAN', 'SCANIA', 'IVECO', 'VOLVO FH', 'DAF', 'TGX', 'RENAULT T']
|
||||
TRAILER_KEYWORDS = ['TRAILER', 'SEMITRAILER', 'PÓTKOCSI', 'UTÁNFUTÓ', 'SCHMITZ', 'KRONE', 'KÖGEL']
|
||||
|
||||
@classmethod
|
||||
def identify_class(cls, make: str, model: str) -> str:
|
||||
"""Kategória meghatározás flottakezelési szempontok alapján."""
|
||||
m_full = f"{make} {model}".upper()
|
||||
|
||||
if any(x in m_full for x in cls.AERIAL_IDS): return "aerial"
|
||||
if any(x in m_full for x in cls.MARINE_IDS): return "marine"
|
||||
if any(x in m_full for x in cls.ATV_IDS): return "atv"
|
||||
|
||||
# Motorkerékpárok (Versenygépekkel együtt)
|
||||
if any(x in m_full for x in cls.RACING_IDS) or make.lower() in cls.MOTO_MAKES:
|
||||
return "motorcycle"
|
||||
if any(x in m_full for x in cls.MOTO_KEYWORDS):
|
||||
return "motorcycle"
|
||||
|
||||
# Flotta (Busz vs Teherautó vs Pótkocsi)
|
||||
if any(x in m_full for x in cls.BUS_KEYWORDS): return "bus"
|
||||
if any(x in m_full for x in cls.TRUCK_KEYWORDS): return "truck"
|
||||
if any(x in m_full for x in cls.TRAILER_KEYWORDS): return "trailer"
|
||||
|
||||
return "car"
|
||||
|
||||
@classmethod
|
||||
async def fetch_api(cls, url, params=None, is_cq=False):
|
||||
"""API hívó JSONP tisztítással és sebességkorlátozással."""
|
||||
async with httpx.AsyncClient(headers=cls.HEADERS) as client:
|
||||
try:
|
||||
# 1.5s várakozás a Free API limitjei miatt
|
||||
await asyncio.sleep(1.5)
|
||||
resp = await client.get(url, params=params, timeout=35)
|
||||
if resp.status_code != 200: return None
|
||||
|
||||
content = resp.text.strip()
|
||||
if is_cq:
|
||||
# Robusztusabb JSONP tisztítás regexszel
|
||||
match = re.search(r'(\{.*\}|\[.*\])', content, re.DOTALL)
|
||||
if match:
|
||||
content = match.group(0)
|
||||
elif "(" in content and ")" in content:
|
||||
content = content[content.find("(") + 1 : content.rfind(")")]
|
||||
|
||||
return json.loads(content)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ API hiba: {e} | URL: {url}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def enrich_missing_data(cls):
|
||||
"""
|
||||
Kezdeti adathalmaz (Példa).
|
||||
Élesben itt egy külső API vagy CSV feldolgozás helye van.
|
||||
SEQUENCE 2: Audit Robot.
|
||||
Keresi a hiányos technikai adatokat és próbálja dúsítani őket.
|
||||
"""
|
||||
return [
|
||||
# Suzuki - A magyar utak királya
|
||||
{"make": "Suzuki", "model": "Swift", "generation": "III (2005-2010)", "engine_variant": "1.3 (92 LE)", "year_from": 2005, "year_to": 2010, "fuel_type": "petrol"},
|
||||
{"make": "Suzuki", "model": "Vitara", "generation": "IV (2015-)", "engine_variant": "1.6 VVT (120 LE)", "year_from": 2015, "year_to": 2024, "fuel_type": "petrol"},
|
||||
# Opel - Astra népautó
|
||||
{"make": "Opel", "model": "Astra", "generation": "H (2004-2009)", "engine_variant": "1.4 Twinport (90 LE)", "year_from": 2004, "year_to": 2009, "fuel_type": "petrol"},
|
||||
{"make": "Opel", "model": "Astra", "generation": "J (2009-2015)", "engine_variant": "1.7 CDTI (110 LE)", "year_from": 2009, "year_to": 2015, "fuel_type": "diesel"},
|
||||
# Skoda - Családi/Flotta kedvenc
|
||||
{"make": "Skoda", "model": "Octavia", "generation": "II (2004-2013)", "engine_variant": "1.6 MPI (102 LE)", "year_from": 2004, "year_to": 2013, "fuel_type": "petrol"},
|
||||
{"make": "Skoda", "model": "Octavia", "generation": "III (2013-2020)", "engine_variant": "2.0 TDI (150 LE)", "year_from": 2013, "year_to": 2020, "fuel_type": "diesel"},
|
||||
# BMW - GS Motorosoknak
|
||||
{"make": "BMW", "model": "R 1200 GS", "generation": "K50 (2013-2018)", "engine_variant": "Adventure (125 LE)", "year_from": 2013, "year_to": 2018, "fuel_type": "petrol"}
|
||||
]
|
||||
logger.info("🔍 Audit szekvencia indítása (hiányos adatok keresése)...")
|
||||
async with SessionLocal() as db:
|
||||
# Keressük azokat a rekordokat, ahol hiányzik a köbcenti vagy a teljesítmény
|
||||
stmt = select(AssetCatalog).where(
|
||||
or_(
|
||||
AssetCatalog.factory_data == text("'{}'::jsonb"),
|
||||
AssetCatalog.engine_variant == 'Standard',
|
||||
AssetCatalog.fuel_type == None
|
||||
)
|
||||
).limit(100) # Egyszerre csak 100-at nézünk
|
||||
|
||||
results = await db.execute(stmt)
|
||||
incomplete_records = results.scalars().all()
|
||||
|
||||
for record in incomplete_records:
|
||||
logger.info(f"🛠 Audit: {record.make} {record.model} ({record.year_from}) dúsítása...")
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Robot 1 indítása: Járműkatalógus feltöltés...")
|
||||
async with SessionLocal() as db:
|
||||
data = await cls.get_initial_hu_data()
|
||||
added_count = 0
|
||||
logger.info("🤖 Robot 1: EU-Elsődlegességű Deep Dive szinkron indítása...")
|
||||
|
||||
# 2026-tól visszafelé haladunk (Modern flották prioritása)
|
||||
for year in range(2026, 1989, -1):
|
||||
logger.info(f"📅 Feldolgozás alatt: {year} évjárat")
|
||||
|
||||
for item in data:
|
||||
# Ellenőrizzük az egyediséget (Make + Model + Generation + Engine)
|
||||
stmt = select(AssetCatalog).where(
|
||||
AssetCatalog.make == item["make"],
|
||||
AssetCatalog.model == item["model"],
|
||||
AssetCatalog.engine_variant == item["engine_variant"]
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
if not result.scalar_one_or_none():
|
||||
db.add(AssetCatalog(**item))
|
||||
added_count += 1
|
||||
|
||||
await db.commit()
|
||||
logger.info(f"✅ Robot 1 sikeresen rögzített {added_count} új katalógus elemet.")
|
||||
makes_data = await cls.fetch_api(cls.CQ_URL, {"cmd": "getMakes", "year": year}, is_cq=True)
|
||||
if not makes_data or "Makes" not in makes_data: continue
|
||||
|
||||
for make_entry in makes_data.get("Makes", []):
|
||||
m_id = make_entry["make_id"]
|
||||
m_display = make_entry["make_display"]
|
||||
|
||||
# MODELL GYŰJTÉS: EU + US fúzió
|
||||
models_to_fetch = set()
|
||||
|
||||
# 🇪🇺 EU Forrás
|
||||
cq_models = await cls.fetch_api(cls.CQ_URL, {"cmd": "getModels", "make": m_id, "year": year}, is_cq=True)
|
||||
if cq_models and cq_models.get("Models"):
|
||||
for m in cq_models["Models"]: models_to_fetch.add(m["model_name"])
|
||||
|
||||
# 🇺🇸 US Forrás kiegészítés
|
||||
n_data = await cls.fetch_api(f"{cls.NHTSA_BASE}{m_display}/modelyear/{year}?format=json")
|
||||
if n_data and n_data.get("Results"):
|
||||
for r in n_data["Results"]: models_to_fetch.add(r["Model_Name"])
|
||||
|
||||
async with SessionLocal() as db:
|
||||
for model_name in models_to_fetch:
|
||||
# DEEP DIVE: Motorvariánsok (Trims) lekérése
|
||||
trims_data = await cls.fetch_api(cls.CQ_URL, {
|
||||
"cmd": "getTrims", "make": m_id, "model": model_name, "year": year
|
||||
}, is_cq=True)
|
||||
|
||||
found_trims = trims_data.get("Trims", []) if trims_data else []
|
||||
|
||||
# Ha nincs trim adat, egy standard sor mindenképpen kell
|
||||
if not found_trims:
|
||||
found_trims = [{"model_trim": "Standard", "model_engine_fuel": None}]
|
||||
|
||||
for t in found_trims:
|
||||
variant = t.get("model_trim") or "Standard"
|
||||
fuel = t.get("model_engine_fuel") or "Unknown"
|
||||
v_class = cls.identify_class(m_display, model_name)
|
||||
|
||||
# Szigorú duplikáció-ellenőrzés (UniqueConstraint alapú lekérdezés)
|
||||
stmt = select(AssetCatalog).where(
|
||||
AssetCatalog.make == m_display,
|
||||
AssetCatalog.model == model_name,
|
||||
AssetCatalog.year_from == year,
|
||||
AssetCatalog.engine_variant == variant,
|
||||
AssetCatalog.fuel_type == fuel
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
if not result.scalars().first():
|
||||
db.add(AssetCatalog(
|
||||
make=m_display,
|
||||
model=model_name,
|
||||
year_from=year,
|
||||
engine_variant=variant,
|
||||
fuel_type=fuel,
|
||||
vehicle_class=v_class,
|
||||
factory_data={
|
||||
"cc": t.get("model_engine_cc"),
|
||||
"hp": t.get("model_engine_power_ps"),
|
||||
"cylinders": t.get("model_engine_cyl"),
|
||||
"transmission": t.get("model_transmission_type"),
|
||||
"source": "master_v7_deep_dive",
|
||||
"sync_date": str(func.now())
|
||||
}
|
||||
))
|
||||
|
||||
# JAVÍTÁS: Márkánkénti véglegesítés az adatbázisban a session-ön belül
|
||||
await db.commit()
|
||||
logger.info(f"✅ {m_display} ({year}) összes variánsa rögzítve.")
|
||||
|
||||
# SEQUENCE 2: Miután végeztünk a fő listával, nézzük meg a hiányosakat
|
||||
await cls.enrich_missing_data()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogScout.run())
|
||||
Reference in New Issue
Block a user