kincses/service-finder
1
0
Fork 0
Code Issues 7 Pull Requests Packages Projects 2 Releases Wiki Activity

feat: v1.7 overhaul - identity hash, triple wallet, financial ledger, and security audit system

Browse Source
This commit is contained in:
Kincses
2026-02-16 00:42:49 +00:00
parent bb02d4ed59
commit d574d3297d
63 changed files with 3710 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
backend/app/workers/catalog_robot.py
View File

@@ -2,178 +2,207 @@ import asyncio
import httpx
import logging
import json
import re
import os
import datetime
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func, or_, text
from sqlalchemy import text
from app.db.session import SessionLocal
from app.models.asset import AssetCatalog
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Robot1-Ghost-Commander-v1.1.9")
logger = logging.getLogger("Robot-v1.0.13-Global-Hunter")
class CatalogScout:
class CatalogMaster:
"""
Robot 1.1.9: Environment Master.
- .env alapú hitelesítés (RDW App Token)
- Prioritás: RDW (EU) -> NHTSA (US) -> CarQuery (Ban-figyeléssel)
- 2.5s lekérési frissítés a biztonságért
Master Hunter Robot v1.0.13 - Global Hunter Edition
- Holland (RDW), Brit (DVLA) és Amerikai (NHTSA) adatbázis integráció.
- Ratio-Filter: Kiszűri a 0.19-es kW/kg arányszámokat.
- Multi-field Power Discovery: Minden lehetséges mezőből kinyeri a kW-ot.
- Dinamikus évjárat kezelés a duplikációk ellen.
"""
CQ_URL = "https://www.carqueryapi.com/api/0.3/"
NHTSA_BASE = "https://vpic.nhtsa.dot.gov/api/vehicles/GetModelsForMakeYear/make/"
RDW_URL = "https://opendata.rdw.nl/resource/ed7h-m8uz.json"
# API Végpontok
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
RDW_AXLE = "https://opendata.rdw.nl/resource/3huj-srit.json"
RDW_BODY = "https://opendata.rdw.nl/resource/vezc-m2t6.json"
# Adatok beolvasása környezeti változókból
UK_DVLA = "https://driver-vehicle-licensing.api.gov.uk/vehicle-enquiry/v1/vehicles"
US_NHTSA = "https://vpic.nhtsa.dot.gov/api/vehicles/DecodeVinValuesBatch/"
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
UK_API_KEY = os.getenv("UK_DVLA_API_KEY")
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json",
"X-App-Token": RDW_TOKEN
HEADERS_RDW = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
HEADERS_UK = {"x-api-key": UK_API_KEY, "Content-Type": "application/json"} if UK_API_KEY else {}
CATEGORY_MAP = {
"Personenauto": "car",
"Motorfiets": "motorcycle",
"Bedrijfsauto": "truck",
"Vrachtwagen": "truck",
"Opleggertrekker": "truck",
"Bus": "bus",
"Aanhangwagen": "trailer",
"Oplegger": "trailer",
"Landbouw- of bosbouwtrekker": "agricultural",
"camper": "camper"
}
# BAN FIGYELŐ ÁLLAPOT
cq_banned_until = None
# --- KATEGÓRIA DEFINÍCIÓK (Szigorúan az eredeti lista szerint) ---
MOTO_MAKES = ['ducati', 'ktm', 'triumph', 'aprilia', 'benelli', 'vespa', 'simson', 'mz', 'etz', 'jawa', 'husqvarna', 'gasgas', 'sherco']
MARINE_IDS = ['DF', 'DT', 'OUTBOARD', 'MARINE', 'JET SKI', 'SEA-DOO', 'WAVERUNNER', 'YACHT', 'BOAT']
AERIAL_IDS = ['CESSNA', 'PIPER', 'AIRBUS', 'BOEING', 'HELICOPTER', 'AIRCRAFT', 'BEECHCRAFT', 'EMBRAER', 'DRONE']
ATV_IDS = ['LT-', 'LTZ', 'LTR', 'KINGQUAD', 'QUAD', 'POLARIS', 'CAN-AM', 'MULE', 'RZR', 'ARCTIC CAT', 'UTV', 'SIDE-BY-SIDE']
RACING_IDS = ['RM-Z', 'KX', 'CRF', 'YZ', 'SX-F', 'XC-W', 'RM125', 'RM250', 'CR125', 'CR250', 'MC450']
MOTO_KEYWORDS = ['CBR', 'GSX', 'YZF', 'NINJA', 'Z1000', 'DR-Z', 'MT-0', 'V-STROM', 'ADVENTURE', 'SCRAMBLER', 'CBF', 'VFR', 'HAYABUSA']
BUS_KEYWORDS = ['BUS', 'COACH', 'INTERCITY', 'SHUTTLE', 'TRANSIT']
TRUCK_KEYWORDS = ['TRUCK', 'SEMI', 'TRACTOR', 'HAULER', 'ACTROS', 'MAN', 'SCANIA', 'IVECO', 'VOLVO FH', 'DAF', 'TGX', 'RENAULT T']
TRAILER_KEYWORDS = ['TRAILER', 'SEMITRAILER', 'PÓTKOCSI', 'UTÁNFUTÓ', 'SCHMITZ', 'KRONE', 'KÖGEL']
FALLBACK_BRANDS = ['Audi', 'BMW', 'Mercedes-Benz', 'Volkswagen', 'Toyota', 'Ford', 'Honda', 'Hyundai', 'Kia', 'Mazda', 'Nissan', 'Volvo', 'Skoda', 'Opel', 'Tesla', 'Lexus', 'Porsche', 'Dacia', 'Suzuki']
@classmethod
def identify_class(cls, make: str, model: str) -> str:
m_full = f"{str(make)} {str(model)}".upper()
if any(x in m_full for x in cls.AERIAL_IDS): return "aerial"
if any(x in m_full for x in cls.MARINE_IDS): return "marine"
if any(x in m_full for x in cls.ATV_IDS): return "atv"
if any(x in m_full or str(make).lower() in cls.MOTO_MAKES for x in (cls.RACING_IDS + cls.MOTO_KEYWORDS)):
return "motorcycle"
if any(x in m_full for x in cls.BUS_KEYWORDS): return "bus"
if any(x in m_full for x in cls.TRUCK_KEYWORDS): return "truck"
if any(x in m_full for x in cls.TRAILER_KEYWORDS): return "trailer"
return "car"
def clean_kw(cls, val):
"""Speciális kW tisztító: ignorálja az 1.0 alatti arányszámokat."""
try:
if val is None: return None
f_val = float(str(val).replace(',', '.'))
if 0 < f_val < 1.0: return None # Ez csak arányszám (kW/kg)
v = int(f_val)
return v if v > 0 else None
except (ValueError, TypeError):
return None
@classmethod
async def fetch_api(cls, url, params=None, is_cq=False):
if is_cq and cls.cq_banned_until and datetime.datetime.now() < cls.cq_banned_until:
return "SILENT_SKIP"
def clean_int(cls, val):
"""Általános egész szám tisztító."""
try:
if val is None: return None
return int(float(str(val).replace(',', '.')))
except (ValueError, TypeError):
return None
async with httpx.AsyncClient(headers=cls.HEADERS, follow_redirects=True) as client:
@classmethod
async def fetch_api(cls, url, params=None, headers=None, method="GET", json_data=None):
"""Univerzális API hívó sebességkorlátozással."""
async with httpx.AsyncClient(headers=headers, follow_redirects=True) as client:
try:
# CarQuery: 5.0mp szünet (Hard Ban ellen), többi: 2.5mp (User kérése szerint)
await asyncio.sleep(5.0 if is_cq else 2.5)
resp = await client.get(url, params=params, timeout=35)
if resp.status_code == 403 or "denied" in resp.text.lower():
logger.error("🚫 CarQuery BAN! 2 óra kényszerpihenő aktiválva.")
cls.cq_banned_until = datetime.datetime.now() + datetime.timedelta(hours=2)
return "DENIED"
if resp.status_code != 200: return None
content = resp.text.strip()
if is_cq:
match = re.search(r'(\{.*\}|\[.*\])', content, re.DOTALL)
if match: content = match.group(0)
return json.loads(content)
await asyncio.sleep(1.2) # Biztonsági késleltetés
if method == "POST":
resp = await client.post(url, json=json_data, timeout=30)
else:
resp = await client.get(url, params=params, timeout=30)
return resp.json() if resp.status_code in [200, 201] else []
except Exception as e:
logger.error(f"❌ API hiba: {e}")
return None
logger.error(f"❌ API Hiba ({url}): {e}")
return []
@classmethod
async def is_model_processed(cls, db: AsyncSession, make: str, model: str, year: int):
stmt = select(AssetCatalog.id).where(AssetCatalog.make == make, AssetCatalog.model == model, AssetCatalog.year_from == year).limit(1)
result = await db.execute(stmt)
return result.scalars().first() is not None
async def get_deep_tech(cls, plate, main_kw=None, vin=None):
"""Nemzetközi dúsítás: Holland -> Brit -> Amerikai sorrendben."""
res = {"kw": cls.clean_kw(main_kw), "fuel": "Unknown", "axles": None, "body": "Standard", "euro": None}
# 1. HOLLAND (RDW) DÚSÍTÁS
fuel_data = await cls.fetch_api(cls.RDW_FUEL, {"kenteken": plate}, headers=cls.HEADERS_RDW)
if fuel_data:
f0 = fuel_data[0]
if not res["kw"]:
res["kw"] = cls.clean_kw(f0.get("nettomaximumvermogen") or f0.get("netto_maximum_vermogen"))
res["fuel"] = f0.get("brandstof_omschrijving", "Unknown")
res["euro"] = f0.get("uitlaatemissieniveau")
# 2. BRIT (DVLA) ELLENŐRZÉS (Ha van UK kulcs és még hiányzik adat)
if cls.UK_API_KEY and (not res["kw"] or not res["euro"]):
uk_data = await cls.fetch_api(cls.UK_DVLA, method="POST", json_data={"registrationNumber": plate}, headers=cls.HEADERS_UK)
if uk_data:
res["kw"] = res["kw"] or cls.clean_kw(uk_data.get("engineCapacity")) # Brit adatok finomítása
res["euro"] = res["euro"] or uk_data.get("euroStatus")
# 3. AMERIKAI (NHTSA) KUTATÁS (Ha van alvázszám)
if vin and len(vin) == 17:
us_data = await cls.fetch_api(cls.US_NHTSA, params={"format": "json", "data": vin})
if us_data and "Results" in us_data:
# Az amerikai adatbázisból kinyerjük a lóerőt (HP), ha a kW még mindig nincs meg
hp = us_data["Results"][0].get("EngineHP")
if hp and not res["kw"]:
res["kw"] = int(float(hp) * 0.7457) # HP -> kW konverzió
# RDW Extra adatok (Tengely, Karosszéria)
axle = await cls.fetch_api(cls.RDW_AXLE, {"kenteken": plate}, headers=cls.HEADERS_RDW)
if axle: res["axles"] = cls.clean_int(axle[0].get("aantal_assen"))
body = await cls.fetch_api(cls.RDW_BODY, {"kenteken": plate}, headers=cls.HEADERS_RDW)
if body: res["body"] = body[0].get("carrosserie_omschrijving", "Standard")
return res
@classmethod
async def auto_heal(cls, db: AsyncSession, cq_active: bool):
logger.info("🛠️ Auto-Heal: Hiányos rekordok dúsítása...")
stmt = select(AssetCatalog).where(AssetCatalog.engine_variant == 'Standard', AssetCatalog.fuel_type == 'Unknown').limit(20)
results = await db.execute(stmt)
for r in results.scalars().all():
# 1. RDW javítás (Holland Open Data + Token)
rdw = await cls.fetch_api(cls.RDW_URL, {"merk": r.make.upper(), "handelsbenaming": r.model.upper(), "$limit": 1})
if rdw and isinstance(rdw, list) and len(rdw) > 0:
item = rdw[0]
r.fuel_type = item.get("brandstof_omschrijving", "Unknown")
r.factory_data.update({"hp": item.get("netto_maximum_vermogen"), "cc": item.get("cilinderinhoud"), "source": "heal_v1.9_rdw"})
async def process_make(cls, db, task_id, make_name):
logger.info(f"🚀 >>> {make_name} GlobalHunter v1.0.13 INDUL...")
offset, limit, total_saved = 0, 1000, 0
unique_variants = {}
while True:
params = {"merk": make_name.upper(), "$limit": limit, "$offset": offset}
main_data = await cls.fetch_api(cls.RDW_MAIN, params, headers=cls.HEADERS_RDW)
if not main_data: break
for item in main_data:
plate = item.get("kenteken")
if not plate: continue
model = str(item.get("handelsbenaming", "Unknown")).upper()
ccm = cls.clean_int(item.get("cilinderinhoud"))
weight = cls.clean_int(item.get("massa_ledig_voertuig") or item.get("massa_rijklaar"))
kw_candidate = item.get("netto_maximum_vermogen") or item.get("vermogen_massarijklaar")
raw_date = item.get("datum_eerste_toelating")
prod_year = int(str(raw_date)[:4]) if raw_date else 2024
v_class = cls.CATEGORY_MAP.get(item.get("voertuigsoort"), "other")
if "kampeerwagen" in str(item.get("inrichting", "")).lower(): v_class = "camper"
# Variáns kulcs: Modell + CCM + Súly + kW + Év = Egyedi technikai ujjlenyomat
variant_key = f"{model}-{ccm}-{weight}-{v_class}-{kw_candidate}-{prod_year}"
if variant_key not in unique_variants:
unique_variants[variant_key] = {
"model": model, "ccm": ccm, "weight": weight, "v_class": v_class,
"plate": plate, "main_kw": kw_candidate, "prod_year": prod_year,
"vin": item.get("vin") # Ha az RDW-ben benne van a VIN
}
if len(main_data) < limit or offset > 90000: break
offset += limit
logger.info(f"📊 {len(unique_variants)} egyedi variáns kutatása indul...")
for key, v in unique_variants.items():
deep = await cls.get_deep_tech(v["plate"], main_kw=v["main_kw"], vin=v["vin"])
try:
db_item = AssetCatalog(
make=make_name.upper(), model=v["model"], vehicle_class=v["v_class"],
fuel_type=deep["fuel"], power_kw=deep["kw"], engine_capacity=v["ccm"],
max_weight_kg=v["weight"], axle_count=deep["axles"], body_type=deep["body"],
year_from=v["prod_year"], euro_class=deep["euro"],
factory_data={
"source": "GlobalHunter-v1.0.13",
"sample_plate": v["plate"],
"enriched_at": str(datetime.datetime.now())
}
)
db.add(db_item)
await db.commit()
total_saved += 1
if total_saved % 50 == 0: logger.info(f"✅ {total_saved} variáns elmentve.")
except Exception:
await db.rollback()
continue
# 2. CQ javítás (Ha nem vagyunk kitiltva)
if cq_active:
t_data = await cls.fetch_api(cls.CQ_URL, {"cmd": "getTrims", "make": r.make.lower(), "model": r.model, "year": r.year_from}, is_cq=True)
if t_data and t_data not in ["DENIED", "SILENT_SKIP"] and "Trims" in t_data:
t = t_data["Trims"][0]
r.engine_variant = t.get("model_trim") or "Standard"
r.factory_data.update({"hp": t.get("model_engine_power_ps"), "cc": t.get("model_engine_cc"), "source": "heal_v1.9_cq"})
await db.execute(text("UPDATE data.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task_id})
await db.commit()
logger.info(f"🏁 {make_name} KÉSZ. {total_saved} rekord rögzítve.")
@classmethod
async def run(cls):
logger.info(f"🤖 Robot 1.9.2 indítása (RDW Token: {'Aktív' if cls.RDW_TOKEN else 'HIÁNYZIK!'})")
for year in range(2026, 1989, -1):
logger.info(f"📅 --- CIKLUS: {year} ---")
cq_now_active = not (cls.cq_banned_until and datetime.datetime.now() < cls.cq_banned_until)
logger.info("🤖 Robot 1.0.13 (Global Hunter) ONLINE")
while True:
async with SessionLocal() as db:
await cls.auto_heal(db, cq_now_active)
# 1. MÁRKALISTA (NHTSA + Fallback)
makes_to_process = []
for b in cls.FALLBACK_BRANDS:
makes_to_process.append({"id": b.lower(), "display": b})
for make in makes_to_process:
models_to_fetch = set()
# A: NHTSA (US)
n_data = await cls.fetch_api(f"{cls.NHTSA_BASE}{make['display']}/modelyear/{year}?format=json")
if n_data and n_data.get("Results"):
for r in n_data["Results"]: models_to_fetch.add(r["Model_Name"])
# B: RDW (Holland) - Tokennel védve
rdw_m = await cls.fetch_api(cls.RDW_URL, {"merk": make['display'].upper(), "$limit": 30})
if rdw_m and isinstance(rdw_m, list):
for r in rdw_m: models_to_fetch.add(r.get("handelsbenaming"))
async with SessionLocal() as db:
for model_name in models_to_fetch:
if not model_name or await cls.is_model_processed(db, make["display"], model_name, year):
continue
# C: CarQuery (Csak ha nincs ban)
found_trims = []
t_data = await cls.fetch_api(cls.CQ_URL, {"cmd": "getTrims", "make": make["id"], "model": model_name, "year": year}, is_cq=True)
if t_data and t_data not in ["DENIED", "SILENT_SKIP"] and "Trims" in t_data:
found_trims = t_data["Trims"]
if not found_trims:
found_trims = [{"model_trim": "Standard", "model_engine_fuel": "Unknown"}]
for t in found_trims:
db.add(AssetCatalog(
make=make["display"], model=model_name, year_from=year,
engine_variant=t.get("model_trim") or "Standard",
fuel_type=t.get("model_engine_fuel") or "Unknown",
vehicle_class=cls.identify_class(make["display"], model_name),
factory_data={
"hp": t.get("model_engine_power_ps"), "cc": t.get("model_engine_cc"),
"source": "ghost_v1.9.2", "sync_date": str(datetime.datetime.now())
}
))
await db.commit()
res = await db.execute(text("SELECT id, make FROM data.catalog_discovery WHERE status = 'pending' LIMIT 1"))
task = res.fetchone()
if task:
await cls.process_make(db, task[0], task[1])
else:
logger.info("😴 Várólista üres. Alvás 60 mp...")
await asyncio.sleep(60)
await asyncio.sleep(1)
if __name__ == "__main__":
asyncio.run(CatalogScout.run())
asyncio.run(CatalogMaster.run())