feat: v1.7 overhaul - identity hash, triple wallet, financial ledger, and security audit system
This commit is contained in:
Binary file not shown.
Binary file not shown.
@@ -11,35 +11,35 @@ from app.models.asset import Asset, AssetCost, AssetTelemetry
|
||||
from app.models.identity import User
|
||||
from app.services.cost_service import cost_service
|
||||
from app.schemas.asset_cost import AssetCostCreate, AssetCostResponse
|
||||
# --- IMPORT JAVÍTVA: Behozzuk a jármű sémát a dúsított adatokhoz ---
|
||||
from app.schemas.asset import AssetResponse
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# --- 1. MODUL: IDENTITÁS (Alapadatok) ---
|
||||
@router.get("/{asset_id}", response_model=Dict[str, Any])
|
||||
# --- 1. MODUL: IDENTITÁS (Alapadatok & Technikai katalógus) ---
|
||||
@router.get("/{asset_id}", response_model=AssetResponse)
|
||||
async def get_asset_identity(
|
||||
asset_id: uuid.UUID,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user)
|
||||
):
|
||||
"""Csak a jármű alapadatai és katalógus információi."""
|
||||
stmt = select(Asset).where(Asset.id == asset_id).options(selectinload(Asset.catalog))
|
||||
"""
|
||||
Visszaadja a jármű alapadatokat és a dúsított katalógus információkat (kW, CCM, tengelyek).
|
||||
A selectinload(Asset.catalog) biztosítja, hogy a technikai adatok is betöltődjenek.
|
||||
"""
|
||||
stmt = (
|
||||
select(Asset)
|
||||
.where(Asset.id == asset_id)
|
||||
.options(selectinload(Asset.catalog))
|
||||
)
|
||||
asset = (await db.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
if not asset:
|
||||
raise HTTPException(status_code=404, detail="Jármű nem található")
|
||||
|
||||
return {
|
||||
"id": asset.id,
|
||||
"vin": asset.vin,
|
||||
"license_plate": asset.license_plate,
|
||||
"name": asset.name,
|
||||
"catalog": {
|
||||
"make": asset.catalog.make,
|
||||
"model": asset.catalog.model,
|
||||
"type": asset.catalog.vehicle_class,
|
||||
"factory_data": getattr(asset.catalog, 'factory_data', {})
|
||||
}
|
||||
}
|
||||
# Közvetlenül az objektumot adjuk vissza, a Pydantic AssetResponse
|
||||
# modellje fogja formázni a kimenetet a dúsított adatokkal együtt.
|
||||
return asset
|
||||
|
||||
# --- 2. MODUL: PÉNZÜGY (Költségek) ---
|
||||
@router.get("/{asset_id}/costs", response_model=Dict[str, Any])
|
||||
@@ -89,11 +89,7 @@ async def create_asset_cost(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Új költség rögzítése.
|
||||
Automatikus: EUR konverzió, Telemetria frissítés, XP jóváírás.
|
||||
"""
|
||||
# Validáció: az asset_id-nak egyeznie kell a path-szal
|
||||
"""Új költség rögzítése automatikus EUR konverzióval."""
|
||||
if cost_in.asset_id != asset_id:
|
||||
raise HTTPException(status_code=400, detail="Asset ID mismatch")
|
||||
|
||||
|
||||
@@ -153,7 +153,7 @@ async def complete_kyc(
|
||||
):
|
||||
"""
|
||||
Step 2: KYC Aktiválás.
|
||||
Itt használjuk a get_current_user-t (nem active), mert a user még inaktív.
|
||||
It használjuk a get_current_user-t (nem active), mert a user még inaktív.
|
||||
"""
|
||||
user = await AuthService.complete_kyc(db, current_user.id, kyc_in)
|
||||
if not user:
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import hashlib
|
||||
import unicodedata
|
||||
import re
|
||||
|
||||
class VINValidator:
|
||||
@@ -44,4 +46,31 @@ class VINValidator:
|
||||
"country": countries.get(vin[0], "Ismeretlen"),
|
||||
"year_code": vin[9], # Modellév kódja
|
||||
"wmi": vin[0:3] # World Manufacturer Identifier
|
||||
}
|
||||
}
|
||||
|
||||
class IdentityNormalizer:
|
||||
@staticmethod
|
||||
def normalize_text(text: str) -> str:
|
||||
"""Tisztítja a szöveget: kisbetű, ékezetmentesítés, szóközök és jelek törlése."""
|
||||
if not text:
|
||||
return ""
|
||||
# 1. Kisbetűre alakítás
|
||||
text = text.lower().strip()
|
||||
# 2. Ékezetek eltávolítása (Unicode normalizálás)
|
||||
text = "".join(
|
||||
c for c in unicodedata.normalize('NFD', text)
|
||||
if unicodedata.category(c) != 'Mn'
|
||||
)
|
||||
# 3. Csak az angol ABC betűi és számok maradjanak
|
||||
return re.sub(r'[^a-z0-9]', '', text)
|
||||
|
||||
@classmethod
|
||||
def generate_person_hash(cls, last_name: str, first_name: str, mothers_name: str, birth_date: str) -> str:
|
||||
"""Létrehozza az egyedi SHA256 ujjlenyomatot a személyhez."""
|
||||
raw_combined = (
|
||||
cls.normalize_text(last_name) +
|
||||
cls.normalize_text(first_name) +
|
||||
cls.normalize_text(mothers_name) +
|
||||
cls.normalize_text(birth_date)
|
||||
)
|
||||
return hashlib.sha256(raw_combined.encode()).hexdigest()
|
||||
Binary file not shown.
@@ -1,10 +1,11 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/db/base.py
|
||||
from app.db.base_class import Base # noqa
|
||||
|
||||
# Közvetlen importok a fájlokból (Circular Import elkerülése)
|
||||
from app.models.address import Address, GeoPostalCode, GeoStreet, GeoStreetType # noqa
|
||||
from app.models.identity import User, Person, VerificationToken, Wallet # noqa
|
||||
from app.models.organization import Organization, OrganizationMember # noqa
|
||||
# Közvetlen importok (HOZZÁADVA az audit és sales modellek)
|
||||
from app.models.address import Address, GeoPostalCode, GeoStreet, GeoStreetType, Branch # noqa
|
||||
from app.models.identity import User, Person, VerificationToken, Wallet # noqa
|
||||
from app.models.organization import Organization, OrganizationMember, OrganizationSalesAssignment # noqa
|
||||
from app.models.audit import SecurityAuditLog, OperationalLog, FinancialLedger # noqa <--- KRITIKUS!
|
||||
from app.models.asset import ( # noqa
|
||||
Asset, AssetCatalog, AssetCost, AssetEvent,
|
||||
AssetFinancials, AssetTelemetry, AssetReview, ExchangeRate
|
||||
@@ -12,11 +13,11 @@ from app.models.asset import ( # noqa
|
||||
from app.models.gamification import ( # noqa
|
||||
PointRule, LevelConfig, UserStats, Badge, UserBadge, Rating, PointsLedger
|
||||
)
|
||||
from app.models.system_config import SystemParameter # noqa
|
||||
from app.models.system import SystemParameter # noqa (system.py használata)
|
||||
from app.models.history import AuditLog, VehicleOwnership # noqa
|
||||
from app.models.document import Document # noqa
|
||||
from app.models.translation import Translation # noqa <--- HOZZÁADVA
|
||||
from app.models.translation import Translation # noqa
|
||||
from app.models.core_logic import ( # noqa
|
||||
SubscriptionTier, OrganizationSubscription, CreditTransaction, ServiceSpecialty
|
||||
)
|
||||
from app.models.security import PendingAction # noqa <--- CSAK A BIZTONSÁG KEDVÉÉRT, HA EZ IS HIÁNYZOTT VOLNA
|
||||
from app.models.security import PendingAction # noqa
|
||||
@@ -1,11 +1,12 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/models/__init__.py
|
||||
|
||||
from app.db.base_class import Base
|
||||
|
||||
# Identitás és Jogosultság
|
||||
from .identity import User, Person, Wallet, UserRole, VerificationToken, SocialAccount
|
||||
|
||||
# Szervezeti struktúra
|
||||
from .organization import Organization, OrganizationMember
|
||||
# Szervezeti struktúra (HOZZÁADVA: OrganizationSalesAssignment)
|
||||
from .organization import Organization, OrganizationMember, OrganizationSalesAssignment
|
||||
|
||||
# Járművek és Eszközök (Digital Twin)
|
||||
from .asset import (
|
||||
@@ -13,24 +14,25 @@ from .asset import (
|
||||
AssetFinancials, AssetTelemetry, AssetReview, ExchangeRate
|
||||
)
|
||||
|
||||
# Szerviz és Szakértelem (ÚJ)
|
||||
# Szerviz és Szakértelem
|
||||
from .service import ServiceProfile, ExpertiseTag, ServiceExpertise, ServiceStaging
|
||||
|
||||
# Földrajzi adatok és Címek
|
||||
from .address import Address, GeoPostalCode, GeoStreet, GeoStreetType
|
||||
from .address import Address, GeoPostalCode, GeoStreet, GeoStreetType, Branch
|
||||
|
||||
# Gamification és Economy
|
||||
from .gamification import PointRule, LevelConfig, UserStats, Badge, UserBadge, Rating, PointsLedger
|
||||
|
||||
# Rendszerkonfiguráció és Alapok
|
||||
from .system_config import SystemParameter
|
||||
# Rendszerkonfiguráció (HASZNÁLJUK a frissített system.py-t!)
|
||||
from .system import SystemParameter
|
||||
from .document import Document
|
||||
from .translation import Translation
|
||||
|
||||
# Üzleti logika és Előfizetés
|
||||
from .core_logic import SubscriptionTier, OrganizationSubscription, CreditTransaction, ServiceSpecialty
|
||||
|
||||
# Naplózás és Biztonság
|
||||
# Naplózás és Biztonság (HOZZÁADVA: audit.py modellek)
|
||||
from .audit import SecurityAuditLog, OperationalLog, FinancialLedger # <--- KRITIKUS!
|
||||
from .history import AuditLog, VehicleOwnership
|
||||
from .security import PendingAction
|
||||
|
||||
@@ -42,16 +44,15 @@ ServiceRecord = AssetEvent
|
||||
|
||||
__all__ = [
|
||||
"Base", "User", "Person", "Wallet", "UserRole", "VerificationToken", "SocialAccount",
|
||||
"Organization", "OrganizationMember",
|
||||
"Organization", "OrganizationMember", "OrganizationSalesAssignment",
|
||||
"Asset", "AssetCatalog", "AssetCost", "AssetEvent", "AssetFinancials",
|
||||
"AssetTelemetry", "AssetReview", "ExchangeRate",
|
||||
"Address", "GeoPostalCode", "GeoStreet", "GeoStreetType",
|
||||
"PointRule", "LevelConfig", "UserStats", "Badge", "UserBadge", "Rating", "PointsLedger",
|
||||
"Address", "GeoPostalCode", "GeoStreet", "GeoStreetType", "Branch",
|
||||
"Point_Rule", "LevelConfig", "UserStats", "Badge", "UserBadge", "Rating", "PointsLedger",
|
||||
"SystemParameter", "Document", "Translation", "PendingAction",
|
||||
"SubscriptionTier", "OrganizationSubscription",
|
||||
"CreditTransaction", "ServiceSpecialty", "AuditLog", "VehicleOwnership",
|
||||
# --- SZERVIZ MODUL (Tisztítva) ---
|
||||
"SecurityAuditLog", "OperationalLog", "FinancialLedger", # <--- KRITIKUS!
|
||||
"ServiceProfile", "ExpertiseTag", "ServiceExpertise", "ServiceStaging",
|
||||
# --- ALIASOK ---
|
||||
"Vehicle", "UserVehicle", "VehicleCatalog", "ServiceRecord"
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,7 +1,9 @@
|
||||
import uuid
|
||||
from sqlalchemy import Column, String, Integer, ForeignKey, Text, DateTime, Float
|
||||
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
|
||||
from sqlalchemy.sql import func
|
||||
# Hozzáadva: Boolean, text, func
|
||||
from sqlalchemy import Column, String, Integer, ForeignKey, Text, DateTime, Float, Boolean, text, func
|
||||
# PostgreSQL specifikus típusok
|
||||
from sqlalchemy.dialects.postgresql import UUID as PG_UUID, JSONB
|
||||
from sqlalchemy.orm import relationship
|
||||
from app.db.base_class import Base
|
||||
|
||||
class GeoPostalCode(Base):
|
||||
@@ -45,4 +47,44 @@ class Address(Base):
|
||||
latitude = Column(Float)
|
||||
longitude = Column(Float)
|
||||
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
# Add to /app/models/address.py
|
||||
class Branch(Base):
|
||||
"""
|
||||
Telephely entitás. A fizikai helyszín, ahol a szolgáltatás vagy flotta-kezelés zajlik.
|
||||
Minden cégnek van legalább egy 'Main' telephelye.
|
||||
"""
|
||||
__tablename__ = "branches"
|
||||
__table_args__ = {"schema": "data"}
|
||||
|
||||
id = Column(PG_UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
organization_id = Column(Integer, ForeignKey("data.organizations.id"), nullable=False)
|
||||
address_id = Column(PG_UUID(as_uuid=True), ForeignKey("data.addresses.id"), nullable=True)
|
||||
|
||||
name = Column(String(100), nullable=False) # pl. "Központi iroda", "Dunakeszi Szerviz"
|
||||
is_main = Column(Boolean, default=False)
|
||||
|
||||
# Részletes címadatok (Denormalizált a gyors kereséshez)
|
||||
postal_code = Column(String(10), index=True)
|
||||
city = Column(String(100), index=True)
|
||||
street_name = Column(String(150))
|
||||
street_type = Column(String(50))
|
||||
house_number = Column(String(20))
|
||||
stairwell = Column(String(20))
|
||||
floor = Column(String(20))
|
||||
door = Column(String(20))
|
||||
hrsz = Column(String(50)) # Helyrajzi szám
|
||||
|
||||
# Telephely specifikus adatok
|
||||
opening_hours = Column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
branch_rating = Column(Float, default=0.0)
|
||||
|
||||
status = Column(String(30), default="active")
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
organization = relationship("Organization", back_populates="branches")
|
||||
address = relationship("Address")
|
||||
# Kapcsolat a szerviz értékelésekkel
|
||||
reviews = relationship("Rating", primaryjoin="and_(Branch.id==foreign(Rating.target_id), Rating.target_type=='branch')")
|
||||
@@ -24,6 +24,16 @@ class AssetCatalog(Base):
|
||||
year_to = Column(Integer)
|
||||
vehicle_class = Column(String)
|
||||
fuel_type = Column(String, index=True)
|
||||
|
||||
# --- ÚJ OSZLOPOK (Ezeket add hozzá!) ---
|
||||
power_kw = Column(Integer, index=True)
|
||||
engine_capacity = Column(Integer, index=True)
|
||||
max_weight_kg = Column(Integer)
|
||||
axle_count = Column(Integer)
|
||||
euro_class = Column(String(20))
|
||||
body_type = Column(String(100))
|
||||
# ---------------------------------------
|
||||
|
||||
engine_code = Column(String)
|
||||
factory_data = Column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
@@ -101,11 +111,17 @@ class AssetAssignment(Base):
|
||||
id = Column(PG_UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
asset_id = Column(PG_UUID(as_uuid=True), ForeignKey("data.assets.id"), nullable=False)
|
||||
organization_id = Column(Integer, ForeignKey("data.organizations.id"), nullable=False)
|
||||
|
||||
# ÚJ: Telephelyi hozzárendelés
|
||||
branch_id = Column(PG_UUID(as_uuid=True), ForeignKey("data.branches.id"), nullable=True)
|
||||
|
||||
assigned_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
released_at = Column(DateTime(timezone=True), nullable=True)
|
||||
status = Column(String(30), default="active")
|
||||
|
||||
asset = relationship("Asset", back_populates="assignments")
|
||||
organization = relationship("Organization")
|
||||
branch = relationship("Branch") # Új kapcsolat
|
||||
|
||||
class AssetEvent(Base):
|
||||
__tablename__ = "asset_events"
|
||||
@@ -144,4 +160,27 @@ class ExchangeRate(Base):
|
||||
id = Column(Integer, primary_key=True)
|
||||
base_currency = Column(String(3), default="EUR")
|
||||
target_currency = Column(String(3), unique=True)
|
||||
rate = Column(Numeric(18, 6), nullable=False)
|
||||
rate = Column(Numeric(18, 6), nullable=False)
|
||||
|
||||
class CatalogDiscovery(Base):
|
||||
"""
|
||||
Discovery tábla: Ide gyűjtjük a piaci 'neveket' (pl. Citroen C3).
|
||||
A Robot innen indulva keresi meg az összes létező technikai variánst.
|
||||
"""
|
||||
__tablename__ = "catalog_discovery"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
make = Column(String(100), nullable=False, index=True)
|
||||
model = Column(String(100), nullable=False, index=True)
|
||||
vehicle_class = Column(String(50), index=True) # car, motorcycle, truck, stb.
|
||||
source = Column(String(50)) # 'hasznaltauto', 'mobile.de'
|
||||
status = Column(String(20), server_default=text("'pending'"), index=True)
|
||||
attempts = Column(Integer, default=0)
|
||||
last_attempt = Column(DateTime(timezone=True))
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
# EGYESÍTETT __table_args__
|
||||
__table_args__ = (
|
||||
UniqueConstraint('make', 'model', 'vehicle_class', name='_make_model_class_uc'),
|
||||
{"schema": "data"}
|
||||
)
|
||||
@@ -1,16 +1,56 @@
|
||||
from sqlalchemy import Column, Integer, String, DateTime, JSON, ForeignKey, text
|
||||
from sqlalchemy import Column, Integer, String, DateTime, JSON, ForeignKey, text, Numeric, Boolean, BigInteger
|
||||
from sqlalchemy.sql import func
|
||||
from app.db.base_class import Base
|
||||
|
||||
class AuditLog(Base):
|
||||
__tablename__ = "audit_logs"
|
||||
__table_args__ = {"schema": "data"}
|
||||
class SecurityAuditLog(Base):
|
||||
""" Kiemelt biztonsági események és a 4-szem elv. """
|
||||
__tablename__ = "security_audit_logs"
|
||||
__table_args__ = {"schema": "data", "extend_existing": True}
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
id = Column(Integer, primary_key=True)
|
||||
action = Column(String(50)) # 'ROLE_CHANGE', 'MANUAL_CREDIT_ADJUST', 'SUB_EXTEND'
|
||||
|
||||
actor_id = Column(Integer, ForeignKey("data.users.id")) # Aki kezdeményezte
|
||||
target_id = Column(Integer, ForeignKey("data.users.id")) # Akivel történt
|
||||
|
||||
# 4-szem elv: csak akkor válik élessé, ha ez nem NULL
|
||||
confirmed_by_id = Column(Integer, ForeignKey("data.users.id"), nullable=True)
|
||||
is_critical = Column(Boolean, default=False) # Szuperadmin hívásoknál True
|
||||
|
||||
payload_before = Column(JSON)
|
||||
payload_after = Column(JSON)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
class OperationalLog(Base):
|
||||
""" Napi üzemi események (Operational). """
|
||||
__tablename__ = "operational_logs"
|
||||
__table_args__ = {"schema": "data", "extend_existing": True}
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True) # <--- EZ HIÁNYZOTT!
|
||||
user_id = Column(Integer, ForeignKey("data.users.id", ondelete="SET NULL"), nullable=True)
|
||||
action = Column(String(100), nullable=False) # pl. "LOGIN", "REGISTER", "DELETE_ASSET"
|
||||
resource_type = Column(String(50)) # pl. "User", "Asset", "Organization"
|
||||
action = Column(String(100), nullable=False) # pl. "ADD_VEHICLE", "UPDATE_COST"
|
||||
resource_type = Column(String(50)) # pl. "Asset", "Expense"
|
||||
resource_id = Column(String(100))
|
||||
details = Column(JSON, server_default=text("'{}'::jsonb"))
|
||||
ip_address = Column(String(45))
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
class FinancialLedger(Base):
|
||||
""" Minden pénz- és kreditmozgás központi naplója. """
|
||||
__tablename__ = "financial_ledger"
|
||||
__table_args__ = {"schema": "data", "extend_existing": True}
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(Integer, ForeignKey("data.users.id"))
|
||||
person_id = Column(BigInteger, ForeignKey("data.persons.id"))
|
||||
|
||||
amount = Column(Numeric(18, 4), nullable=False)
|
||||
currency = Column(String(10)) # 'HUF', 'CREDIT', 'COIN'
|
||||
|
||||
transaction_type = Column(String(50)) # 'PURCHASE', 'HUNTING_COMMISSION', 'FARMING_COMMISSION'
|
||||
|
||||
# Üzletkötői követhetőség
|
||||
related_agent_id = Column(Integer, ForeignKey("data.users.id"), nullable=True)
|
||||
|
||||
details = Column(JSON, server_default=text("'{}'::jsonb"))
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -9,29 +9,34 @@ from app.db.base_class import Base
|
||||
class UserRole(str, enum.Enum):
|
||||
superadmin = "superadmin"
|
||||
admin = "admin"
|
||||
region_admin = "region_admin"
|
||||
country_admin = "country_admin"
|
||||
moderator = "moderator"
|
||||
sales_agent = "sales_agent"
|
||||
user = "user"
|
||||
service = "service"
|
||||
service_owner = "service_owner"
|
||||
fleet_manager = "fleet_manager"
|
||||
driver = "driver"
|
||||
|
||||
class Person(Base):
|
||||
"""
|
||||
Természetes személy identitása.
|
||||
A bot által talált személyek is ide kerülnek (is_ghost=True).
|
||||
Azonosítás: Név + Anyja neve + Születési adatok alapján.
|
||||
Természetes személy identitása. A DNS szint.
|
||||
Itt tároljuk az örök adatokat, amik nem vesznek el account törléskor.
|
||||
"""
|
||||
__tablename__ = "persons"
|
||||
__table_args__ = {"schema": "data"}
|
||||
__table_args__ = {"schema": "data", "extend_existing": True}
|
||||
|
||||
id = Column(BigInteger, primary_key=True, index=True)
|
||||
id_uuid = Column(PG_UUID(as_uuid=True), default=uuid.uuid4, unique=True, nullable=False)
|
||||
address_id = Column(PG_UUID(as_uuid=True), ForeignKey("data.addresses.id"), nullable=True)
|
||||
|
||||
# --- KRITIKUS: EGYEDI AZONOSÍTÓ HASH (Normalizált adatokból) ---
|
||||
identity_hash = Column(String(64), unique=True, index=True, nullable=True)
|
||||
|
||||
last_name = Column(String, nullable=False)
|
||||
first_name = Column(String, nullable=False)
|
||||
phone = Column(String, nullable=True)
|
||||
|
||||
# --- TERMÉSZETES AZONOSÍTÓK (Azonosításhoz, nem publikus) ---
|
||||
mothers_last_name = Column(String)
|
||||
mothers_first_name = Column(String)
|
||||
birth_place = Column(String)
|
||||
@@ -40,8 +45,14 @@ class Person(Base):
|
||||
identity_docs = Column(JSON, server_default=text("'{}'::jsonb"))
|
||||
ice_contact = Column(JSON, server_default=text("'{}'::jsonb"))
|
||||
|
||||
is_active = Column(Boolean, default=False, nullable=False)
|
||||
is_ghost = Column(Boolean, default=True, nullable=False) # Bot találta = True, Regisztrált = False
|
||||
# --- ÖRÖK ADATOK (Person szint) ---
|
||||
lifetime_xp = Column(BigInteger, server_default=text("0"))
|
||||
penalty_points = Column(Integer, server_default=text("0")) # 0-3 szint
|
||||
social_reputation = Column(Numeric(3, 2), server_default=text("1.00")) # 1.00 = 100%
|
||||
|
||||
is_sales_agent = Column(Boolean, server_default=text("false"))
|
||||
is_active = Column(Boolean, default=True, nullable=False)
|
||||
is_ghost = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
|
||||
@@ -50,28 +61,39 @@ class Person(Base):
|
||||
memberships = relationship("OrganizationMember", back_populates="person")
|
||||
|
||||
class User(Base):
|
||||
"""
|
||||
Login entitás. Bármikor törölhető (GDPR), de Person-höz kötött.
|
||||
"""
|
||||
__tablename__ = "users"
|
||||
__table_args__ = {"schema": "data"}
|
||||
__table_args__ = {"schema": "data", "extend_existing": True}
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
email = Column(String, unique=True, index=True, nullable=False)
|
||||
hashed_password = Column(String, nullable=True)
|
||||
role = Column(Enum(UserRole), default=UserRole.user)
|
||||
is_active = Column(Boolean, default=False)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
|
||||
person_id = Column(BigInteger, ForeignKey("data.persons.id"), nullable=True)
|
||||
folder_slug = Column(String(12), unique=True, index=True)
|
||||
|
||||
refresh_token_hash = Column(String(255), nullable=True)
|
||||
two_factor_secret = Column(String(100), nullable=True)
|
||||
two_factor_enabled = Column(Boolean, default=False)
|
||||
# --- ELŐFIZETÉS ÉS VIP (Időkorlátos logika) ---
|
||||
subscription_plan = Column(String(30), server_default=text("'FREE'"))
|
||||
subscription_expires_at = Column(DateTime(timezone=True), nullable=True)
|
||||
is_vip = Column(Boolean, server_default=text("false"))
|
||||
|
||||
# --- REFERRAL ÉS SALES (Üzletkötői hálózat) ---
|
||||
referral_code = Column(String(20), unique=True)
|
||||
referred_by_id = Column(Integer, ForeignKey("data.users.id"), nullable=True)
|
||||
# Farming üzletkötő (Átruházható cégkezelő)
|
||||
current_sales_agent_id = Column(Integer, ForeignKey("data.users.id"), nullable=True)
|
||||
|
||||
is_active = Column(Boolean, default=False)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
folder_slug = Column(String(12), unique=True, index=True)
|
||||
|
||||
preferred_language = Column(String(5), server_default="hu")
|
||||
region_code = Column(String(5), server_default="HU")
|
||||
preferred_currency = Column(String(3), server_default="HUF")
|
||||
|
||||
scope_level = Column(String(30), server_default="individual")
|
||||
scope_level = Column(String(30), server_default="individual") # global, region, country, entity, individual
|
||||
scope_id = Column(String(50))
|
||||
custom_permissions = Column(JSON, server_default=text("'{}'::jsonb"))
|
||||
|
||||
@@ -79,18 +101,25 @@ class User(Base):
|
||||
|
||||
person = relationship("Person", back_populates="users")
|
||||
wallet = relationship("Wallet", back_populates="user", uselist=False)
|
||||
stats = relationship("UserStats", back_populates="user", uselist=False)
|
||||
ownership_history = relationship("VehicleOwnership", back_populates="user")
|
||||
owned_organizations = relationship("Organization", back_populates="owner")
|
||||
social_accounts = relationship("SocialAccount", back_populates="user", cascade="all, delete-orphan")
|
||||
|
||||
class Wallet(Base):
|
||||
__tablename__ = "wallets"; __table_args__ = {"schema": "data"}
|
||||
""" A 3-as felosztású pénztárca. """
|
||||
__tablename__ = "wallets"
|
||||
__table_args__ = {"schema": "data", "extend_existing": True}
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
user_id = Column(Integer, ForeignKey("data.users.id"), unique=True)
|
||||
coin_balance = Column(Numeric(18, 2), default=0.00); credit_balance = Column(Numeric(18, 2), default=0.00); currency = Column(String(3), default="HUF")
|
||||
|
||||
earned_credits = Column(Numeric(18, 4), server_default=text("0")) # Munka + Referral
|
||||
purchased_credits = Column(Numeric(18, 4), server_default=text("0")) # Vásárolt
|
||||
service_coins = Column(Numeric(18, 4), server_default=text("0")) # Csak hirdetésre!
|
||||
|
||||
currency = Column(String(3), default="HUF")
|
||||
user = relationship("User", back_populates="wallet")
|
||||
|
||||
# ... (VerificationToken és SocialAccount változatlan) ...
|
||||
|
||||
class VerificationToken(Base):
|
||||
__tablename__ = "verification_tokens"; __table_args__ = {"schema": "data"}
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
|
||||
@@ -61,6 +61,11 @@ class Organization(Base):
|
||||
status = Column(String(30), default="pending_verification")
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
|
||||
# --- ÚJ: Előfizetés és Méret korlátok ---
|
||||
subscription_plan = Column(String(30), server_default=text("'FREE'"), index=True)
|
||||
base_asset_limit = Column(Integer, server_default=text("1"))
|
||||
purchased_extra_slots = Column(Integer, server_default=text("0"))
|
||||
|
||||
notification_settings = Column(JSON, server_default=text("'{\"notify_owner\": true, \"alert_days_before\": [30, 15, 7, 1]}'::jsonb"))
|
||||
external_integration_config = Column(JSON, server_default=text("'{}'::jsonb"))
|
||||
|
||||
@@ -70,6 +75,10 @@ class Organization(Base):
|
||||
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
|
||||
|
||||
# --- ÚJ: Dual Twin Tulajdonjog logika ---
|
||||
# Individual esetén False, Business esetén True
|
||||
is_ownership_transferable = Column(Boolean, server_default=text("true"))
|
||||
|
||||
# Kapcsolatok
|
||||
assets = relationship("AssetAssignment", back_populates="organization", cascade="all, delete-orphan")
|
||||
@@ -77,6 +86,7 @@ class Organization(Base):
|
||||
owner = relationship("User", back_populates="owned_organizations")
|
||||
financials = relationship("OrganizationFinancials", back_populates="organization", cascade="all, delete-orphan")
|
||||
service_profile = relationship("ServiceProfile", back_populates="organization", uselist=False)
|
||||
branches = relationship("Branch", back_populates="organization", cascade="all, delete-orphan")
|
||||
|
||||
class OrganizationFinancials(Base):
|
||||
"""Cégek éves gazdasági adatai elemzéshez."""
|
||||
@@ -111,4 +121,15 @@ class OrganizationMember(Base):
|
||||
|
||||
organization = relationship("Organization", back_populates="members")
|
||||
user = relationship("User")
|
||||
person = relationship("Person", back_populates="memberships")
|
||||
person = relationship("Person", back_populates="memberships")
|
||||
|
||||
class OrganizationSalesAssignment(Base):
|
||||
"""Összeköti a céget az aktuális üzletkötővel a jutalék miatt."""
|
||||
__tablename__ = "org_sales_assignments"
|
||||
__table_args__ = {"schema": "data"}
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
organization_id = Column(Integer, ForeignKey("data.organizations.id"))
|
||||
agent_user_id = Column(Integer, ForeignKey("data.users.id")) # Ő kapja a Farming díjat
|
||||
assigned_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
is_active = Column(Boolean, default=True)
|
||||
@@ -86,12 +86,17 @@ class ServiceStaging(Base):
|
||||
name = Column(String, nullable=False, index=True)
|
||||
|
||||
# --- Strukturált cím adatok (A kérésedre bontva) ---
|
||||
postal_code = Column(String(10), nullable=True, index=True) # Irányítószám
|
||||
city = Column(String(100), nullable=True, index=True) # Település
|
||||
street = Column(String(255), nullable=True) # Utca és közterület jellege (pl. Diófa utca)
|
||||
house_number = Column(String(50), nullable=True) # Házszám, emelet, ajtó
|
||||
full_address = Column(String, nullable=True) # Az eredeti, egybefüggő cím (ha van)
|
||||
|
||||
postal_code = Column(String(10), index=True)
|
||||
city = Column(String(100), index=True)
|
||||
street_name = Column(String(150))
|
||||
street_type = Column(String(50)) # utca, út, tér...
|
||||
house_number = Column(String(20))
|
||||
stairwell = Column(String(20)) # lépcsőház
|
||||
floor = Column(String(20)) # emelet
|
||||
door = Column(String(20)) # ajtó
|
||||
hrsz = Column(String(50)) # helyrajzi szám
|
||||
|
||||
full_address = Column(String) # Eredeti string (audit célból)
|
||||
# --- Elérhetőségek ---
|
||||
contact_phone = Column(String, nullable=True)
|
||||
email = Column(String, nullable=True)
|
||||
@@ -111,4 +116,14 @@ class ServiceStaging(Base):
|
||||
status = Column(String(20), server_default=text("'pending'"), index=True)
|
||||
trust_score = Column(Integer, default=0)
|
||||
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
class DiscoveryParameter(Base):
|
||||
__tablename__ = "discovery_parameters"
|
||||
__table_args__ = {"schema": "data"}
|
||||
id = Column(Integer, primary_key=True)
|
||||
city = Column(String(100), nullable=False)
|
||||
keyword = Column(String(100), nullable=False) # pl. "autóvillamosság"
|
||||
country_code = Column(String(2), default="HU")
|
||||
is_active = Column(Boolean, default=True)
|
||||
last_run_at = Column(DateTime(timezone=True))
|
||||
@@ -7,7 +7,11 @@ class SystemParameter(Base):
|
||||
__table_args__ = {"schema": "data", "extend_existing": True}
|
||||
|
||||
key = Column(String, primary_key=True, index=True, nullable=False)
|
||||
# Csoportosítás az Admin felületnek (pl. 'xp', 'scout', 'routing')
|
||||
category = Column(String, index=True, server_default="general")
|
||||
value = Column(JSON, nullable=False)
|
||||
is_active = Column(Boolean, default=True)
|
||||
description = Column(String)
|
||||
# Kötelező audit mező: ki módosította utoljára?
|
||||
last_modified_by = Column(String, nullable=True)
|
||||
updated_at = Column(DateTime(timezone=True), onupdate=func.now(), server_default=func.now())
|
||||
Binary file not shown.
@@ -5,6 +5,7 @@ from datetime import datetime
|
||||
|
||||
# --- KATALÓGUS SÉMÁK (Gyári adatok) ---
|
||||
class AssetCatalogBase(BaseModel):
|
||||
"""Alap katalógus adatok, amik a technikai dúsításból származnak."""
|
||||
make: str
|
||||
model: str
|
||||
generation: Optional[str] = None
|
||||
@@ -13,39 +14,57 @@ class AssetCatalogBase(BaseModel):
|
||||
vehicle_class: Optional[str] = None
|
||||
fuel_type: Optional[str] = None
|
||||
engine_code: Optional[str] = None
|
||||
|
||||
# --- ÚJ TECHNIKAI MEZŐK (Robot v1.0.8 Smart Hunter adatai) ---
|
||||
power_kw: Optional[int] = None
|
||||
engine_capacity: Optional[int] = None
|
||||
max_weight_kg: Optional[int] = None
|
||||
axle_count: Optional[int] = None
|
||||
body_type: Optional[str] = None
|
||||
|
||||
class AssetCatalogResponse(AssetCatalogBase):
|
||||
"""Katalógus válasz séma azonosítóval és extra gyári adatokkal."""
|
||||
id: int
|
||||
factory_data: Optional[Dict[str, Any]] = None # A robot által gyűjtött adatok
|
||||
factory_data: Optional[Dict[str, Any]] = None
|
||||
|
||||
# Pydantic v2 konfiguráció az ORM (SQLAlchemy) támogatáshoz
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# --- JÁRMŰ SÉMÁK (Asset) ---
|
||||
class AssetBase(BaseModel):
|
||||
"""Jármű példány alapadatai (egyedi azonosítók)."""
|
||||
vin: str = Field(..., min_length=17, max_length=17)
|
||||
license_plate: str
|
||||
name: Optional[str] = None
|
||||
year_of_manufacture: Optional[int] = None
|
||||
|
||||
class AssetCreate(AssetBase):
|
||||
# A létrehozáshoz kellenek a katalógus infók is
|
||||
"""Séma új jármű felvételéhez."""
|
||||
make: str
|
||||
model: str
|
||||
vehicle_class: Optional[str] = "land"
|
||||
vehicle_class: Optional[str] = "car"
|
||||
fuel_type: Optional[str] = None
|
||||
current_reading: Optional[int] = 0
|
||||
|
||||
class AssetResponse(AssetBase):
|
||||
"""
|
||||
Teljes jármű válasz séma.
|
||||
Ez a séma tartalmazza a 'catalog' objektumot, amiben a dúsított műszaki adatok vannak.
|
||||
"""
|
||||
id: UUID
|
||||
catalog_id: int
|
||||
is_verified: bool
|
||||
catalog: AssetCatalogResponse # Ez a pont kapcsolja össze a dúsított technikai adatokat
|
||||
status: str
|
||||
is_verified: bool
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# --- DIGITÁLIS IKER (Full Profile) ---
|
||||
# Ez a séma felel a 9 pontos költség és a mélységi szerviz adatok átadásáért
|
||||
class AssetFullProfile(BaseModel):
|
||||
"""
|
||||
Komplex jelentésekhez használt séma.
|
||||
Összefogja az identitást, telemetriát, pénzügyeket és szerviztörténetet.
|
||||
"""
|
||||
identity: Dict[str, Any]
|
||||
telemetry: Dict[str, Any]
|
||||
financial_summary: Dict[str, Any]
|
||||
|
||||
@@ -32,12 +32,17 @@ class UserKYCComplete(BaseModel):
|
||||
birth_date: date
|
||||
mothers_last_name: str
|
||||
mothers_first_name: str
|
||||
# Bontott címmezők (B pont szerint)
|
||||
address_zip: str
|
||||
address_city: str
|
||||
address_street_name: str
|
||||
address_street_type: str
|
||||
address_house_number: str
|
||||
address_hrsz: Optional[str] = None
|
||||
address_stairwell: Optional[str] = None # Lépcsőház
|
||||
address_floor: Optional[str] = None # Emelet
|
||||
address_door: Optional[str] = None # Ajtó
|
||||
address_hrsz: Optional[str] = None # Helyrajzi szám
|
||||
|
||||
identity_docs: Dict[str, DocumentDetail]
|
||||
ice_contact: ICEContact
|
||||
preferred_currency: Optional[str] = Field("HUF", max_length=3)
|
||||
|
||||
20
backend/app/schemas/service.py
Normal file
20
backend/app/schemas/service.py
Normal file
@@ -0,0 +1,20 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
class ServiceCreateInternal(BaseModel):
|
||||
name: str
|
||||
postal_code: str
|
||||
city: str
|
||||
street_name: str
|
||||
street_type: str
|
||||
house_number: str
|
||||
stairwell: Optional[str] = None
|
||||
floor: Optional[str] = None
|
||||
door: Optional[str] = None
|
||||
hrsz: Optional[str] = None
|
||||
|
||||
contact_phone: Optional[str] = None
|
||||
email: Optional[str] = None
|
||||
website: Optional[str] = None
|
||||
source: str
|
||||
external_id: Optional[str] = None
|
||||
Binary file not shown.
Binary file not shown.
@@ -112,25 +112,23 @@ class AuthService:
|
||||
async def complete_kyc(db: AsyncSession, user_id: int, kyc_in: UserKYCComplete):
|
||||
"""
|
||||
Step 2: Atomi Tranzakció.
|
||||
Itt dől el minden: Adatok rögzítése, Shadow Identity ellenőrzés,
|
||||
Flotta és Wallet létrehozás, majd a fiók aktiválása.
|
||||
Módosított verzió: Meglévő biztonsági logika + Telephely (Branch) integráció.
|
||||
"""
|
||||
try:
|
||||
# 1. User és Person betöltése
|
||||
stmt = select(User).options(joinedload(User.person)).where(User.id == user_id)
|
||||
res = await db.execute(stmt)
|
||||
user = res.scalar_one_or_none()
|
||||
if not user: return None
|
||||
|
||||
# --- 1. BIZTONSÁG: User folder_slug generálása ---
|
||||
# Ha Google-lel jött vagy még nincs slugja, most kap egyet.
|
||||
# --- BIZTONSÁG: Slug generálása ---
|
||||
if not user.folder_slug:
|
||||
user.folder_slug = generate_secure_slug(length=12)
|
||||
|
||||
# Pénznem beállítása
|
||||
if hasattr(kyc_in, 'preferred_currency') and kyc_in.preferred_currency:
|
||||
user.preferred_currency = kyc_in.preferred_currency
|
||||
|
||||
# --- 2. Shadow Identity keresése (Már létezik-e ez a fizikai személy?) ---
|
||||
# --- SHADOW IDENTITY ELLENŐRZÉS ---
|
||||
identity_stmt = select(Person).where(and_(
|
||||
Person.mothers_last_name == kyc_in.mothers_last_name,
|
||||
Person.mothers_first_name == kyc_in.mothers_first_name,
|
||||
@@ -140,15 +138,12 @@ class AuthService:
|
||||
existing_person = (await db.execute(identity_stmt)).scalar_one_or_none()
|
||||
|
||||
if existing_person:
|
||||
# Ha találtunk egyezést, összekötjük a User-t a meglévő Person-nel
|
||||
user.person_id = existing_person.id
|
||||
active_person = existing_person
|
||||
logger.info(f"Shadow Identity linked: User {user_id} -> Person {existing_person.id}")
|
||||
else:
|
||||
# Ha nem, a saját (regisztrációkor létrehozott) Person-t töltjük fel
|
||||
active_person = user.person
|
||||
|
||||
# --- 3. Cím rögzítése GeoService segítségével ---
|
||||
# --- CÍM RÖGZÍTÉSE ---
|
||||
addr_id = await GeoService.get_or_create_full_address(
|
||||
db,
|
||||
zip_code=kyc_in.address_zip,
|
||||
@@ -159,30 +154,26 @@ class AuthService:
|
||||
parcel_id=kyc_in.address_hrsz
|
||||
)
|
||||
|
||||
# --- 4. Személyes adatok frissítése ---
|
||||
# --- SZEMÉLYES ADATOK FRISSÍTÉSE ---
|
||||
active_person.mothers_last_name = kyc_in.mothers_last_name
|
||||
active_person.mothers_first_name = kyc_in.mothers_first_name
|
||||
active_person.birth_place = kyc_in.birth_place
|
||||
active_person.birth_date = kyc_in.birth_date
|
||||
active_person.phone = kyc_in.phone_number
|
||||
active_person.address_id = addr_id
|
||||
|
||||
# Dokumentumok és ICE kontakt mentése JSON-ként
|
||||
active_person.identity_docs = jsonable_encoder(kyc_in.identity_docs)
|
||||
active_person.ice_contact = jsonable_encoder(kyc_in.ice_contact)
|
||||
|
||||
# A Person most válik aktívvá
|
||||
active_person.is_active = True
|
||||
|
||||
# --- 5. EGYÉNI FLOTTA LÉTREHOZÁSA (A KYC szerves része) ---
|
||||
# Itt generáljuk a flotta mappáját is (folder_slug)
|
||||
# --- EGYÉNI FLOTTA LÉTREHOZÁSA ---
|
||||
new_org = Organization(
|
||||
full_name=f"{active_person.last_name} {active_person.first_name} Egyéni Flotta",
|
||||
name=f"{active_person.last_name} Flotta",
|
||||
folder_slug=generate_secure_slug(length=12), # FLOTTA SLUG
|
||||
folder_slug=generate_secure_slug(length=12),
|
||||
org_type=OrgType.individual,
|
||||
owner_id=user.id,
|
||||
is_transferable=False,
|
||||
is_transferable=False, # Step 2: Individual flotta nem átruházható
|
||||
is_ownership_transferable=False, # A te új meződ
|
||||
is_active=True,
|
||||
status="verified",
|
||||
language=user.preferred_language,
|
||||
@@ -192,24 +183,36 @@ class AuthService:
|
||||
db.add(new_org)
|
||||
await db.flush()
|
||||
|
||||
# Flotta tagság (Owner)
|
||||
# --- ÚJ: MAIN BRANCH (KÖZPONTI TELEPHELY) LÉTREHOZÁSA ---
|
||||
# Magánszemélynél a megadott cím lesz az első telephely is.
|
||||
from app.models.address import Branch
|
||||
new_branch = Branch(
|
||||
organization_id=new_org.id,
|
||||
address_id=addr_id,
|
||||
name="Központ / Otthon",
|
||||
is_main=True,
|
||||
postal_code=kyc_in.address_zip,
|
||||
city=kyc_in.address_city,
|
||||
street_name=kyc_in.address_street_name,
|
||||
street_type=kyc_in.address_street_type,
|
||||
house_number=kyc_in.address_house_number,
|
||||
hrsz=kyc_in.address_hrsz,
|
||||
status="active"
|
||||
)
|
||||
db.add(new_branch)
|
||||
await db.flush()
|
||||
|
||||
# --- TAGSÁG, WALLET, STATS ---
|
||||
db.add(OrganizationMember(
|
||||
organization_id=new_org.id,
|
||||
user_id=user.id,
|
||||
role="owner",
|
||||
permissions={"can_add_asset": True, "can_view_costs": True, "is_admin": True}
|
||||
))
|
||||
|
||||
# --- 6. PÉNZTÁRCA ÉS GAMIFICATION LÉTREHOZÁSA ---
|
||||
db.add(Wallet(
|
||||
user_id=user.id,
|
||||
coin_balance=0,
|
||||
credit_balance=0,
|
||||
currency=user.preferred_currency or "HUF"
|
||||
))
|
||||
db.add(Wallet(user_id=user.id, currency=user.preferred_currency or "HUF"))
|
||||
db.add(UserStats(user_id=user.id, total_xp=0, current_level=1))
|
||||
|
||||
# --- 7. AKTIVÁLÁS ÉS AUDIT ---
|
||||
# --- 7. AKTIVÁLÁS ÉS AUDIT (Ami az előzőből kimaradt) ---
|
||||
user.is_active = True
|
||||
|
||||
await security_service.log_event(
|
||||
@@ -223,7 +226,7 @@ class AuthService:
|
||||
"status": "active",
|
||||
"user_folder": user.folder_slug,
|
||||
"organization_id": new_org.id,
|
||||
"organization_folder": new_org.folder_slug,
|
||||
"branch_id": str(new_branch.id), # Új telephely az auditban
|
||||
"wallet_created": True
|
||||
}
|
||||
)
|
||||
@@ -231,6 +234,7 @@ class AuthService:
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
return user
|
||||
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(f"KYC Atomi Tranzakció Hiba: {str(e)}")
|
||||
|
||||
@@ -45,22 +45,42 @@ class GeoService:
|
||||
"""), {"n": street_type.lower()})
|
||||
|
||||
# 4. Központi Address rekord rögzítése
|
||||
full_text = f"{zip_code} {city}, {street_name} {street_type} {house_number}"
|
||||
addr_res = await db.execute(text("""
|
||||
INSERT INTO data.addresses (postal_code_id, street_name, street_type, house_number, parcel_id, full_address_text)
|
||||
VALUES (:zid, :sn, :st, :hn, :pid, :txt)
|
||||
full_text = f"{zip_code} {city}, {street_name} {street_type} {house_number}."
|
||||
if stairwell: full_text += f" {stairwell}. lph,"
|
||||
if floor: full_text += f" {floor}. em,"
|
||||
if door: full_text += f" {door}. ajtó"
|
||||
|
||||
query = text("""
|
||||
INSERT INTO data.addresses (
|
||||
postal_code_id, street_name, street_type, house_number,
|
||||
stairwell, floor, door, parcel_id, full_address_text
|
||||
)
|
||||
VALUES (
|
||||
(SELECT id FROM data.geo_postal_codes WHERE zip_code = :z AND city = :c LIMIT 1),
|
||||
:sn, :st, :hn, :sw, :fl, :dr, :pid, :txt
|
||||
)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING id
|
||||
"""), {
|
||||
"zid": zip_id, "sn": street_name, "st": street_type, "hn": house_number, "pid": parcel_id, "txt": full_text
|
||||
})
|
||||
addr_id = addr_res.scalar()
|
||||
""")
|
||||
|
||||
params = {
|
||||
"z": zip_code, "c": city, "sn": street_name, "st": street_type,
|
||||
"hn": house_number, "sw": stairwell, "fl": floor, "dr": door,
|
||||
"pid": parcel_id, "txt": full_text
|
||||
}
|
||||
|
||||
res = await db.execute(query, params)
|
||||
addr_id = res.scalar()
|
||||
|
||||
if not addr_id:
|
||||
# Ha már létezett, lekérjük az azonosítót
|
||||
# Ha már létezett ilyen részletes cím, lekérjük
|
||||
addr_id = (await db.execute(text("""
|
||||
SELECT id FROM data.addresses
|
||||
WHERE postal_code_id = :zid AND street_name = :sn AND street_type = :st AND house_number = :hn
|
||||
"""), {"zid": zip_id, "sn": street_name, "st": street_type, "hn": house_number})).scalar()
|
||||
WHERE street_name = :sn AND house_number = :hn
|
||||
AND (stairwell IS NOT DISTINCT FROM :sw)
|
||||
AND (floor IS NOT DISTINCT FROM :fl)
|
||||
AND (door IS NOT DISTINCT FROM :dr)
|
||||
LIMIT 1
|
||||
"""), params)).scalar()
|
||||
|
||||
return addr_id
|
||||
Binary file not shown.
Binary file not shown.
61
backend/app/workers/brand_seeder.py
Normal file
61
backend/app/workers/brand_seeder.py
Normal file
@@ -0,0 +1,61 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
from sqlalchemy import text
|
||||
from app.db.session import SessionLocal
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("Smart-Seeder-v1.0.2")
|
||||
|
||||
async def seed_with_priority():
|
||||
# RDW lekérdezés: Márka, Fő kategória és darabszám
|
||||
# Olyan márkákat keresünk, amikből legalább 10 db van
|
||||
URL = "https://opendata.rdw.nl/resource/m9d7-ebf2.json?$select=merk,voertuigsoort,count(*)%20as%20total&$group=merk,voertuigsoort&$having=total%20>=%2010"
|
||||
|
||||
logger.info("📥 Adatok lekérése az RDW-től prioritásos besoroláshoz...")
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
try:
|
||||
resp = await client.get(URL)
|
||||
if resp.status_code != 200:
|
||||
logger.error(f"❌ API hiba: {resp.status_code}")
|
||||
return
|
||||
|
||||
raw_data = resp.json()
|
||||
async with SessionLocal() as db:
|
||||
for entry in raw_data:
|
||||
make = entry.get("merk", "").upper()
|
||||
v_kind = entry.get("voertuigsoort", "")
|
||||
|
||||
# --- PRIORITÁS LOGIKA ---
|
||||
# 1. Személyautó (Personenauto) -> 'pending' (Azonnali feldolgozás)
|
||||
# 2. Motor (Motorfiets) -> 'queued_motor'
|
||||
# 3. Minden más -> 'queued_heavy'
|
||||
|
||||
status = 'queued_heavy'
|
||||
if "Personenauto" in v_kind:
|
||||
status = 'pending'
|
||||
elif "Motorfiets" in v_kind:
|
||||
status = 'queued_motor'
|
||||
|
||||
query = text("""
|
||||
INSERT INTO data.catalog_discovery (make, model, vehicle_class, source, status)
|
||||
VALUES (:make, 'ALL_VARIANTS', :v_class, 'smart_seeder_v2_1', :status)
|
||||
ON CONFLICT (make, model, vehicle_class) DO UPDATE
|
||||
SET status = EXCLUDED.status WHERE data.catalog_discovery.status = 'pending';
|
||||
""")
|
||||
|
||||
await db.execute(query, {
|
||||
"make": make,
|
||||
"v_class": v_kind,
|
||||
"status": status
|
||||
})
|
||||
|
||||
await db.commit()
|
||||
logger.info("✅ A Discovery lista feltöltve és prioritizálva (Autók az élen)!")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Hiba: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_with_priority())
|
||||
@@ -2,178 +2,207 @@ import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
import datetime
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func, or_, text
|
||||
from sqlalchemy import text
|
||||
from app.db.session import SessionLocal
|
||||
from app.models.asset import AssetCatalog
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("Robot1-Ghost-Commander-v1.1.9")
|
||||
logger = logging.getLogger("Robot-v1.0.13-Global-Hunter")
|
||||
|
||||
class CatalogScout:
|
||||
class CatalogMaster:
|
||||
"""
|
||||
Robot 1.1.9: Environment Master.
|
||||
- .env alapú hitelesítés (RDW App Token)
|
||||
- Prioritás: RDW (EU) -> NHTSA (US) -> CarQuery (Ban-figyeléssel)
|
||||
- 2.5s lekérési frissítés a biztonságért
|
||||
Master Hunter Robot v1.0.13 - Global Hunter Edition
|
||||
- Holland (RDW), Brit (DVLA) és Amerikai (NHTSA) adatbázis integráció.
|
||||
- Ratio-Filter: Kiszűri a 0.19-es kW/kg arányszámokat.
|
||||
- Multi-field Power Discovery: Minden lehetséges mezőből kinyeri a kW-ot.
|
||||
- Dinamikus évjárat kezelés a duplikációk ellen.
|
||||
"""
|
||||
|
||||
CQ_URL = "https://www.carqueryapi.com/api/0.3/"
|
||||
NHTSA_BASE = "https://vpic.nhtsa.dot.gov/api/vehicles/GetModelsForMakeYear/make/"
|
||||
RDW_URL = "https://opendata.rdw.nl/resource/ed7h-m8uz.json"
|
||||
# API Végpontok
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_AXLE = "https://opendata.rdw.nl/resource/3huj-srit.json"
|
||||
RDW_BODY = "https://opendata.rdw.nl/resource/vezc-m2t6.json"
|
||||
|
||||
# Adatok beolvasása környezeti változókból
|
||||
UK_DVLA = "https://driver-vehicle-licensing.api.gov.uk/vehicle-enquiry/v1/vehicles"
|
||||
US_NHTSA = "https://vpic.nhtsa.dot.gov/api/vehicles/DecodeVinValuesBatch/"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
UK_API_KEY = os.getenv("UK_DVLA_API_KEY")
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Accept": "application/json",
|
||||
"X-App-Token": RDW_TOKEN
|
||||
HEADERS_RDW = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
HEADERS_UK = {"x-api-key": UK_API_KEY, "Content-Type": "application/json"} if UK_API_KEY else {}
|
||||
|
||||
CATEGORY_MAP = {
|
||||
"Personenauto": "car",
|
||||
"Motorfiets": "motorcycle",
|
||||
"Bedrijfsauto": "truck",
|
||||
"Vrachtwagen": "truck",
|
||||
"Opleggertrekker": "truck",
|
||||
"Bus": "bus",
|
||||
"Aanhangwagen": "trailer",
|
||||
"Oplegger": "trailer",
|
||||
"Landbouw- of bosbouwtrekker": "agricultural",
|
||||
"camper": "camper"
|
||||
}
|
||||
|
||||
# BAN FIGYELŐ ÁLLAPOT
|
||||
cq_banned_until = None
|
||||
|
||||
# --- KATEGÓRIA DEFINÍCIÓK (Szigorúan az eredeti lista szerint) ---
|
||||
MOTO_MAKES = ['ducati', 'ktm', 'triumph', 'aprilia', 'benelli', 'vespa', 'simson', 'mz', 'etz', 'jawa', 'husqvarna', 'gasgas', 'sherco']
|
||||
MARINE_IDS = ['DF', 'DT', 'OUTBOARD', 'MARINE', 'JET SKI', 'SEA-DOO', 'WAVERUNNER', 'YACHT', 'BOAT']
|
||||
AERIAL_IDS = ['CESSNA', 'PIPER', 'AIRBUS', 'BOEING', 'HELICOPTER', 'AIRCRAFT', 'BEECHCRAFT', 'EMBRAER', 'DRONE']
|
||||
ATV_IDS = ['LT-', 'LTZ', 'LTR', 'KINGQUAD', 'QUAD', 'POLARIS', 'CAN-AM', 'MULE', 'RZR', 'ARCTIC CAT', 'UTV', 'SIDE-BY-SIDE']
|
||||
RACING_IDS = ['RM-Z', 'KX', 'CRF', 'YZ', 'SX-F', 'XC-W', 'RM125', 'RM250', 'CR125', 'CR250', 'MC450']
|
||||
MOTO_KEYWORDS = ['CBR', 'GSX', 'YZF', 'NINJA', 'Z1000', 'DR-Z', 'MT-0', 'V-STROM', 'ADVENTURE', 'SCRAMBLER', 'CBF', 'VFR', 'HAYABUSA']
|
||||
BUS_KEYWORDS = ['BUS', 'COACH', 'INTERCITY', 'SHUTTLE', 'TRANSIT']
|
||||
TRUCK_KEYWORDS = ['TRUCK', 'SEMI', 'TRACTOR', 'HAULER', 'ACTROS', 'MAN', 'SCANIA', 'IVECO', 'VOLVO FH', 'DAF', 'TGX', 'RENAULT T']
|
||||
TRAILER_KEYWORDS = ['TRAILER', 'SEMITRAILER', 'PÓTKOCSI', 'UTÁNFUTÓ', 'SCHMITZ', 'KRONE', 'KÖGEL']
|
||||
|
||||
FALLBACK_BRANDS = ['Audi', 'BMW', 'Mercedes-Benz', 'Volkswagen', 'Toyota', 'Ford', 'Honda', 'Hyundai', 'Kia', 'Mazda', 'Nissan', 'Volvo', 'Skoda', 'Opel', 'Tesla', 'Lexus', 'Porsche', 'Dacia', 'Suzuki']
|
||||
|
||||
@classmethod
|
||||
def identify_class(cls, make: str, model: str) -> str:
|
||||
m_full = f"{str(make)} {str(model)}".upper()
|
||||
if any(x in m_full for x in cls.AERIAL_IDS): return "aerial"
|
||||
if any(x in m_full for x in cls.MARINE_IDS): return "marine"
|
||||
if any(x in m_full for x in cls.ATV_IDS): return "atv"
|
||||
if any(x in m_full or str(make).lower() in cls.MOTO_MAKES for x in (cls.RACING_IDS + cls.MOTO_KEYWORDS)):
|
||||
return "motorcycle"
|
||||
if any(x in m_full for x in cls.BUS_KEYWORDS): return "bus"
|
||||
if any(x in m_full for x in cls.TRUCK_KEYWORDS): return "truck"
|
||||
if any(x in m_full for x in cls.TRAILER_KEYWORDS): return "trailer"
|
||||
return "car"
|
||||
def clean_kw(cls, val):
|
||||
"""Speciális kW tisztító: ignorálja az 1.0 alatti arányszámokat."""
|
||||
try:
|
||||
if val is None: return None
|
||||
f_val = float(str(val).replace(',', '.'))
|
||||
if 0 < f_val < 1.0: return None # Ez csak arányszám (kW/kg)
|
||||
v = int(f_val)
|
||||
return v if v > 0 else None
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def fetch_api(cls, url, params=None, is_cq=False):
|
||||
if is_cq and cls.cq_banned_until and datetime.datetime.now() < cls.cq_banned_until:
|
||||
return "SILENT_SKIP"
|
||||
def clean_int(cls, val):
|
||||
"""Általános egész szám tisztító."""
|
||||
try:
|
||||
if val is None: return None
|
||||
return int(float(str(val).replace(',', '.')))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
async with httpx.AsyncClient(headers=cls.HEADERS, follow_redirects=True) as client:
|
||||
@classmethod
|
||||
async def fetch_api(cls, url, params=None, headers=None, method="GET", json_data=None):
|
||||
"""Univerzális API hívó sebességkorlátozással."""
|
||||
async with httpx.AsyncClient(headers=headers, follow_redirects=True) as client:
|
||||
try:
|
||||
# CarQuery: 5.0mp szünet (Hard Ban ellen), többi: 2.5mp (User kérése szerint)
|
||||
await asyncio.sleep(5.0 if is_cq else 2.5)
|
||||
resp = await client.get(url, params=params, timeout=35)
|
||||
|
||||
if resp.status_code == 403 or "denied" in resp.text.lower():
|
||||
logger.error("🚫 CarQuery BAN! 2 óra kényszerpihenő aktiválva.")
|
||||
cls.cq_banned_until = datetime.datetime.now() + datetime.timedelta(hours=2)
|
||||
return "DENIED"
|
||||
|
||||
if resp.status_code != 200: return None
|
||||
content = resp.text.strip()
|
||||
if is_cq:
|
||||
match = re.search(r'(\{.*\}|\[.*\])', content, re.DOTALL)
|
||||
if match: content = match.group(0)
|
||||
return json.loads(content)
|
||||
await asyncio.sleep(1.2) # Biztonsági késleltetés
|
||||
if method == "POST":
|
||||
resp = await client.post(url, json=json_data, timeout=30)
|
||||
else:
|
||||
resp = await client.get(url, params=params, timeout=30)
|
||||
return resp.json() if resp.status_code in [200, 201] else []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ API hiba: {e}")
|
||||
return None
|
||||
logger.error(f"❌ API Hiba ({url}): {e}")
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def is_model_processed(cls, db: AsyncSession, make: str, model: str, year: int):
|
||||
stmt = select(AssetCatalog.id).where(AssetCatalog.make == make, AssetCatalog.model == model, AssetCatalog.year_from == year).limit(1)
|
||||
result = await db.execute(stmt)
|
||||
return result.scalars().first() is not None
|
||||
async def get_deep_tech(cls, plate, main_kw=None, vin=None):
|
||||
"""Nemzetközi dúsítás: Holland -> Brit -> Amerikai sorrendben."""
|
||||
res = {"kw": cls.clean_kw(main_kw), "fuel": "Unknown", "axles": None, "body": "Standard", "euro": None}
|
||||
|
||||
# 1. HOLLAND (RDW) DÚSÍTÁS
|
||||
fuel_data = await cls.fetch_api(cls.RDW_FUEL, {"kenteken": plate}, headers=cls.HEADERS_RDW)
|
||||
if fuel_data:
|
||||
f0 = fuel_data[0]
|
||||
if not res["kw"]:
|
||||
res["kw"] = cls.clean_kw(f0.get("nettomaximumvermogen") or f0.get("netto_maximum_vermogen"))
|
||||
res["fuel"] = f0.get("brandstof_omschrijving", "Unknown")
|
||||
res["euro"] = f0.get("uitlaatemissieniveau")
|
||||
|
||||
# 2. BRIT (DVLA) ELLENŐRZÉS (Ha van UK kulcs és még hiányzik adat)
|
||||
if cls.UK_API_KEY and (not res["kw"] or not res["euro"]):
|
||||
uk_data = await cls.fetch_api(cls.UK_DVLA, method="POST", json_data={"registrationNumber": plate}, headers=cls.HEADERS_UK)
|
||||
if uk_data:
|
||||
res["kw"] = res["kw"] or cls.clean_kw(uk_data.get("engineCapacity")) # Brit adatok finomítása
|
||||
res["euro"] = res["euro"] or uk_data.get("euroStatus")
|
||||
|
||||
# 3. AMERIKAI (NHTSA) KUTATÁS (Ha van alvázszám)
|
||||
if vin and len(vin) == 17:
|
||||
us_data = await cls.fetch_api(cls.US_NHTSA, params={"format": "json", "data": vin})
|
||||
if us_data and "Results" in us_data:
|
||||
# Az amerikai adatbázisból kinyerjük a lóerőt (HP), ha a kW még mindig nincs meg
|
||||
hp = us_data["Results"][0].get("EngineHP")
|
||||
if hp and not res["kw"]:
|
||||
res["kw"] = int(float(hp) * 0.7457) # HP -> kW konverzió
|
||||
|
||||
# RDW Extra adatok (Tengely, Karosszéria)
|
||||
axle = await cls.fetch_api(cls.RDW_AXLE, {"kenteken": plate}, headers=cls.HEADERS_RDW)
|
||||
if axle: res["axles"] = cls.clean_int(axle[0].get("aantal_assen"))
|
||||
|
||||
body = await cls.fetch_api(cls.RDW_BODY, {"kenteken": plate}, headers=cls.HEADERS_RDW)
|
||||
if body: res["body"] = body[0].get("carrosserie_omschrijving", "Standard")
|
||||
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def auto_heal(cls, db: AsyncSession, cq_active: bool):
|
||||
logger.info("🛠️ Auto-Heal: Hiányos rekordok dúsítása...")
|
||||
stmt = select(AssetCatalog).where(AssetCatalog.engine_variant == 'Standard', AssetCatalog.fuel_type == 'Unknown').limit(20)
|
||||
results = await db.execute(stmt)
|
||||
for r in results.scalars().all():
|
||||
# 1. RDW javítás (Holland Open Data + Token)
|
||||
rdw = await cls.fetch_api(cls.RDW_URL, {"merk": r.make.upper(), "handelsbenaming": r.model.upper(), "$limit": 1})
|
||||
if rdw and isinstance(rdw, list) and len(rdw) > 0:
|
||||
item = rdw[0]
|
||||
r.fuel_type = item.get("brandstof_omschrijving", "Unknown")
|
||||
r.factory_data.update({"hp": item.get("netto_maximum_vermogen"), "cc": item.get("cilinderinhoud"), "source": "heal_v1.9_rdw"})
|
||||
async def process_make(cls, db, task_id, make_name):
|
||||
logger.info(f"🚀 >>> {make_name} GlobalHunter v1.0.13 INDUL...")
|
||||
offset, limit, total_saved = 0, 1000, 0
|
||||
unique_variants = {}
|
||||
|
||||
while True:
|
||||
params = {"merk": make_name.upper(), "$limit": limit, "$offset": offset}
|
||||
main_data = await cls.fetch_api(cls.RDW_MAIN, params, headers=cls.HEADERS_RDW)
|
||||
if not main_data: break
|
||||
|
||||
for item in main_data:
|
||||
plate = item.get("kenteken")
|
||||
if not plate: continue
|
||||
|
||||
model = str(item.get("handelsbenaming", "Unknown")).upper()
|
||||
ccm = cls.clean_int(item.get("cilinderinhoud"))
|
||||
weight = cls.clean_int(item.get("massa_ledig_voertuig") or item.get("massa_rijklaar"))
|
||||
kw_candidate = item.get("netto_maximum_vermogen") or item.get("vermogen_massarijklaar")
|
||||
|
||||
raw_date = item.get("datum_eerste_toelating")
|
||||
prod_year = int(str(raw_date)[:4]) if raw_date else 2024
|
||||
|
||||
v_class = cls.CATEGORY_MAP.get(item.get("voertuigsoort"), "other")
|
||||
if "kampeerwagen" in str(item.get("inrichting", "")).lower(): v_class = "camper"
|
||||
|
||||
# Variáns kulcs: Modell + CCM + Súly + kW + Év = Egyedi technikai ujjlenyomat
|
||||
variant_key = f"{model}-{ccm}-{weight}-{v_class}-{kw_candidate}-{prod_year}"
|
||||
|
||||
if variant_key not in unique_variants:
|
||||
unique_variants[variant_key] = {
|
||||
"model": model, "ccm": ccm, "weight": weight, "v_class": v_class,
|
||||
"plate": plate, "main_kw": kw_candidate, "prod_year": prod_year,
|
||||
"vin": item.get("vin") # Ha az RDW-ben benne van a VIN
|
||||
}
|
||||
|
||||
if len(main_data) < limit or offset > 90000: break
|
||||
offset += limit
|
||||
|
||||
logger.info(f"📊 {len(unique_variants)} egyedi variáns kutatása indul...")
|
||||
|
||||
for key, v in unique_variants.items():
|
||||
deep = await cls.get_deep_tech(v["plate"], main_kw=v["main_kw"], vin=v["vin"])
|
||||
try:
|
||||
db_item = AssetCatalog(
|
||||
make=make_name.upper(), model=v["model"], vehicle_class=v["v_class"],
|
||||
fuel_type=deep["fuel"], power_kw=deep["kw"], engine_capacity=v["ccm"],
|
||||
max_weight_kg=v["weight"], axle_count=deep["axles"], body_type=deep["body"],
|
||||
year_from=v["prod_year"], euro_class=deep["euro"],
|
||||
factory_data={
|
||||
"source": "GlobalHunter-v1.0.13",
|
||||
"sample_plate": v["plate"],
|
||||
"enriched_at": str(datetime.datetime.now())
|
||||
}
|
||||
)
|
||||
db.add(db_item)
|
||||
await db.commit()
|
||||
total_saved += 1
|
||||
if total_saved % 50 == 0: logger.info(f"✅ {total_saved} variáns elmentve.")
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
continue
|
||||
|
||||
# 2. CQ javítás (Ha nem vagyunk kitiltva)
|
||||
if cq_active:
|
||||
t_data = await cls.fetch_api(cls.CQ_URL, {"cmd": "getTrims", "make": r.make.lower(), "model": r.model, "year": r.year_from}, is_cq=True)
|
||||
if t_data and t_data not in ["DENIED", "SILENT_SKIP"] and "Trims" in t_data:
|
||||
t = t_data["Trims"][0]
|
||||
r.engine_variant = t.get("model_trim") or "Standard"
|
||||
r.factory_data.update({"hp": t.get("model_engine_power_ps"), "cc": t.get("model_engine_cc"), "source": "heal_v1.9_cq"})
|
||||
|
||||
await db.execute(text("UPDATE data.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task_id})
|
||||
await db.commit()
|
||||
logger.info(f"🏁 {make_name} KÉSZ. {total_saved} rekord rögzítve.")
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info(f"🤖 Robot 1.9.2 indítása (RDW Token: {'Aktív' if cls.RDW_TOKEN else 'HIÁNYZIK!'})")
|
||||
|
||||
for year in range(2026, 1989, -1):
|
||||
logger.info(f"📅 --- CIKLUS: {year} ---")
|
||||
|
||||
cq_now_active = not (cls.cq_banned_until and datetime.datetime.now() < cls.cq_banned_until)
|
||||
|
||||
logger.info("🤖 Robot 1.0.13 (Global Hunter) ONLINE")
|
||||
while True:
|
||||
async with SessionLocal() as db:
|
||||
await cls.auto_heal(db, cq_now_active)
|
||||
|
||||
# 1. MÁRKALISTA (NHTSA + Fallback)
|
||||
makes_to_process = []
|
||||
for b in cls.FALLBACK_BRANDS:
|
||||
makes_to_process.append({"id": b.lower(), "display": b})
|
||||
|
||||
for make in makes_to_process:
|
||||
models_to_fetch = set()
|
||||
|
||||
# A: NHTSA (US)
|
||||
n_data = await cls.fetch_api(f"{cls.NHTSA_BASE}{make['display']}/modelyear/{year}?format=json")
|
||||
if n_data and n_data.get("Results"):
|
||||
for r in n_data["Results"]: models_to_fetch.add(r["Model_Name"])
|
||||
|
||||
# B: RDW (Holland) - Tokennel védve
|
||||
rdw_m = await cls.fetch_api(cls.RDW_URL, {"merk": make['display'].upper(), "$limit": 30})
|
||||
if rdw_m and isinstance(rdw_m, list):
|
||||
for r in rdw_m: models_to_fetch.add(r.get("handelsbenaming"))
|
||||
|
||||
async with SessionLocal() as db:
|
||||
for model_name in models_to_fetch:
|
||||
if not model_name or await cls.is_model_processed(db, make["display"], model_name, year):
|
||||
continue
|
||||
|
||||
# C: CarQuery (Csak ha nincs ban)
|
||||
found_trims = []
|
||||
t_data = await cls.fetch_api(cls.CQ_URL, {"cmd": "getTrims", "make": make["id"], "model": model_name, "year": year}, is_cq=True)
|
||||
if t_data and t_data not in ["DENIED", "SILENT_SKIP"] and "Trims" in t_data:
|
||||
found_trims = t_data["Trims"]
|
||||
|
||||
if not found_trims:
|
||||
found_trims = [{"model_trim": "Standard", "model_engine_fuel": "Unknown"}]
|
||||
|
||||
for t in found_trims:
|
||||
db.add(AssetCatalog(
|
||||
make=make["display"], model=model_name, year_from=year,
|
||||
engine_variant=t.get("model_trim") or "Standard",
|
||||
fuel_type=t.get("model_engine_fuel") or "Unknown",
|
||||
vehicle_class=cls.identify_class(make["display"], model_name),
|
||||
factory_data={
|
||||
"hp": t.get("model_engine_power_ps"), "cc": t.get("model_engine_cc"),
|
||||
"source": "ghost_v1.9.2", "sync_date": str(datetime.datetime.now())
|
||||
}
|
||||
))
|
||||
await db.commit()
|
||||
res = await db.execute(text("SELECT id, make FROM data.catalog_discovery WHERE status = 'pending' LIMIT 1"))
|
||||
task = res.fetchone()
|
||||
if task:
|
||||
await cls.process_make(db, task[0], task[1])
|
||||
else:
|
||||
logger.info("😴 Várólista üres. Alvás 60 mp...")
|
||||
await asyncio.sleep(60)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogScout.run())
|
||||
asyncio.run(CatalogMaster.run())
|
||||
@@ -1,282 +1,161 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import uuid
|
||||
import os
|
||||
import sys
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import selectinload
|
||||
from app.db.session import SessionLocal
|
||||
|
||||
# Modellek importálása
|
||||
from app.models.service import ServiceProfile, ExpertiseTag
|
||||
from app.models.organization import Organization, OrganizationFinancials, OrgType, OrgUserRole, OrganizationMember
|
||||
from app.models.identity import Person
|
||||
from app.models.address import Address, GeoPostalCode
|
||||
from geoalchemy2.elements import WKTElement
|
||||
from datetime import datetime, timezone
|
||||
# Modellek - Az új v1.3 struktúra
|
||||
from app.models.service import ServiceStaging, DiscoveryParameter
|
||||
|
||||
# Naplózás beállítása
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("Robot2-Dunakeszi-Detective")
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger("Robot-v1.3-ContinentalScout")
|
||||
|
||||
class ServiceHunter:
|
||||
"""
|
||||
Robot 2.7.2: Dunakeszi Detective - Deep Model Integration.
|
||||
Logika:
|
||||
1. Helyi CSV (Saját beküldés - Cím alapú Geocoding-al - 50 pont Trust)
|
||||
2. OSM (Közösségi adat - 10 pont Trust)
|
||||
3. Google (Adatpótlás/Fallback - 30 pont Trust)
|
||||
Robot v1.3.0: Continental Scout.
|
||||
EU-szintű felderítő motor, Discovery tábla alapú vezérléssel.
|
||||
"""
|
||||
OVERPASS_URL = "http://overpass-api.de/api/interpreter"
|
||||
PLACES_NEW_URL = "https://places.googleapis.com/v1/places:searchNearby"
|
||||
GEOCODE_URL = "https://maps.googleapis.com/maps/api/geocode/json"
|
||||
GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
|
||||
LOCAL_CSV_PATH = "/app/app/workers/local_services.csv"
|
||||
|
||||
@classmethod
|
||||
async def geocode_address(cls, address_text):
|
||||
"""Cím szövegből GPS koordinátát és címkomponenseket csinál."""
|
||||
if not cls.GOOGLE_API_KEY:
|
||||
logger.warning("⚠️ Google API kulcs hiányzik!")
|
||||
return None
|
||||
|
||||
params = {"address": address_text, "key": cls.GOOGLE_API_KEY}
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(cls.GEOCODE_URL, params=params, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
if data.get("results"):
|
||||
result = data["results"][0]
|
||||
loc = result["geometry"]["location"]
|
||||
|
||||
# Címkomponensek kinyerése a kötelező mezőkhöz
|
||||
components = result.get("address_components", [])
|
||||
parsed = {"lat": loc["lat"], "lng": loc["lng"], "zip": "", "city": "", "street": "Ismeretlen", "type": "utca", "number": "1"}
|
||||
|
||||
for c in components:
|
||||
types = c.get("types", [])
|
||||
if "postal_code" in types: parsed["zip"] = c["long_name"]
|
||||
if "locality" in types: parsed["city"] = c["long_name"]
|
||||
if "route" in types: parsed["street"] = c["long_name"]
|
||||
if "street_number" in types: parsed["number"] = c["long_name"]
|
||||
|
||||
logger.info(f"📍 Geocoding sikeres: {address_text}")
|
||||
return parsed
|
||||
else:
|
||||
logger.error(f"❌ Geocoding hiba: {resp.status_code}")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Geocoding hiba: {e}")
|
||||
return None
|
||||
async def get_coordinates(cls, city, country_code):
|
||||
"""Város központjának lekérése a keresés indításához."""
|
||||
params = {"address": f"{city}, {country_code}", "key": cls.GOOGLE_API_KEY}
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(cls.GEOCODE_URL, params=params)
|
||||
if resp.status_code == 200:
|
||||
results = resp.json().get("results")
|
||||
if results:
|
||||
loc = results[0]["geometry"]["location"]
|
||||
return loc["lat"], loc["lng"]
|
||||
return None, None
|
||||
|
||||
@classmethod
|
||||
async def get_google_place_details_new(cls, lat, lon):
|
||||
"""Google Places API (New) - Adatpótlás FieldMask használatával."""
|
||||
if not cls.GOOGLE_API_KEY:
|
||||
return None
|
||||
async def get_google_places(cls, lat, lon, keyword):
|
||||
"""Google Places New API - Javított, 400-as hiba elleni védelemmel."""
|
||||
if not cls.GOOGLE_API_KEY: return []
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-Goog-Api-Key": cls.GOOGLE_API_KEY,
|
||||
"X-Goog-FieldMask": "places.displayName,places.id,places.types,places.internationalPhoneNumber,places.websiteUri"
|
||||
"X-Goog-FieldMask": "places.displayName,places.id,places.types,places.internationalPhoneNumber,places.websiteUri,places.formattedAddress"
|
||||
}
|
||||
|
||||
# A 'keyword' a TextQuery-hez kellene, a SearchNearby-nél típusokat (includedTypes) használunk.
|
||||
# EU szintű trükk: Ha nincs pontos típus, a 'car_repair' az alapértelmezett.
|
||||
payload = {
|
||||
"includedTypes": ["car_repair", "gas_station", "ev_charging_station", "car_wash", "motorcycle_repair"],
|
||||
"maxResultCount": 1,
|
||||
"includedTypes": ["car_repair", "gas_station", "car_wash", "motorcycle_repair"],
|
||||
"maxResultCount": 20,
|
||||
"locationRestriction": {
|
||||
"circle": {
|
||||
"center": {"latitude": lat, "longitude": lon},
|
||||
"radius": 40.0
|
||||
"radius": 5000.0 # 5km körzet
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(cls.PLACES_NEW_URL, json=payload, headers=headers, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
places = resp.json().get("places", [])
|
||||
if places:
|
||||
p = places[0]
|
||||
return {
|
||||
"name": p.get("displayName", {}).get("text"),
|
||||
"google_id": p.get("id"),
|
||||
"types": p.get("types", []),
|
||||
"phone": p.get("internationalPhoneNumber"),
|
||||
"website": p.get("websiteUri")
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Google kiegészítő hívás hiba: {e}")
|
||||
return None
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(cls.PLACES_NEW_URL, json=payload, headers=headers)
|
||||
if resp.status_code == 200:
|
||||
return resp.json().get("places", [])
|
||||
else:
|
||||
logger.error(f"❌ Google API hiba ({resp.status_code}): {resp.text}")
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def import_local_csv(cls, db: AsyncSession):
|
||||
"""Manuális adatok betöltése CSV-ből."""
|
||||
if not os.path.exists(cls.LOCAL_CSV_PATH):
|
||||
return
|
||||
async def save_to_staging(cls, db: AsyncSession, data: dict):
|
||||
"""Mentés a Staging táblába 9-mezős bontással."""
|
||||
stmt = select(ServiceStaging).where(ServiceStaging.external_id == str(data['external_id']))
|
||||
if (await db.execute(stmt)).scalar_one_or_none(): return
|
||||
|
||||
try:
|
||||
with open(cls.LOCAL_CSV_PATH, mode='r', encoding='utf-8') as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
geo_data = None
|
||||
if row.get('cim'):
|
||||
geo_data = await cls.geocode_address(row['cim'])
|
||||
|
||||
if geo_data:
|
||||
element = {
|
||||
"tags": {
|
||||
"name": row['nev'], "phone": row.get('telefon'),
|
||||
"website": row.get('web'), "amenity": row.get('tipus', 'car_repair'),
|
||||
"addr:full": row.get('cim'),
|
||||
"addr:city": geo_data["city"], "addr:zip": geo_data["zip"],
|
||||
"addr:street": geo_data["street"], "addr:type": geo_data["type"],
|
||||
"addr:number": geo_data["number"]
|
||||
},
|
||||
"lat": geo_data["lat"], "lon": geo_data["lng"]
|
||||
}
|
||||
await cls.save_service_deep(db, element, source="local_manual")
|
||||
logger.info("✅ Helyi CSV adatok feldolgozva.")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ CSV feldolgozási hiba: {e}")
|
||||
|
||||
@classmethod
|
||||
async def get_or_create_person(cls, db: AsyncSession, name: str) -> Person:
|
||||
"""Ghost Person kezelése."""
|
||||
names = name.split(' ', 1)
|
||||
last_name = names[0]
|
||||
first_name = names[1] if len(names) > 1 else "Ismeretlen"
|
||||
stmt = select(Person).where(Person.last_name == last_name, Person.first_name == first_name)
|
||||
result = await db.execute(stmt); person = result.scalar_one_or_none()
|
||||
if not person:
|
||||
person = Person(last_name=last_name, first_name=first_name, is_ghost=True, is_active=False)
|
||||
db.add(person); await db.flush()
|
||||
return person
|
||||
|
||||
@classmethod
|
||||
async def enrich_financials(cls, db: AsyncSession, org_id: int):
|
||||
"""Pénzügyi rekord inicializálása."""
|
||||
financial = OrganizationFinancials(
|
||||
organization_id=org_id, year=datetime.now(timezone.utc).year - 1, source="bot_discovery"
|
||||
new_entry = ServiceStaging(
|
||||
name=data['name'],
|
||||
source=data['source'],
|
||||
external_id=str(data['external_id']),
|
||||
# Itt történik a 9-mezős bontás (ha érkezik adat)
|
||||
postal_code=data.get('zip'),
|
||||
city=data.get('city'),
|
||||
street_name=data.get('street'),
|
||||
street_type=data.get('street_type', 'utca'),
|
||||
house_number=data.get('number'),
|
||||
full_address=data.get('full_address'),
|
||||
contact_phone=data.get('phone'),
|
||||
website=data.get('website'),
|
||||
raw_data=data.get('raw', {}),
|
||||
status="pending",
|
||||
trust_score=data.get('trust', 10)
|
||||
)
|
||||
db.add(financial)
|
||||
|
||||
@classmethod
|
||||
async def save_service_deep(cls, db: AsyncSession, element: dict, source="osm"):
|
||||
"""Mély mentés a modelled specifikus mezőneveivel és kötelező értékeivel."""
|
||||
tags = element.get("tags", {})
|
||||
lat, lon = element.get("lat"), element.get("lon")
|
||||
if not lat or not lon: return
|
||||
|
||||
osm_name = tags.get("name") or tags.get("brand") or tags.get("operator")
|
||||
google_data = None
|
||||
if not osm_name or osm_name.lower() in ['aprilia', 'bosch', 'shell', 'mol', 'omv', 'ismeretlen']:
|
||||
google_data = await cls.get_google_place_details_new(lat, lon)
|
||||
|
||||
final_name = (google_data["name"] if google_data else osm_name) or "Ismeretlen Szolgáltató"
|
||||
|
||||
stmt = select(Organization).where(Organization.full_name == final_name)
|
||||
result = await db.execute(stmt); org = result.scalar_one_or_none()
|
||||
|
||||
if not org:
|
||||
# 1. Address létrehozása (a kötelező mezőket kitöltjük az átadott tags-ből vagy alapértékkel)
|
||||
new_addr = Address(
|
||||
latitude=lat,
|
||||
longitude=lon,
|
||||
full_address_text=tags.get("addr:full") or f"2120 Dunakeszi, {tags.get('addr:street', 'Ismeretlen')} {tags.get('addr:housenumber', '1')}",
|
||||
street_name=tags.get("addr:street") or "Ismeretlen",
|
||||
street_type=tags.get("addr:type") or "utca",
|
||||
house_number=tags.get("addr:number") or tags.get("addr:housenumber") or "1"
|
||||
)
|
||||
db.add(new_addr); await db.flush()
|
||||
|
||||
# 2. Organization létrehozása (a modelled alapján ezek a mezők itt vannak)
|
||||
org = Organization(
|
||||
full_name=final_name,
|
||||
name=final_name[:50],
|
||||
org_type=OrgType.service,
|
||||
address_id=new_addr.id,
|
||||
address_city=tags.get("addr:city") or "Dunakeszi",
|
||||
address_zip=tags.get("addr:zip") or "2120",
|
||||
address_street_name=new_addr.street_name,
|
||||
address_street_type=new_addr.street_type,
|
||||
address_house_number=new_addr.house_number
|
||||
)
|
||||
db.add(org); await db.flush()
|
||||
|
||||
# 3. Service Profile
|
||||
trust = 50 if source == "local_manual" else (30 if google_data else 10)
|
||||
spec = {"brands": [], "types": google_data["types"] if google_data else [], "osm_tags": tags}
|
||||
if tags.get("brand"): spec["brands"].append(tags.get("brand"))
|
||||
|
||||
profile = ServiceProfile(
|
||||
organization_id=org.id,
|
||||
location=WKTElement(f'POINT({lon} {lat})', srid=4326),
|
||||
status="ghost",
|
||||
trust_score=trust,
|
||||
google_place_id=google_data["google_id"] if google_data else None,
|
||||
specialization_tags=spec,
|
||||
website=google_data["website"] if google_data else tags.get("website"),
|
||||
contact_phone=google_data["phone"] if google_data else tags.get("phone")
|
||||
)
|
||||
db.add(profile)
|
||||
|
||||
# 4. Tulajdonos rögzítése
|
||||
owner_name = tags.get("operator") or tags.get("contact:person")
|
||||
if owner_name and len(owner_name) > 3:
|
||||
person = await cls.get_or_create_person(db, owner_name)
|
||||
db.add(OrganizationMember(
|
||||
organization_id=org.id,
|
||||
person_id=person.id,
|
||||
role=OrgUserRole.OWNER,
|
||||
is_verified=False
|
||||
))
|
||||
|
||||
await cls.enrich_financials(db, org.id)
|
||||
await db.flush()
|
||||
logger.info(f"✨ [{source.upper()}] Mentve: {final_name} (Bizalom: {trust})")
|
||||
db.add(new_entry)
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Robot 2.7.2: Dunakeszi Detective indítása...")
|
||||
logger.info("🤖 Robot v1.3.0: Continental Scout elindult...")
|
||||
|
||||
# Kapcsolódási védelem
|
||||
connected = False
|
||||
while not connected:
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
await db.execute(text("SELECT 1"))
|
||||
connected = True
|
||||
except Exception as e:
|
||||
logger.warning(f"⏳ Várakozás a hálózatra (shared-postgres host?): {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
while True:
|
||||
async with SessionLocal() as db:
|
||||
try:
|
||||
await db.execute(text("SET search_path TO data, public"))
|
||||
# 1. Beküldött CSV feldolgozása (Geocoding-al)
|
||||
await cls.import_local_csv(db)
|
||||
await db.commit()
|
||||
|
||||
# 2. OSM Szkennelés
|
||||
query = """[out:json][timeout:120];area["name"="Dunakeszi"]->.city;(nwr["shop"~"car_repair|motorcycle_repair|tyres|car_parts|motorcycle"](area.city);nwr["amenity"~"car_repair|vehicle_inspection|motorcycle_repair|fuel|charging_station|car_wash"](area.city);nwr["amenity"~"car_repair|fuel|charging_station"](around:5000, 47.63, 19.13););out center;"""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(cls.OVERPASS_URL, data={"data": query}, timeout=120)
|
||||
if resp.status_code == 200:
|
||||
elements = resp.json().get("elements", [])
|
||||
for el in elements:
|
||||
await cls.save_service_deep(db, el, source="osm")
|
||||
await db.commit()
|
||||
# 1. Paraméterek lekérése a táblából
|
||||
stmt = select(DiscoveryParameter).where(DiscoveryParameter.is_active == True)
|
||||
tasks = (await db.execute(stmt)).scalars().all()
|
||||
|
||||
for task in tasks:
|
||||
logger.info(f"🔎 Felderítés: {task.city} ({task.country_code}) -> {task.keyword}")
|
||||
|
||||
# Koordináták beszerzése a kereséshez
|
||||
lat, lon = await cls.get_coordinates(task.city, task.country_code)
|
||||
if not lat: continue
|
||||
|
||||
# --- GOOGLE FÁZIS ---
|
||||
google_places = await cls.get_google_places(lat, lon, task.keyword)
|
||||
for p in google_places:
|
||||
await cls.save_to_staging(db, {
|
||||
"external_id": p.get('id'),
|
||||
"name": p.get('displayName', {}).get('text'),
|
||||
"full_address": p.get('formattedAddress'),
|
||||
"phone": p.get('internationalPhoneNumber'),
|
||||
"website": p.get('websiteUri'),
|
||||
"source": "google",
|
||||
"raw": p,
|
||||
"trust": 30
|
||||
})
|
||||
|
||||
# --- OSM FÁZIS (EU kompatibilis lekérdezés) ---
|
||||
osm_query = f"""[out:json][timeout:60];
|
||||
(nwr["amenity"~"car_repair|fuel"](around:5000, {lat}, {lon}););
|
||||
out center;"""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(cls.OVERPASS_URL, data={"data": osm_query})
|
||||
if resp.status_code == 200:
|
||||
for el in resp.json().get("elements", []):
|
||||
t = el.get("tags", {})
|
||||
await cls.save_to_staging(db, {
|
||||
"external_id": f"osm_{el['id']}",
|
||||
"name": t.get('name', 'Ismeretlen szerviz'),
|
||||
"city": t.get('addr:city', task.city),
|
||||
"zip": t.get('addr:postcode'),
|
||||
"street": t.get('addr:street'),
|
||||
"number": t.get('addr:housenumber'),
|
||||
"source": "osm",
|
||||
"raw": el,
|
||||
"trust": 15
|
||||
})
|
||||
|
||||
task.last_run_at = datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
logger.info(f"✅ {task.city} felderítve.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Futáshiba: {e}")
|
||||
logger.error(f"💥 Kritikus hiba a ciklusban: {e}")
|
||||
|
||||
logger.info("😴 Scan kész, 24 óra pihenő...")
|
||||
await asyncio.sleep(86400)
|
||||
logger.info("😴 Minden aktív feladat kész. Alvás 1 órán át...")
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(ServiceHunter.run())
|
||||
282
backend/app/workers/service_hunter_old.py
Normal file
282
backend/app/workers/service_hunter_old.py
Normal file
@@ -0,0 +1,282 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import uuid
|
||||
import os
|
||||
import sys
|
||||
import csv
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import selectinload
|
||||
from app.db.session import SessionLocal
|
||||
|
||||
# Modellek importálása
|
||||
from app.models.service import ServiceProfile, ExpertiseTag
|
||||
from app.models.organization import Organization, OrganizationFinancials, OrgType, OrgUserRole, OrganizationMember
|
||||
from app.models.identity import Person
|
||||
from app.models.address import Address, GeoPostalCode
|
||||
from geoalchemy2.elements import WKTElement
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# Naplózás beállítása
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("Robot2-Dunakeszi-Detective")
|
||||
|
||||
class ServiceHunter:
|
||||
"""
|
||||
Robot 2.7.2: Dunakeszi Detective - Deep Model Integration.
|
||||
Logika:
|
||||
1. Helyi CSV (Saját beküldés - Cím alapú Geocoding-al - 50 pont Trust)
|
||||
2. OSM (Közösségi adat - 10 pont Trust)
|
||||
3. Google (Adatpótlás/Fallback - 30 pont Trust)
|
||||
"""
|
||||
OVERPASS_URL = "http://overpass-api.de/api/interpreter"
|
||||
PLACES_NEW_URL = "https://places.googleapis.com/v1/places:searchNearby"
|
||||
GEOCODE_URL = "https://maps.googleapis.com/maps/api/geocode/json"
|
||||
GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
|
||||
LOCAL_CSV_PATH = "/app/app/workers/local_services.csv"
|
||||
|
||||
@classmethod
|
||||
async def geocode_address(cls, address_text):
|
||||
"""Cím szövegből GPS koordinátát és címkomponenseket csinál."""
|
||||
if not cls.GOOGLE_API_KEY:
|
||||
logger.warning("⚠️ Google API kulcs hiányzik!")
|
||||
return None
|
||||
|
||||
params = {"address": address_text, "key": cls.GOOGLE_API_KEY}
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(cls.GEOCODE_URL, params=params, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
if data.get("results"):
|
||||
result = data["results"][0]
|
||||
loc = result["geometry"]["location"]
|
||||
|
||||
# Címkomponensek kinyerése a kötelező mezőkhöz
|
||||
components = result.get("address_components", [])
|
||||
parsed = {"lat": loc["lat"], "lng": loc["lng"], "zip": "", "city": "", "street": "Ismeretlen", "type": "utca", "number": "1"}
|
||||
|
||||
for c in components:
|
||||
types = c.get("types", [])
|
||||
if "postal_code" in types: parsed["zip"] = c["long_name"]
|
||||
if "locality" in types: parsed["city"] = c["long_name"]
|
||||
if "route" in types: parsed["street"] = c["long_name"]
|
||||
if "street_number" in types: parsed["number"] = c["long_name"]
|
||||
|
||||
logger.info(f"📍 Geocoding sikeres: {address_text}")
|
||||
return parsed
|
||||
else:
|
||||
logger.error(f"❌ Geocoding hiba: {resp.status_code}")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Geocoding hiba: {e}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def get_google_place_details_new(cls, lat, lon):
|
||||
"""Google Places API (New) - Adatpótlás FieldMask használatával."""
|
||||
if not cls.GOOGLE_API_KEY:
|
||||
return None
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-Goog-Api-Key": cls.GOOGLE_API_KEY,
|
||||
"X-Goog-FieldMask": "places.displayName,places.id,places.types,places.internationalPhoneNumber,places.websiteUri"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"includedTypes": ["car_repair", "gas_station", "ev_charging_station", "car_wash", "motorcycle_repair"],
|
||||
"maxResultCount": 1,
|
||||
"locationRestriction": {
|
||||
"circle": {
|
||||
"center": {"latitude": lat, "longitude": lon},
|
||||
"radius": 40.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(cls.PLACES_NEW_URL, json=payload, headers=headers, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
places = resp.json().get("places", [])
|
||||
if places:
|
||||
p = places[0]
|
||||
return {
|
||||
"name": p.get("displayName", {}).get("text"),
|
||||
"google_id": p.get("id"),
|
||||
"types": p.get("types", []),
|
||||
"phone": p.get("internationalPhoneNumber"),
|
||||
"website": p.get("websiteUri")
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Google kiegészítő hívás hiba: {e}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def import_local_csv(cls, db: AsyncSession):
|
||||
"""Manuális adatok betöltése CSV-ből."""
|
||||
if not os.path.exists(cls.LOCAL_CSV_PATH):
|
||||
return
|
||||
|
||||
try:
|
||||
with open(cls.LOCAL_CSV_PATH, mode='r', encoding='utf-8') as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
geo_data = None
|
||||
if row.get('cim'):
|
||||
geo_data = await cls.geocode_address(row['cim'])
|
||||
|
||||
if geo_data:
|
||||
element = {
|
||||
"tags": {
|
||||
"name": row['nev'], "phone": row.get('telefon'),
|
||||
"website": row.get('web'), "amenity": row.get('tipus', 'car_repair'),
|
||||
"addr:full": row.get('cim'),
|
||||
"addr:city": geo_data["city"], "addr:zip": geo_data["zip"],
|
||||
"addr:street": geo_data["street"], "addr:type": geo_data["type"],
|
||||
"addr:number": geo_data["number"]
|
||||
},
|
||||
"lat": geo_data["lat"], "lon": geo_data["lng"]
|
||||
}
|
||||
await cls.save_service_deep(db, element, source="local_manual")
|
||||
logger.info("✅ Helyi CSV adatok feldolgozva.")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ CSV feldolgozási hiba: {e}")
|
||||
|
||||
@classmethod
|
||||
async def get_or_create_person(cls, db: AsyncSession, name: str) -> Person:
|
||||
"""Ghost Person kezelése."""
|
||||
names = name.split(' ', 1)
|
||||
last_name = names[0]
|
||||
first_name = names[1] if len(names) > 1 else "Ismeretlen"
|
||||
stmt = select(Person).where(Person.last_name == last_name, Person.first_name == first_name)
|
||||
result = await db.execute(stmt); person = result.scalar_one_or_none()
|
||||
if not person:
|
||||
person = Person(last_name=last_name, first_name=first_name, is_ghost=True, is_active=False)
|
||||
db.add(person); await db.flush()
|
||||
return person
|
||||
|
||||
@classmethod
|
||||
async def enrich_financials(cls, db: AsyncSession, org_id: int):
|
||||
"""Pénzügyi rekord inicializálása."""
|
||||
financial = OrganizationFinancials(
|
||||
organization_id=org_id, year=datetime.now(timezone.utc).year - 1, source="bot_discovery"
|
||||
)
|
||||
db.add(financial)
|
||||
|
||||
@classmethod
|
||||
async def save_service_deep(cls, db: AsyncSession, element: dict, source="osm"):
|
||||
"""Mély mentés a modelled specifikus mezőneveivel és kötelező értékeivel."""
|
||||
tags = element.get("tags", {})
|
||||
lat, lon = element.get("lat"), element.get("lon")
|
||||
if not lat or not lon: return
|
||||
|
||||
osm_name = tags.get("name") or tags.get("brand") or tags.get("operator")
|
||||
google_data = None
|
||||
if not osm_name or osm_name.lower() in ['aprilia', 'bosch', 'shell', 'mol', 'omv', 'ismeretlen']:
|
||||
google_data = await cls.get_google_place_details_new(lat, lon)
|
||||
|
||||
final_name = (google_data["name"] if google_data else osm_name) or "Ismeretlen Szolgáltató"
|
||||
|
||||
stmt = select(Organization).where(Organization.full_name == final_name)
|
||||
result = await db.execute(stmt); org = result.scalar_one_or_none()
|
||||
|
||||
if not org:
|
||||
# 1. Address létrehozása (a kötelező mezőket kitöltjük az átadott tags-ből vagy alapértékkel)
|
||||
new_addr = Address(
|
||||
latitude=lat,
|
||||
longitude=lon,
|
||||
full_address_text=tags.get("addr:full") or f"2120 Dunakeszi, {tags.get('addr:street', 'Ismeretlen')} {tags.get('addr:housenumber', '1')}",
|
||||
street_name=tags.get("addr:street") or "Ismeretlen",
|
||||
street_type=tags.get("addr:type") or "utca",
|
||||
house_number=tags.get("addr:number") or tags.get("addr:housenumber") or "1"
|
||||
)
|
||||
db.add(new_addr); await db.flush()
|
||||
|
||||
# 2. Organization létrehozása (a modelled alapján ezek a mezők itt vannak)
|
||||
org = Organization(
|
||||
full_name=final_name,
|
||||
name=final_name[:50],
|
||||
org_type=OrgType.service,
|
||||
address_id=new_addr.id,
|
||||
address_city=tags.get("addr:city") or "Dunakeszi",
|
||||
address_zip=tags.get("addr:zip") or "2120",
|
||||
address_street_name=new_addr.street_name,
|
||||
address_street_type=new_addr.street_type,
|
||||
address_house_number=new_addr.house_number
|
||||
)
|
||||
db.add(org); await db.flush()
|
||||
|
||||
# 3. Service Profile
|
||||
trust = 50 if source == "local_manual" else (30 if google_data else 10)
|
||||
spec = {"brands": [], "types": google_data["types"] if google_data else [], "osm_tags": tags}
|
||||
if tags.get("brand"): spec["brands"].append(tags.get("brand"))
|
||||
|
||||
profile = ServiceProfile(
|
||||
organization_id=org.id,
|
||||
location=WKTElement(f'POINT({lon} {lat})', srid=4326),
|
||||
status="ghost",
|
||||
trust_score=trust,
|
||||
google_place_id=google_data["google_id"] if google_data else None,
|
||||
specialization_tags=spec,
|
||||
website=google_data["website"] if google_data else tags.get("website"),
|
||||
contact_phone=google_data["phone"] if google_data else tags.get("phone")
|
||||
)
|
||||
db.add(profile)
|
||||
|
||||
# 4. Tulajdonos rögzítése
|
||||
owner_name = tags.get("operator") or tags.get("contact:person")
|
||||
if owner_name and len(owner_name) > 3:
|
||||
person = await cls.get_or_create_person(db, owner_name)
|
||||
db.add(OrganizationMember(
|
||||
organization_id=org.id,
|
||||
person_id=person.id,
|
||||
role=OrgUserRole.OWNER,
|
||||
is_verified=False
|
||||
))
|
||||
|
||||
await cls.enrich_financials(db, org.id)
|
||||
await db.flush()
|
||||
logger.info(f"✨ [{source.upper()}] Mentve: {final_name} (Bizalom: {trust})")
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Robot 2.7.2: Dunakeszi Detective indítása...")
|
||||
|
||||
# Kapcsolódási védelem
|
||||
connected = False
|
||||
while not connected:
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
await db.execute(text("SELECT 1"))
|
||||
connected = True
|
||||
except Exception as e:
|
||||
logger.warning(f"⏳ Várakozás a hálózatra (shared-postgres host?): {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
while True:
|
||||
async with SessionLocal() as db:
|
||||
try:
|
||||
await db.execute(text("SET search_path TO data, public"))
|
||||
# 1. Beküldött CSV feldolgozása (Geocoding-al)
|
||||
await cls.import_local_csv(db)
|
||||
await db.commit()
|
||||
|
||||
# 2. OSM Szkennelés
|
||||
query = """[out:json][timeout:120];area["name"="Dunakeszi"]->.city;(nwr["shop"~"car_repair|motorcycle_repair|tyres|car_parts|motorcycle"](area.city);nwr["amenity"~"car_repair|vehicle_inspection|motorcycle_repair|fuel|charging_station|car_wash"](area.city);nwr["amenity"~"car_repair|fuel|charging_station"](around:5000, 47.63, 19.13););out center;"""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(cls.OVERPASS_URL, data={"data": query}, timeout=120)
|
||||
if resp.status_code == 200:
|
||||
elements = resp.json().get("elements", [])
|
||||
for el in elements:
|
||||
await cls.save_service_deep(db, el, source="osm")
|
||||
await db.commit()
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Futáshiba: {e}")
|
||||
|
||||
logger.info("😴 Scan kész, 24 óra pihenő...")
|
||||
await asyncio.sleep(86400)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(ServiceHunter.run())
|
||||
125
backend/app/workers/technical_enricher.py
Normal file
125
backend/app/workers/technical_enricher.py
Normal file
@@ -0,0 +1,125 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import datetime
|
||||
from sqlalchemy import text
|
||||
from app.db.session import SessionLocal
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("Robot-v1.0.4-Master-Enricher")
|
||||
|
||||
class TechEnricher:
|
||||
"""
|
||||
Master Enricher v1.0.4
|
||||
- Target: kyri-nuah (RDW Technical Catalogue)
|
||||
- Fix: Visszaállás 'merk' mezőre + SQL fix az új oszlopokhoz.
|
||||
"""
|
||||
|
||||
API_URL = "https://opendata.rdw.nl/resource/kyri-nuah.json"
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_data(cls, make, model):
|
||||
# Tisztítás: Ha a modell névben benne van a márka, levágjuk
|
||||
clean_model = str(model).upper().replace(str(make).upper(), "").strip()
|
||||
|
||||
# Ha a modellnév csak szám vagy túl rövid, az RDW nem fogja szeretni
|
||||
if len(clean_model) < 2:
|
||||
return None
|
||||
|
||||
# PRÓBA 1: A 'merk' mezővel (Ez a leggyakoribb)
|
||||
params = {
|
||||
"merk": make.upper(),
|
||||
"handelsbenaming": clean_model,
|
||||
"$limit": 1
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(headers=cls.HEADERS) as client:
|
||||
try:
|
||||
await asyncio.sleep(1.1)
|
||||
resp = await client.get(cls.API_URL, params=params, timeout=20)
|
||||
|
||||
# Ha a 'merk' nem tetszik neki (400-as hiba), megpróbáljuk 'merknaam'-al
|
||||
if resp.status_code == 400:
|
||||
params = {"merknaam": make.upper(), "handelsbenaming": clean_model, "$limit": 1}
|
||||
resp = await client.get(cls.TECH_API_URL, params=params, timeout=20)
|
||||
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
return data[0] if data else None
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"❌ API Hiba: {e}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🚀 Master Enricher v1.0.4 - Új oszlopok töltése indul...")
|
||||
|
||||
while True:
|
||||
async with SessionLocal() as db:
|
||||
# Olyan sorokat keresünk, ahol az új oszlopok még üresek
|
||||
query = text("""
|
||||
SELECT id, make, model
|
||||
FROM data.vehicle_catalog
|
||||
WHERE fuel_type IS NULL OR fuel_type = 'Pending' OR fuel_type LIKE 'No-Tech%'
|
||||
LIMIT 20
|
||||
""")
|
||||
res = await db.execute(query)
|
||||
tasks = res.fetchall()
|
||||
|
||||
if not tasks:
|
||||
logger.info("😴 Minden adat kész. Alvás 5 perc...")
|
||||
await asyncio.sleep(300)
|
||||
continue
|
||||
|
||||
for t_id, make, model in tasks:
|
||||
logger.info(f"🧪 Gazdagítás: {make} | {model}")
|
||||
tech = await cls.fetch_tech_data(make, model)
|
||||
|
||||
if tech:
|
||||
# RDW mezők kinyerése
|
||||
kw = tech.get("netto_maximum_vermogen_kw")
|
||||
ccm = tech.get("cilinderinhoud")
|
||||
weight = tech.get("technisch_toelaatbare_maximum_massa")
|
||||
axles = tech.get("aantal_assen")
|
||||
euro = tech.get("milieuklasse_eg_goedkeuring_licht")
|
||||
fuel = tech.get("brandstof_omschrijving_brandstof_stam", "Standard")
|
||||
|
||||
# Biztonságos konverzió
|
||||
def clean_num(v):
|
||||
try: return int(float(v)) if v else None
|
||||
except: return None
|
||||
|
||||
update_query = text("""
|
||||
UPDATE data.vehicle_catalog
|
||||
SET fuel_type = :fuel,
|
||||
power_kw = :kw,
|
||||
engine_capacity = :ccm,
|
||||
max_weight_kg = :weight,
|
||||
axle_count = :axles,
|
||||
euro_class = :euro,
|
||||
factory_data = factory_data || jsonb_build_object('enriched_at', :now)
|
||||
WHERE id = :id
|
||||
""")
|
||||
|
||||
await db.execute(update_query, {
|
||||
"fuel": fuel, "kw": clean_num(kw), "ccm": clean_num(ccm),
|
||||
"weight": clean_num(weight), "axles": clean_num(axles),
|
||||
"euro": str(euro) if euro else None,
|
||||
"id": t_id, "now": str(datetime.datetime.now())
|
||||
})
|
||||
await db.commit()
|
||||
logger.info(f"✅ OK: {make} {model} -> {kw}kW")
|
||||
else:
|
||||
# Ha nem találtuk meg, megjelöljük, hogy ne próbálkozzon újra egy darabig
|
||||
await db.execute(text("UPDATE data.vehicle_catalog SET fuel_type = 'No-Tech-V4' WHERE id = :id"), {"id": t_id})
|
||||
await db.commit()
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(TechEnricher.run())
|
||||
Reference in New Issue
Block a user