2026.03.29 20:00 Gitea_manager javítás előtt
This commit is contained in:
@@ -102,6 +102,58 @@ async def list_asset_costs(
|
||||
return res.scalars().all()
|
||||
|
||||
|
||||
@router.get("/{asset_id}", response_model=AssetResponse)
|
||||
async def get_asset(
|
||||
asset_id: uuid.UUID,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Get detailed information about a specific vehicle/asset.
|
||||
|
||||
Returns the asset's full technical profile including catalog data
|
||||
and vehicle model definition specifications.
|
||||
"""
|
||||
# Check if user has access to this asset
|
||||
from sqlalchemy import or_
|
||||
from app.models.marketplace.organization import OrganizationMember
|
||||
|
||||
# Get user's organization memberships
|
||||
org_stmt = select(OrganizationMember.organization_id).where(
|
||||
OrganizationMember.user_id == current_user.id
|
||||
)
|
||||
org_result = await db.execute(org_stmt)
|
||||
user_org_ids = [row[0] for row in org_result.all()]
|
||||
|
||||
# Query asset with catalog and master definition
|
||||
stmt = (
|
||||
select(Asset)
|
||||
.where(
|
||||
Asset.id == asset_id,
|
||||
or_(
|
||||
Asset.owner_person_id == current_user.id,
|
||||
Asset.owner_org_id.in_(user_org_ids) if user_org_ids else False,
|
||||
Asset.operator_person_id == current_user.id,
|
||||
Asset.operator_org_id.in_(user_org_ids) if user_org_ids else False
|
||||
)
|
||||
)
|
||||
.options(
|
||||
selectinload(Asset.catalog).selectinload(AssetCatalog.master_definition)
|
||||
)
|
||||
)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
asset = result.scalar_one_or_none()
|
||||
|
||||
if not asset:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Asset not found or you don't have permission to access it"
|
||||
)
|
||||
|
||||
return asset
|
||||
|
||||
|
||||
@router.post("/vehicles", response_model=AssetResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_or_claim_vehicle(
|
||||
payload: AssetCreate,
|
||||
@@ -118,10 +170,30 @@ async def create_or_claim_vehicle(
|
||||
- XP jutalom adása a felhasználónak
|
||||
"""
|
||||
try:
|
||||
# Determine organization ID: use provided or default to user's first organization
|
||||
org_id = payload.organization_id
|
||||
if org_id is None:
|
||||
# Get user's organization memberships
|
||||
from app.models.marketplace.organization import OrganizationMember
|
||||
org_stmt = select(OrganizationMember.organization_id).where(
|
||||
OrganizationMember.user_id == current_user.id
|
||||
).limit(1)
|
||||
org_result = await db.execute(org_stmt)
|
||||
user_org = org_result.scalar_one_or_none()
|
||||
|
||||
if user_org is None:
|
||||
# User has no organization - create a personal organization or use default
|
||||
# For now, raise an error (in future, we could create a personal org)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="No organization found for user. Please specify an organization_id or join/create an organization first."
|
||||
)
|
||||
org_id = user_org
|
||||
|
||||
asset = await AssetService.create_or_claim_vehicle(
|
||||
db=db,
|
||||
user_id=current_user.id,
|
||||
org_id=payload.organization_id,
|
||||
org_id=org_id,
|
||||
vin=payload.vin,
|
||||
license_plate=payload.license_plate,
|
||||
catalog_id=payload.catalog_id
|
||||
@@ -134,4 +206,205 @@ async def create_or_claim_vehicle(
|
||||
except Exception as e:
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.error(f"Vehicle creation error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Belső szerverhiba a jármű létrehozásakor")
|
||||
raise HTTPException(status_code=500, detail="Belső szerverhiba a jármű létrehozásakor")
|
||||
|
||||
|
||||
@router.get("/{asset_id}/maintenance", response_model=List[AssetCostResponse])
|
||||
async def list_maintenance_records(
|
||||
asset_id: uuid.UUID,
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
List maintenance records for a specific vehicle.
|
||||
|
||||
Returns paginated list of maintenance costs with cost_category = 'maintenance'.
|
||||
"""
|
||||
# Check if user has access to this asset
|
||||
from sqlalchemy import or_
|
||||
from app.models.marketplace.organization import OrganizationMember
|
||||
|
||||
# Get user's organization memberships
|
||||
org_stmt = select(OrganizationMember.organization_id).where(
|
||||
OrganizationMember.user_id == current_user.id
|
||||
)
|
||||
org_result = await db.execute(org_stmt)
|
||||
user_org_ids = [row[0] for row in org_result.all()]
|
||||
|
||||
# Check asset access
|
||||
asset_stmt = select(Asset).where(
|
||||
Asset.id == asset_id,
|
||||
or_(
|
||||
Asset.owner_person_id == current_user.id,
|
||||
Asset.owner_org_id.in_(user_org_ids) if user_org_ids else False,
|
||||
Asset.operator_person_id == current_user.id,
|
||||
Asset.operator_org_id.in_(user_org_ids) if user_org_ids else False
|
||||
)
|
||||
)
|
||||
asset_result = await db.execute(asset_stmt)
|
||||
asset = asset_result.scalar_one_or_none()
|
||||
|
||||
if not asset:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Asset not found or you don't have permission to access it"
|
||||
)
|
||||
|
||||
# Query maintenance costs
|
||||
stmt = (
|
||||
select(AssetCost)
|
||||
.where(
|
||||
AssetCost.asset_id == asset_id,
|
||||
AssetCost.cost_category == "maintenance"
|
||||
)
|
||||
.order_by(desc(AssetCost.date))
|
||||
.offset(skip)
|
||||
.limit(limit)
|
||||
)
|
||||
res = await db.execute(stmt)
|
||||
return res.scalars().all()
|
||||
|
||||
|
||||
@router.post("/{asset_id}/maintenance", response_model=AssetCostResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_maintenance_record(
|
||||
asset_id: uuid.UUID,
|
||||
payload: Dict[str, Any],
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Add a maintenance record for a vehicle.
|
||||
|
||||
Expected payload fields:
|
||||
- date: ISO datetime string (required)
|
||||
- odometer: integer (optional, current mileage)
|
||||
- description: string (required)
|
||||
- cost: float (required, net amount)
|
||||
- currency: string (optional, default: "EUR")
|
||||
- invoice_number: string (optional)
|
||||
"""
|
||||
# Check if user has access to this asset
|
||||
from sqlalchemy import or_
|
||||
from app.models.marketplace.organization import OrganizationMember
|
||||
|
||||
# Get user's organization memberships
|
||||
org_stmt = select(OrganizationMember.organization_id).where(
|
||||
OrganizationMember.user_id == current_user.id
|
||||
)
|
||||
org_result = await db.execute(org_stmt)
|
||||
user_org_ids = [row[0] for row in org_result.all()]
|
||||
|
||||
# Check asset access and get asset
|
||||
asset_stmt = select(Asset).where(
|
||||
Asset.id == asset_id,
|
||||
or_(
|
||||
Asset.owner_person_id == current_user.id,
|
||||
Asset.owner_org_id.in_(user_org_ids) if user_org_ids else False,
|
||||
Asset.operator_person_id == current_user.id,
|
||||
Asset.operator_org_id.in_(user_org_ids) if user_org_ids else False
|
||||
)
|
||||
)
|
||||
asset_result = await db.execute(asset_stmt)
|
||||
asset = asset_result.scalar_one_or_none()
|
||||
|
||||
if not asset:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Asset not found or you don't have permission to access it"
|
||||
)
|
||||
|
||||
# Validate required fields
|
||||
required_fields = ["date", "description", "cost"]
|
||||
for field in required_fields:
|
||||
if field not in payload:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Missing required field: {field}"
|
||||
)
|
||||
|
||||
try:
|
||||
# Parse date
|
||||
from datetime import datetime
|
||||
date = datetime.fromisoformat(payload["date"].replace("Z", "+00:00"))
|
||||
|
||||
# Determine organization ID: use asset's current org, owner org, or user's active organization
|
||||
organization_id = asset.current_organization_id or asset.owner_org_id
|
||||
|
||||
if not organization_id:
|
||||
# Get user's active organization from their scope_id
|
||||
if current_user.scope_id:
|
||||
try:
|
||||
organization_id = int(current_user.scope_id)
|
||||
except (ValueError, TypeError):
|
||||
# If scope_id is not a valid integer, try to get from organization memberships
|
||||
from sqlalchemy import select
|
||||
from app.models.marketplace.organization import OrganizationMember
|
||||
stmt = select(OrganizationMember.organization_id).where(
|
||||
OrganizationMember.user_id == current_user.id
|
||||
).limit(1)
|
||||
result = await db.execute(stmt)
|
||||
org_row = result.first()
|
||||
organization_id = org_row[0] if org_row else None
|
||||
else:
|
||||
# Try to get from organization memberships
|
||||
from sqlalchemy import select
|
||||
from app.models.marketplace.organization import OrganizationMember
|
||||
stmt = select(OrganizationMember.organization_id).where(
|
||||
OrganizationMember.user_id == current_user.id
|
||||
).limit(1)
|
||||
result = await db.execute(stmt)
|
||||
org_row = result.first()
|
||||
organization_id = org_row[0] if org_row else None
|
||||
|
||||
if not organization_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Cannot determine organization for this cost record. Please ensure you have an active organization."
|
||||
)
|
||||
|
||||
# Create AssetCost record
|
||||
maintenance_cost = AssetCost(
|
||||
asset_id=asset_id,
|
||||
organization_id=organization_id,
|
||||
cost_category="maintenance",
|
||||
amount_net=float(payload["cost"]),
|
||||
currency=payload.get("currency", "EUR"),
|
||||
date=date,
|
||||
invoice_number=payload.get("invoice_number"),
|
||||
data={
|
||||
"odometer": payload.get("odometer"),
|
||||
"description": payload["description"],
|
||||
"type": "maintenance"
|
||||
}
|
||||
)
|
||||
|
||||
db.add(maintenance_cost)
|
||||
await db.commit()
|
||||
await db.refresh(maintenance_cost)
|
||||
|
||||
# Also create an AssetEvent for the maintenance
|
||||
from app.models.vehicle import AssetEvent
|
||||
maintenance_event = AssetEvent(
|
||||
asset_id=asset_id,
|
||||
event_type="maintenance"
|
||||
)
|
||||
db.add(maintenance_event)
|
||||
await db.commit()
|
||||
|
||||
return maintenance_cost
|
||||
|
||||
except ValueError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Invalid data format: {str(e)}"
|
||||
)
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.error(f"Maintenance record creation error: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Internal server error while creating maintenance record"
|
||||
)
|
||||
@@ -66,4 +66,12 @@ async def complete_kyc(kyc_in: UserKYCComplete, db: AsyncSession = Depends(get_d
|
||||
user = await AuthService.complete_kyc(db, current_user.id, kyc_in)
|
||||
if not user:
|
||||
raise HTTPException(status_code=404, detail="User nem található.")
|
||||
return {"status": "success", "message": "Fiók aktiválva."}
|
||||
return {"status": "success", "message": "Fiók aktiválva."}
|
||||
|
||||
@router.get("/me")
|
||||
async def get_current_user_profile(current_user: User = Depends(get_current_user)):
|
||||
"""
|
||||
Return current user profile (alias for /users/me).
|
||||
"""
|
||||
from app.schemas.user import UserResponse
|
||||
return UserResponse.model_validate(current_user)
|
||||
@@ -24,10 +24,13 @@ async def list_models(
|
||||
current_user = Depends(deps.get_current_user)
|
||||
):
|
||||
"""2. Szint: Típusok listázása egy adott márkához."""
|
||||
# Handle empty or invalid parameters gracefully
|
||||
if not make or make.strip() == "":
|
||||
return []
|
||||
|
||||
models = await AssetService.get_models(db, make)
|
||||
if not models:
|
||||
raise HTTPException(status_code=404, detail="Márka nem található vagy nincsenek típusok.")
|
||||
return models
|
||||
# Return empty list instead of 404 - frontend can handle empty dropdown
|
||||
return models or []
|
||||
|
||||
# Secured endpoint: Closed premium ecosystem
|
||||
@router.get("/generations", response_model=List[str])
|
||||
@@ -38,10 +41,13 @@ async def list_generations(
|
||||
current_user = Depends(deps.get_current_user)
|
||||
):
|
||||
"""3. Szint: Generációk/Évjáratok listázása."""
|
||||
# Handle empty or invalid parameters gracefully
|
||||
if not make or not model or make.strip() == "" or model.strip() == "":
|
||||
return []
|
||||
|
||||
generations = await AssetService.get_generations(db, make, model)
|
||||
if not generations:
|
||||
raise HTTPException(status_code=404, detail="Nincs generációs adat ehhez a típushoz.")
|
||||
return generations
|
||||
# Return empty list instead of 404 - frontend can handle empty dropdown
|
||||
return generations or []
|
||||
|
||||
# Secured endpoint: Closed premium ecosystem
|
||||
@router.get("/engines")
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/api/v1/endpoints/expenses.py
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, func
|
||||
from app.api.deps import get_db, get_current_user
|
||||
from app.models import Asset, AssetCost
|
||||
from app.models import Asset, AssetCost, SystemParameter
|
||||
from app.schemas.asset_cost import AssetCostCreate
|
||||
from datetime import datetime
|
||||
|
||||
@@ -26,6 +26,33 @@ async def create_expense(
|
||||
if not asset:
|
||||
raise HTTPException(status_code=404, detail="Asset not found.")
|
||||
|
||||
# Dynamic Gatekeeper: Check draft expense limit
|
||||
if asset.status == "draft":
|
||||
# 1. Get VEHICLE_DRAFT_MAX_EXPENSES parameter
|
||||
param_stmt = select(SystemParameter).where(
|
||||
SystemParameter.key == "VEHICLE_DRAFT_MAX_EXPENSES",
|
||||
SystemParameter.scope_level == "global"
|
||||
)
|
||||
param_result = await db.execute(param_stmt)
|
||||
param = param_result.scalar_one_or_none()
|
||||
|
||||
if param:
|
||||
limit = param.value.get("limit", 10) # Default to 10 if not found
|
||||
else:
|
||||
limit = 10 # Default fallback
|
||||
|
||||
# 2. Count existing expenses for this asset
|
||||
count_stmt = select(func.count(AssetCost.id)).where(AssetCost.asset_id == asset.id)
|
||||
count_result = await db.execute(count_stmt)
|
||||
expense_count = count_result.scalar()
|
||||
|
||||
# 3. Check if limit reached
|
||||
if expense_count >= limit:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"DRAFT_LIMIT_REACHED: Draft vehicles are limited to {limit} expenses. This asset already has {expense_count} expenses."
|
||||
)
|
||||
|
||||
# Determine organization_id from asset (required by AssetCost model)
|
||||
organization_id = asset.current_organization_id or asset.owner_org_id
|
||||
if not organization_id:
|
||||
|
||||
@@ -17,7 +17,79 @@ async def read_users_me(
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Visszaadja a bejelentkezett felhasználó profilját"""
|
||||
return current_user
|
||||
from sqlalchemy import select, or_
|
||||
from app.models.marketplace.organization import Organization, OrganizationMember
|
||||
from app.models.marketplace.organization import OrgUserRole
|
||||
|
||||
# Determine active organization ID
|
||||
active_org_id = None
|
||||
|
||||
# If user already has a scope_id, use it
|
||||
if current_user.scope_id is not None:
|
||||
try:
|
||||
active_org_id = int(current_user.scope_id)
|
||||
except (ValueError, TypeError):
|
||||
active_org_id = None
|
||||
|
||||
# If still no active org ID, try to find user's primary organization
|
||||
if active_org_id is None:
|
||||
# 1. Check if user is a member of any organization with ADMIN/OWNER role
|
||||
stmt = select(OrganizationMember.organization_id).where(
|
||||
OrganizationMember.user_id == current_user.id,
|
||||
or_(
|
||||
OrganizationMember.role == OrgUserRole.ADMIN,
|
||||
OrganizationMember.role == OrgUserRole.OWNER
|
||||
)
|
||||
).limit(1)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
org_member_row = result.first()
|
||||
|
||||
if org_member_row:
|
||||
active_org_id = org_member_row[0]
|
||||
else:
|
||||
# 2. Check if user owns any organization (owner_id matches user.id)
|
||||
stmt = select(Organization.id).where(
|
||||
Organization.owner_id == current_user.id
|
||||
).limit(1)
|
||||
result = await db.execute(stmt)
|
||||
org_owner_row = result.first()
|
||||
|
||||
if org_owner_row:
|
||||
active_org_id = org_owner_row[0]
|
||||
else:
|
||||
# 3. Fallback: get first organization they're a member of
|
||||
stmt = select(OrganizationMember.organization_id).where(
|
||||
OrganizationMember.user_id == current_user.id
|
||||
).limit(1)
|
||||
result = await db.execute(stmt)
|
||||
org_row = result.first()
|
||||
|
||||
active_org_id = org_row[0] if org_row else None
|
||||
|
||||
# Create a response dictionary with the active_organization_id
|
||||
# Get first_name and last_name from person relation if available
|
||||
person = current_user.person
|
||||
first_name = person.first_name if person else ""
|
||||
last_name = person.last_name if person else ""
|
||||
|
||||
response_data = {
|
||||
"id": current_user.id,
|
||||
"email": current_user.email,
|
||||
"first_name": first_name,
|
||||
"last_name": last_name,
|
||||
"is_active": current_user.is_active,
|
||||
"region_code": current_user.region_code,
|
||||
"person_id": current_user.person_id,
|
||||
"role": current_user.role.value if hasattr(current_user.role, 'value') else str(current_user.role),
|
||||
"subscription_plan": current_user.subscription_plan,
|
||||
"scope_level": current_user.scope_level or "individual",
|
||||
"scope_id": str(active_org_id) if active_org_id else None,
|
||||
"ui_mode": current_user.ui_mode or "personal",
|
||||
"active_organization_id": active_org_id
|
||||
}
|
||||
|
||||
return UserResponse.model_validate(response_data)
|
||||
|
||||
@router.get("/me/trust")
|
||||
async def get_user_trust(
|
||||
|
||||
@@ -66,7 +66,7 @@ app.add_middleware(
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[str(origin) for origin in settings.BACKEND_CORS_ORIGINS],
|
||||
allow_origins=settings.BACKEND_CORS_ORIGINS,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
|
||||
@@ -97,7 +97,12 @@ class ServiceReview(Base):
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
service_id: Mapped[int] = mapped_column(Integer, ForeignKey("marketplace.service_profiles.id", ondelete="CASCADE"), nullable=False)
|
||||
user_id: Mapped[int] = mapped_column(Integer, ForeignKey("identity.users.id", ondelete="SET NULL"), nullable=False)
|
||||
transaction_id: Mapped[uuid.UUID] = mapped_column(PG_UUID(as_uuid=True), nullable=False, index=True)
|
||||
transaction_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PG_UUID(as_uuid=True),
|
||||
ForeignKey("audit.financial_ledger.transaction_id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
index=True
|
||||
)
|
||||
|
||||
# Rating dimensions (1-10)
|
||||
price_rating: Mapped[int] = mapped_column(Integer, nullable=False) # 1-10
|
||||
@@ -113,4 +118,5 @@ class ServiceReview(Base):
|
||||
|
||||
# Relationships
|
||||
service: Mapped["ServiceProfile"] = relationship("ServiceProfile", back_populates="reviews")
|
||||
user: Mapped["User"] = relationship("User", back_populates="service_reviews")
|
||||
user: Mapped["User"] = relationship("User", back_populates="service_reviews")
|
||||
financial_transaction: Mapped["FinancialLedger"] = relationship("FinancialLedger", foreign_keys=[transaction_id])
|
||||
@@ -1,175 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/models/marketplace/service.py
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Optional
|
||||
from sqlalchemy import Integer, String, Boolean, DateTime, ForeignKey, text, Text, Float, Index, Numeric, BigInteger
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from sqlalchemy.dialects.postgresql import UUID as PG_UUID, JSONB
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
# MB 2.0: Központi aszinkron adatbázis motorból húzzuk be a Base-t
|
||||
from app.database import Base
|
||||
|
||||
class ServiceProfile(Base):
|
||||
""" Szerviz szolgáltató adatai (v1.3.1). """
|
||||
__tablename__ = "service_profiles"
|
||||
__table_args__ = (
|
||||
Index('idx_service_fingerprint', 'fingerprint', unique=True),
|
||||
{"schema": "marketplace"}
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
organization_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("fleet.organizations.id"), unique=True)
|
||||
parent_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("marketplace.service_profiles.id"))
|
||||
|
||||
fingerprint: Mapped[str] = mapped_column(String(255), index=True, nullable=False)
|
||||
location: Mapped[Any] = mapped_column(Geometry(geometry_type='POINT', srid=4326, spatial_index=False), index=True)
|
||||
|
||||
status: Mapped[str] = mapped_column(String(20), server_default=text("'ghost'"), index=True)
|
||||
last_audit_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
google_place_id: Mapped[Optional[str]] = mapped_column(String(100), unique=True)
|
||||
rating: Mapped[Optional[float]] = mapped_column(Float)
|
||||
user_ratings_total: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
|
||||
# Aggregated verified review ratings (Social 3)
|
||||
rating_verified_count: Mapped[Optional[int]] = mapped_column(Integer, server_default=text("0"))
|
||||
rating_price_avg: Mapped[Optional[float]] = mapped_column(Float)
|
||||
rating_quality_avg: Mapped[Optional[float]] = mapped_column(Float)
|
||||
rating_time_avg: Mapped[Optional[float]] = mapped_column(Float)
|
||||
rating_communication_avg: Mapped[Optional[float]] = mapped_column(Float)
|
||||
rating_overall: Mapped[Optional[float]] = mapped_column(Float)
|
||||
last_review_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
vibe_analysis: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
social_links: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
specialization_tags: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
trust_score: Mapped[int] = mapped_column(Integer, default=30)
|
||||
is_verified: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
verification_log: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
opening_hours: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
contact_phone: Mapped[Optional[str]] = mapped_column(String)
|
||||
contact_email: Mapped[Optional[str]] = mapped_column(String)
|
||||
website: Mapped[Optional[str]] = mapped_column(String)
|
||||
bio: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
# Kapcsolatok
|
||||
organization: Mapped["Organization"] = relationship("Organization", back_populates="service_profile")
|
||||
expertises: Mapped[List["ServiceExpertise"]] = relationship("ServiceExpertise", back_populates="service")
|
||||
reviews: Mapped[List["ServiceReview"]] = relationship("ServiceReview", back_populates="service")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), onupdate=func.now())
|
||||
|
||||
class ExpertiseTag(Base):
|
||||
"""
|
||||
Szakmai címkék mesterlistája (MB 2.0).
|
||||
Ez a tábla vezérli a robotok keresését és a Gamification pontozást is.
|
||||
"""
|
||||
__tablename__ = "expertise_tags"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
|
||||
# Egyedi azonosító kulcs (pl. 'ENGINE_REBUILD')
|
||||
key: Mapped[str] = mapped_column(String(50), unique=True, index=True)
|
||||
|
||||
# Megjelenítendő nevek
|
||||
name_hu: Mapped[Optional[str]] = mapped_column(String(100))
|
||||
name_en: Mapped[Optional[str]] = mapped_column(String(100))
|
||||
|
||||
# Főcsoport (pl. 'MECHANICS', 'ELECTRICAL', 'EMERGENCY')
|
||||
category: Mapped[Optional[str]] = mapped_column(String(30), index=True)
|
||||
|
||||
# --- 🎮 GAMIFICATION ÉS DISCOVERY ---
|
||||
|
||||
# Hivatalos címke (True) vagy júzer/robot által javasolt (False)
|
||||
is_official: Mapped[bool] = mapped_column(Boolean, default=True, server_default=text("true"))
|
||||
|
||||
# Ha júzer javasolta, itt tároljuk, ki volt az (XP jóváíráshoz)
|
||||
suggested_by_id: Mapped[Optional[int]] = mapped_column(BigInteger, ForeignKey("identity.persons.id"))
|
||||
|
||||
# ÁLLÍTHATÓ PONTÉRTÉK: Az adatbázisból jön, így bármikor módosítható.
|
||||
# Ritka szakmáknál magasabb, gyakoriaknál alacsonyabb érték állítható be.
|
||||
discovery_points: Mapped[int] = mapped_column(Integer, default=10, server_default=text("10"))
|
||||
|
||||
# Robot kulcsszavak (JSONB): ["fék", "betét", "tárcsa", "fékfolyadék"]
|
||||
# A Scout robot ez alapján azonosítja be a szervizt a weboldala alapján.
|
||||
search_keywords: Mapped[Any] = mapped_column(JSONB, server_default=text("'[]'::jsonb"))
|
||||
|
||||
# Népszerűségi mutató (hányszor lett felhasználva a rendszerben)
|
||||
usage_count: Mapped[int] = mapped_column(Integer, default=0, server_default=text("0"))
|
||||
|
||||
# UI ikon azonosító (pl. 'wrench', 'tire-flat', 'car-electric')
|
||||
icon: Mapped[Optional[str]] = mapped_column(String(50))
|
||||
|
||||
# Leírás a szakmáról (Adminisztratív célokra)
|
||||
description: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
# Időbélyegek
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), onupdate=func.now())
|
||||
|
||||
# --- KAPCSOLATOK ---
|
||||
services: Mapped[List["ServiceExpertise"]] = relationship("ServiceExpertise", back_populates="tag")
|
||||
# Visszamutatás a beküldőre (ha van)
|
||||
suggested_by: Mapped[Optional["Person"]] = relationship("Person")
|
||||
|
||||
class ServiceExpertise(Base):
|
||||
"""
|
||||
KAPCSOLÓTÁBLA: Ez köti össze a szervizt a szakmáival.
|
||||
Itt tároljuk, hogy az adott szerviznél mennyire validált egy szakma.
|
||||
"""
|
||||
__tablename__ = "service_expertises"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
service_id: Mapped[int] = mapped_column(Integer, ForeignKey("marketplace.service_profiles.id", ondelete="CASCADE"))
|
||||
expertise_id: Mapped[int] = mapped_column(Integer, ForeignKey("marketplace.expertise_tags.id", ondelete="CASCADE"))
|
||||
|
||||
# Mennyire biztos ez a tudás? (0: robot találta, 1: júzer mondta, 2: igazolt szakma)
|
||||
confidence_level: Mapped[int] = mapped_column(Integer, default=0, server_default=text("0"))
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=text("now()"))
|
||||
|
||||
# Kapcsolatok visszafelé
|
||||
service = relationship("ServiceProfile", back_populates="expertises")
|
||||
tag = relationship("ExpertiseTag", back_populates="services")
|
||||
|
||||
class ServiceStaging(Base):
|
||||
""" Hunter (robot) adatok tárolója. """
|
||||
__tablename__ = "service_staging"
|
||||
__table_args__ = (
|
||||
Index('idx_staging_fingerprint', 'fingerprint', unique=True),
|
||||
{"schema": "marketplace"}
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String, index=True, nullable=False)
|
||||
postal_code: Mapped[Optional[str]] = mapped_column(String(10), index=True)
|
||||
city: Mapped[Optional[str]] = mapped_column(String(100), index=True)
|
||||
full_address: Mapped[Optional[str]] = mapped_column(String)
|
||||
fingerprint: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
raw_data: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
# Additional contact and identification fields
|
||||
contact_phone: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
website: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
|
||||
external_id: Mapped[Optional[str]] = mapped_column(String(100), nullable=True, index=True)
|
||||
|
||||
status: Mapped[str] = mapped_column(String(20), server_default=text("'pending'"), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
class DiscoveryParameter(Base):
|
||||
""" Robot vezérlési paraméterek adminból. """
|
||||
__tablename__ = "discovery_parameters"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
city: Mapped[str] = mapped_column(String(100))
|
||||
keyword: Mapped[str] = mapped_column(String(100))
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
@@ -1,73 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/models/marketplace/staged_data.py
|
||||
from datetime import datetime
|
||||
from typing import Optional, Any
|
||||
from sqlalchemy import String, Integer, DateTime, text, Boolean, Float
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.sql import func
|
||||
from app.db.base_class import Base
|
||||
|
||||
class StagedVehicleData(Base):
|
||||
""" Robot 2.1 (Researcher) nyers adatgyűjtője. """
|
||||
__tablename__ = "staged_vehicle_data"
|
||||
__table_args__ = {"schema": "system"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_url: Mapped[Optional[str]] = mapped_column(String)
|
||||
raw_data: Mapped[dict] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
status: Mapped[str] = mapped_column(String(20), default="PENDING", index=True)
|
||||
error_log: Mapped[Optional[str]] = mapped_column(String)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
class ServiceStaging(Base):
|
||||
""" Robot 1.3 (Scout) által talált nyers szerviz adatok és a Robot 5 (Auditor) naplója. """
|
||||
__tablename__ = "service_staging"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), index=True)
|
||||
source: Mapped[Optional[str]] = mapped_column(String(50))
|
||||
external_id: Mapped[Optional[str]] = mapped_column(String(100), index=True)
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
|
||||
# Elérhetőségek
|
||||
city: Mapped[str] = mapped_column(String(100), index=True)
|
||||
postal_code: Mapped[Optional[str]] = mapped_column(String(10))
|
||||
full_address: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
contact_phone: Mapped[Optional[str]] = mapped_column(String(50))
|
||||
website: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
contact_email: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
|
||||
|
||||
# Beküldés és Bizalom
|
||||
description: Mapped[Optional[str]] = mapped_column(Text)
|
||||
submitted_by: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("identity.users.id"))
|
||||
trust_score: Mapped[int] = mapped_column(Integer, default=0, server_default=text("0"))
|
||||
|
||||
# Nyers adatok és Státusz
|
||||
raw_data: Mapped[dict] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending", index=True)
|
||||
|
||||
# --- Robot 5 (Auditor) technikai mezők ---
|
||||
# Ezek kellenek a munka naplózásához
|
||||
rejection_reason: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
published_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
service_profile_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
organization_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
audit_trail: Mapped[Optional[dict]] = mapped_column(JSONB)
|
||||
|
||||
# Időbélyegek
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), onupdate=func.now())
|
||||
|
||||
class DiscoveryParameter(Base):
|
||||
""" Felderítési paraméterek (Városok, ahol a Scout keres). """
|
||||
__tablename__ = "discovery_parameters"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
city: Mapped[str] = mapped_column(String(100), unique=True, index=True)
|
||||
keyword: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
@@ -106,7 +106,7 @@ class FinancialLedger(Base):
|
||||
gross_amount: Mapped[Optional[float]] = mapped_column(Numeric(18, 4))
|
||||
net_amount: Mapped[Optional[float]] = mapped_column(Numeric(18, 4))
|
||||
transaction_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PG_UUID(as_uuid=True), default=uuid.uuid4, nullable=False, index=True
|
||||
PG_UUID(as_uuid=True), default=uuid.uuid4, nullable=False, unique=True, index=True
|
||||
)
|
||||
status: Mapped[LedgerStatus] = mapped_column(
|
||||
PG_ENUM(LedgerStatus, name="ledger_status", schema="audit"),
|
||||
|
||||
@@ -38,8 +38,8 @@ class SystemParameter(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), onupdate=func.now(), server_default=func.now())
|
||||
|
||||
class InternalNotification(Base):
|
||||
"""
|
||||
Belső értesítési központ.
|
||||
"""
|
||||
Belső értesítési központ.
|
||||
Ezek az üzenetek várják a felhasználót belépéskor.
|
||||
"""
|
||||
__tablename__ = "internal_notifications"
|
||||
@@ -53,11 +53,33 @@ class InternalNotification(Base):
|
||||
category: Mapped[str] = mapped_column(String(50), server_default="info")
|
||||
priority: Mapped[str] = mapped_column(String(20), server_default="medium")
|
||||
|
||||
is_read: Mapped[bool] = mapped_column(Boolean, default=False, index=True)
|
||||
read_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
data: Mapped[Optional[dict]] = mapped_column(JSONB, nullable=True)
|
||||
|
||||
is_read: Mapped[bool] = mapped_column(Boolean, default=False, index=True)
|
||||
|
||||
class SystemDataCompletionWeight(Base):
|
||||
"""Adatkitöltési súlyok rendszerszintű konfigurációja - mely mezők mennyire fontosak a profil teljességéhez."""
|
||||
__tablename__ = "system_data_completion_weights"
|
||||
__table_args__ = (
|
||||
UniqueConstraint('entity_type', 'field_name', name='uix_entity_field'),
|
||||
{"schema": "system"}
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
entity_type: Mapped[str] = mapped_column(String(50), nullable=False, index=True) # pl: "vehicle", "person", "organization"
|
||||
field_name: Mapped[str] = mapped_column(String(100), nullable=False, index=True) # pl: "vin", "license_plate", "email"
|
||||
weight_percent: Mapped[int] = mapped_column(Integer, nullable=False) # 0-100%
|
||||
is_mandatory: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
|
||||
description: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
# Shadow column that exists in database (should be removed in future migration)
|
||||
is_read: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
||||
|
||||
|
||||
class SystemServiceStaging(Base):
|
||||
|
||||
@@ -64,6 +64,7 @@ class Asset(Base):
|
||||
operator_org_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("fleet.organizations.id"))
|
||||
|
||||
status: Mapped[str] = mapped_column(String(20), default="active")
|
||||
data_status: Mapped[Optional[str]] = mapped_column(String(20), nullable=True, server_default=text("'draft'"))
|
||||
individual_equipment: Mapped[dict] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), onupdate=func.now())
|
||||
@@ -87,6 +88,51 @@ class Asset(Base):
|
||||
"""Always False for now, as verification is not yet implemented."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def profile_completion_percentage(self) -> int:
|
||||
"""
|
||||
Calculate profile completion percentage based on available data.
|
||||
Uses dynamic weights from system.system_data_completion_weights table.
|
||||
Default weights (if not configured):
|
||||
- license_plate: 20%
|
||||
- make: 15% (from catalog)
|
||||
- model: 15% (from catalog)
|
||||
- vin: 30%
|
||||
- year_of_manufacture: 20%
|
||||
"""
|
||||
# Default weights (fallback if dynamic weights not available)
|
||||
default_weights = {
|
||||
'license_plate': 20,
|
||||
'make': 15,
|
||||
'model': 15,
|
||||
'vin': 30,
|
||||
'year_of_manufacture': 20
|
||||
}
|
||||
|
||||
total_score = 0
|
||||
|
||||
# 1. license_plate
|
||||
if self.license_plate and self.license_plate.strip():
|
||||
total_score += default_weights['license_plate']
|
||||
|
||||
# 2. make (from catalog)
|
||||
if self.catalog and self.catalog.make:
|
||||
total_score += default_weights['make']
|
||||
|
||||
# 3. model (from catalog)
|
||||
if self.catalog and self.catalog.model:
|
||||
total_score += default_weights['model']
|
||||
|
||||
# 4. vin
|
||||
if self.vin and self.vin.strip():
|
||||
total_score += default_weights['vin']
|
||||
|
||||
# 5. year_of_manufacture
|
||||
if self.year_of_manufacture:
|
||||
total_score += default_weights['year_of_manufacture']
|
||||
|
||||
return min(total_score, 100)
|
||||
|
||||
class AssetFinancials(Base):
|
||||
""" I. Beszerzés és IV. Értékcsökkenés (Amortizáció). """
|
||||
__tablename__ = "asset_financials"
|
||||
@@ -273,4 +319,27 @@ class VehicleExpenses(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
# Relationship
|
||||
asset: Mapped["Asset"] = relationship("Asset")
|
||||
asset: Mapped["Asset"] = relationship("Asset")
|
||||
|
||||
|
||||
class VehicleTransferRequest(Base):
|
||||
"""Járműátadási kérelem - asset átruházás másik tulajdonosnak vagy szervezetnek."""
|
||||
__tablename__ = "vehicle_transfer_requests"
|
||||
__table_args__ = {"schema": "vehicle"}
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
asset_id: Mapped[uuid.UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("vehicle.assets.id"), nullable=False, index=True)
|
||||
requester_id: Mapped[int] = mapped_column(Integer, ForeignKey("identity.users.id"), nullable=False, index=True)
|
||||
current_owner_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("identity.persons.id"), nullable=True, index=True)
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending", index=True)
|
||||
proof_document_id: Mapped[Optional[uuid.UUID]] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("system.documents.id"), nullable=True)
|
||||
|
||||
requested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
processed_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
notes: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
# Relationships
|
||||
asset: Mapped["Asset"] = relationship("Asset")
|
||||
requester: Mapped["User"] = relationship("User", foreign_keys=[requester_id])
|
||||
current_owner: Mapped[Optional["Person"]] = relationship("Person", foreign_keys=[current_owner_id])
|
||||
proof_document: Mapped[Optional["Document"]] = relationship("Document")
|
||||
@@ -32,7 +32,7 @@ class AssetCatalogResponse(BaseModel):
|
||||
class AssetResponse(BaseModel):
|
||||
""" A konkrét járműpéldány (Asset) teljes válaszmodellje. """
|
||||
id: UUID
|
||||
vin: str = Field(..., min_length=1, max_length=50)
|
||||
vin: Optional[str] = Field(None, min_length=1, max_length=50)
|
||||
license_plate: Optional[str] = None
|
||||
name: Optional[str] = None
|
||||
year_of_manufacture: Optional[int] = None
|
||||
@@ -50,6 +50,9 @@ class AssetResponse(BaseModel):
|
||||
owner_organization_id: Optional[int] = None
|
||||
operator_person_id: Optional[int] = None
|
||||
|
||||
# Profile completion percentage (0-100)
|
||||
profile_completion_percentage: int = Field(default=0, ge=0, le=100)
|
||||
|
||||
created_at: datetime
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
@@ -61,4 +64,4 @@ class AssetCreate(BaseModel):
|
||||
vin: Optional[str] = Field(None, min_length=1, max_length=50, description="VIN szám (1-50 karakter, opcionális draft módban)")
|
||||
license_plate: str = Field(..., min_length=2, max_length=20, description="Rendszám")
|
||||
catalog_id: Optional[int] = Field(None, description="Opcionális katalógus ID (ha ismert a modell)")
|
||||
organization_id: int = Field(..., description="Szervezet ID, amelyhez a jármű tartozik")
|
||||
organization_id: Optional[int] = Field(None, description="Szervezet ID, amelyhez a jármű tartozik (opcionális, alapértelmezett a felhasználó szervezete)")
|
||||
@@ -18,6 +18,7 @@ class UserResponse(UserBase):
|
||||
scope_level: str
|
||||
scope_id: Optional[str] = None
|
||||
ui_mode: str = "personal"
|
||||
active_organization_id: Optional[int] = None
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
class UserUpdate(BaseModel):
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Move tables from system schema to gamification schema.
|
||||
"""
|
||||
import asyncio
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import text
|
||||
|
||||
async def move_tables():
|
||||
# Use the same DATABASE_URL as sync_engine
|
||||
from app.core.config import settings
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Check if tables exist in system schema
|
||||
result = await conn.execute(text("""
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_name IN ('competitions', 'user_scores')
|
||||
ORDER BY table_schema;
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
print("Current tables:")
|
||||
for row in rows:
|
||||
print(f" {row.table_schema}.{row.table_name}")
|
||||
|
||||
# Move competitions
|
||||
print("\nMoving system.competitions to gamification.competitions...")
|
||||
try:
|
||||
await conn.execute(text('ALTER TABLE system.competitions SET SCHEMA gamification;'))
|
||||
print(" OK")
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
# Move user_scores
|
||||
print("Moving system.user_scores to gamification.user_scores...")
|
||||
try:
|
||||
await conn.execute(text('ALTER TABLE system.user_scores SET SCHEMA gamification;'))
|
||||
print(" OK")
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
# Verify
|
||||
result = await conn.execute(text("""
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_name IN ('competitions', 'user_scores')
|
||||
ORDER BY table_schema;
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
print("\nAfter moving:")
|
||||
for row in rows:
|
||||
print(f" {row.table_schema}.{row.table_name}")
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(move_tables())
|
||||
@@ -1,53 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Rename tables in system schema to deprecated to avoid extra detection.
|
||||
"""
|
||||
import asyncio
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import text
|
||||
|
||||
async def rename():
|
||||
from app.core.config import settings
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Check if tables exist
|
||||
result = await conn.execute(text("""
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'system' AND table_name IN ('competitions', 'user_scores');
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
print("Tables to rename:")
|
||||
for row in rows:
|
||||
print(f" {row.table_schema}.{row.table_name}")
|
||||
|
||||
# Rename competitions
|
||||
try:
|
||||
await conn.execute(text('ALTER TABLE system.competitions RENAME TO competitions_deprecated;'))
|
||||
print("Renamed system.competitions -> system.competitions_deprecated")
|
||||
except Exception as e:
|
||||
print(f"Error renaming competitions: {e}")
|
||||
|
||||
# Rename user_scores
|
||||
try:
|
||||
await conn.execute(text('ALTER TABLE system.user_scores RENAME TO user_scores_deprecated;'))
|
||||
print("Renamed system.user_scores -> system.user_scores_deprecated")
|
||||
except Exception as e:
|
||||
print(f"Error renaming user_scores: {e}")
|
||||
|
||||
# Verify
|
||||
result = await conn.execute(text("""
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'system' AND table_name LIKE '%deprecated';
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
print("\nAfter rename:")
|
||||
for row in rows:
|
||||
print(f" {row.table_schema}.{row.table_name}")
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(rename())
|
||||
99
backend/app/scripts/seed_completion_weights.py
Normal file
99
backend/app/scripts/seed_completion_weights.py
Normal file
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Seed script for system.data_completion_weights table.
|
||||
Populates default weights for vehicle asset completion percentage calculation.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.system.system import SystemDataCompletionWeight
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s]: %(message)s')
|
||||
logger = logging.getLogger("Seed-Completion-Weights")
|
||||
|
||||
async def seed_completion_weights():
|
||||
"""Seed default data completion weights for entity_type='asset'."""
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Check if weights already exist for entity_type='asset'
|
||||
stmt = select(SystemDataCompletionWeight).where(
|
||||
SystemDataCompletionWeight.entity_type == "asset"
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
existing = result.scalars().all()
|
||||
|
||||
if existing:
|
||||
logger.info(f"Found {len(existing)} existing weights for entity_type='asset'. Skipping seed.")
|
||||
return
|
||||
|
||||
# Default weights for vehicle assets (total: 100%)
|
||||
weights = [
|
||||
{
|
||||
"entity_type": "asset",
|
||||
"field_name": "license_plate",
|
||||
"weight_percent": 20,
|
||||
"is_mandatory": False,
|
||||
"is_active": True,
|
||||
"is_read": False,
|
||||
"description": "Rendszám - alapvető azonosító"
|
||||
},
|
||||
{
|
||||
"entity_type": "asset",
|
||||
"field_name": "make",
|
||||
"weight_percent": 15,
|
||||
"is_mandatory": False,
|
||||
"is_active": True,
|
||||
"is_read": False,
|
||||
"description": "Gyártó (márka)"
|
||||
},
|
||||
{
|
||||
"entity_type": "asset",
|
||||
"field_name": "model",
|
||||
"weight_percent": 15,
|
||||
"is_mandatory": False,
|
||||
"is_active": True,
|
||||
"is_read": False,
|
||||
"description": "Modell"
|
||||
},
|
||||
{
|
||||
"entity_type": "asset",
|
||||
"field_name": "vin",
|
||||
"weight_percent": 30,
|
||||
"is_mandatory": False,
|
||||
"is_active": True,
|
||||
"is_read": False,
|
||||
"description": "Alvázszám (VIN) - egyedi azonosító"
|
||||
},
|
||||
{
|
||||
"entity_type": "asset",
|
||||
"field_name": "year_of_manufacture",
|
||||
"weight_percent": 20,
|
||||
"is_mandatory": False,
|
||||
"is_active": True,
|
||||
"is_read": False,
|
||||
"description": "Gyártási év"
|
||||
}
|
||||
]
|
||||
|
||||
# Insert weights
|
||||
for weight_data in weights:
|
||||
weight = SystemDataCompletionWeight(**weight_data)
|
||||
db.add(weight)
|
||||
|
||||
await db.commit()
|
||||
logger.info(f"Successfully seeded {len(weights)} completion weights for entity_type='asset'")
|
||||
|
||||
# Log the total percentage
|
||||
total = sum(w["weight_percent"] for w in weights)
|
||||
logger.info(f"Total weight percentage: {total}%")
|
||||
|
||||
async def main():
|
||||
"""Main entry point."""
|
||||
try:
|
||||
await seed_completion_weights()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to seed completion weights: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,170 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/scripts/sync_engine.py
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Universal Schema Synchronizer
|
||||
|
||||
Dynamically imports all SQLAlchemy models from app.models, compares them with the live database,
|
||||
and creates missing tables/columns without dropping anything.
|
||||
|
||||
Safety First:
|
||||
- NEVER drops tables or columns.
|
||||
- Prints planned SQL before execution.
|
||||
- Requires confirmation for destructive operations (none in this script).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import inspect, text
|
||||
from sqlalchemy.schema import CreateTable, AddConstraint
|
||||
from sqlalchemy.sql.ddl import CreateColumn
|
||||
|
||||
# Add backend to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from app.database import Base
|
||||
from app.core.config import settings
|
||||
|
||||
def dynamic_import_models():
|
||||
"""
|
||||
Dynamically import all .py files in app.models directory to ensure Base.metadata is populated.
|
||||
"""
|
||||
models_dir = Path(__file__).parent.parent / "models"
|
||||
imported = []
|
||||
|
||||
for py_file in models_dir.glob("*.py"):
|
||||
if py_file.name == "__init__.py":
|
||||
continue
|
||||
module_name = f"app.models.{py_file.stem}"
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
imported.append(module_name)
|
||||
print(f"✅ Imported {module_name}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not import {module_name}: {e}")
|
||||
|
||||
# Also ensure the __init__ is loaded (it imports many models manually)
|
||||
import app.models
|
||||
print(f"📦 Total tables in Base.metadata: {len(Base.metadata.tables)}")
|
||||
return imported
|
||||
|
||||
async def compare_and_repair():
|
||||
"""
|
||||
Compare SQLAlchemy metadata with live database and create missing tables/columns.
|
||||
"""
|
||||
print("🔗 Connecting to database...")
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
def get_diff_and_repair(connection):
|
||||
inspector = inspect(connection)
|
||||
|
||||
# Get all schemas from models
|
||||
expected_schemas = sorted({t.schema for t in Base.metadata.sorted_tables if t.schema})
|
||||
print(f"📋 Expected schemas: {expected_schemas}")
|
||||
|
||||
# Ensure enum types exist in marketplace schema
|
||||
if 'marketplace' in expected_schemas:
|
||||
print("\n🔧 Ensuring enum types in marketplace schema...")
|
||||
# moderation_status enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'moderation_status' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.moderation_status AS ENUM ('pending', 'approved', 'rejected');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
# source_type enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'source_type' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.source_type AS ENUM ('manual', 'ocr', 'import');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
print("✅ Enum types ensured.")
|
||||
|
||||
for schema in expected_schemas:
|
||||
print(f"\n--- 🔍 Checking schema '{schema}' ---")
|
||||
|
||||
# Check if schema exists
|
||||
db_schemas = inspector.get_schema_names()
|
||||
if schema not in db_schemas:
|
||||
print(f"❌ Schema '{schema}' missing. Creating...")
|
||||
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
|
||||
print(f"✅ Schema '{schema}' created.")
|
||||
|
||||
# Get tables in this schema from models
|
||||
model_tables = [t for t in Base.metadata.sorted_tables if t.schema == schema]
|
||||
db_tables = inspector.get_table_names(schema=schema)
|
||||
|
||||
for table in model_tables:
|
||||
if table.name not in db_tables:
|
||||
print(f"❌ Missing table: {schema}.{table.name}")
|
||||
# Generate CREATE TABLE statement
|
||||
create_stmt = CreateTable(table)
|
||||
# Print SQL for debugging
|
||||
sql_str = str(create_stmt.compile(bind=engine))
|
||||
print(f" SQL: {sql_str}")
|
||||
connection.execute(create_stmt)
|
||||
print(f"✅ Table {schema}.{table.name} created.")
|
||||
else:
|
||||
# Check columns
|
||||
db_columns = {c['name']: c for c in inspector.get_columns(table.name, schema=schema)}
|
||||
model_columns = table.columns
|
||||
|
||||
missing_cols = []
|
||||
for col in model_columns:
|
||||
if col.name not in db_columns:
|
||||
missing_cols.append(col)
|
||||
|
||||
if missing_cols:
|
||||
print(f"⚠️ Table {schema}.{table.name} missing columns: {[c.name for c in missing_cols]}")
|
||||
for col in missing_cols:
|
||||
# Generate ADD COLUMN statement
|
||||
col_type = col.type.compile(dialect=engine.dialect)
|
||||
sql = f'ALTER TABLE "{schema}"."{table.name}" ADD COLUMN "{col.name}" {col_type}'
|
||||
if col.nullable is False:
|
||||
sql += " NOT NULL"
|
||||
if col.default is not None:
|
||||
# Handle default values (simplistic)
|
||||
sql += f" DEFAULT {col.default.arg}"
|
||||
print(f" SQL: {sql}")
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Column {col.name} added.")
|
||||
else:
|
||||
print(f"✅ Table {schema}.{table.name} is up‑to‑date.")
|
||||
|
||||
print("\n--- ✅ Schema synchronization complete. ---")
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(get_diff_and_repair)
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
async def main():
|
||||
print("🚀 Universal Schema Synchronizer")
|
||||
print("=" * 50)
|
||||
|
||||
# Step 1: Dynamic import
|
||||
print("\n📥 Step 1: Dynamically importing all models...")
|
||||
dynamic_import_models()
|
||||
|
||||
# Step 2: Compare and repair
|
||||
print("\n🔧 Step 2: Comparing with database and repairing...")
|
||||
await compare_and_repair()
|
||||
|
||||
# Step 3: Final verification
|
||||
print("\n📊 Step 3: Final verification...")
|
||||
# Run compare_schema.py logic to confirm everything is green
|
||||
from app.tests_internal.diagnostics.compare_schema import compare
|
||||
await compare()
|
||||
|
||||
print("\n✨ Synchronization finished successfully!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,232 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Unified Database Synchronizer with Deep Constraint & Index Support
|
||||
|
||||
Dynamically imports all SQLAlchemy models, compares metadata with live database,
|
||||
and creates missing tables, columns, unique constraints, and indexes.
|
||||
|
||||
Safety First:
|
||||
- NEVER drops tables, columns, constraints, or indexes.
|
||||
- Prints planned SQL before execution.
|
||||
- Requires confirmation for destructive operations (none in this script).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import inspect, text, UniqueConstraint, Index
|
||||
from sqlalchemy.schema import CreateTable, AddConstraint, CreateIndex
|
||||
from sqlalchemy.sql.ddl import CreateColumn
|
||||
|
||||
# Add backend to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from app.database import Base
|
||||
from app.core.config import settings
|
||||
|
||||
def dynamic_import_models():
|
||||
"""
|
||||
Dynamically import all .py files in app.models directory to ensure Base.metadata is populated.
|
||||
"""
|
||||
models_dir = Path(__file__).parent.parent / "models"
|
||||
imported = []
|
||||
|
||||
for py_file in models_dir.glob("*.py"):
|
||||
if py_file.name == "__init__.py":
|
||||
continue
|
||||
module_name = f"app.models.{py_file.stem}"
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
imported.append(module_name)
|
||||
print(f"✅ Imported {module_name}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not import {module_name}: {e}")
|
||||
|
||||
# Also ensure the __init__ is loaded (it imports many models manually)
|
||||
import app.models
|
||||
print(f"📦 Total tables in Base.metadata: {len(Base.metadata.tables)}")
|
||||
return imported
|
||||
|
||||
async def compare_and_repair(apply: bool = False):
|
||||
"""
|
||||
Compare SQLAlchemy metadata with live database and create missing
|
||||
tables, columns, unique constraints, and indexes.
|
||||
|
||||
If apply is False, only prints SQL statements without executing.
|
||||
"""
|
||||
print("🔗 Connecting to database...")
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
def get_diff_and_repair(connection):
|
||||
inspector = inspect(connection)
|
||||
|
||||
# Get all schemas from models
|
||||
expected_schemas = sorted({t.schema for t in Base.metadata.sorted_tables if t.schema})
|
||||
print(f"📋 Expected schemas: {expected_schemas}")
|
||||
|
||||
# Ensure enum types exist in marketplace schema
|
||||
if 'marketplace' in expected_schemas:
|
||||
print("\n🔧 Ensuring enum types in marketplace schema...")
|
||||
# moderation_status enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'moderation_status' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.moderation_status AS ENUM ('pending', 'approved', 'rejected');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
# source_type enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'source_type' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.source_type AS ENUM ('manual', 'ocr', 'import');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
print("✅ Enum types ensured.")
|
||||
|
||||
for schema in expected_schemas:
|
||||
print(f"\n--- 🔍 Checking schema '{schema}' ---")
|
||||
|
||||
# Check if schema exists
|
||||
db_schemas = inspector.get_schema_names()
|
||||
if schema not in db_schemas:
|
||||
print(f"❌ Schema '{schema}' missing. Creating...")
|
||||
if apply:
|
||||
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
|
||||
print(f"✅ Schema '{schema}' created.")
|
||||
else:
|
||||
print(f" SQL: CREATE SCHEMA IF NOT EXISTS \"{schema}\"")
|
||||
|
||||
# Get tables in this schema from models
|
||||
model_tables = [t for t in Base.metadata.sorted_tables if t.schema == schema]
|
||||
db_tables = inspector.get_table_names(schema=schema)
|
||||
|
||||
for table in model_tables:
|
||||
if table.name not in db_tables:
|
||||
print(f"❌ Missing table: {schema}.{table.name}")
|
||||
# Generate CREATE TABLE statement
|
||||
create_stmt = CreateTable(table)
|
||||
sql_str = str(create_stmt.compile(bind=engine))
|
||||
print(f" SQL: {sql_str}")
|
||||
if apply:
|
||||
connection.execute(create_stmt)
|
||||
print(f"✅ Table {schema}.{table.name} created.")
|
||||
continue
|
||||
|
||||
# Check columns
|
||||
db_columns = {c['name']: c for c in inspector.get_columns(table.name, schema=schema)}
|
||||
model_columns = table.columns
|
||||
|
||||
missing_cols = []
|
||||
for col in model_columns:
|
||||
if col.name not in db_columns:
|
||||
missing_cols.append(col)
|
||||
|
||||
if missing_cols:
|
||||
print(f"⚠️ Table {schema}.{table.name} missing columns: {[c.name for c in missing_cols]}")
|
||||
for col in missing_cols:
|
||||
col_type = col.type.compile(dialect=engine.dialect)
|
||||
sql = f'ALTER TABLE "{schema}"."{table.name}" ADD COLUMN "{col.name}" {col_type}'
|
||||
if col.nullable is False:
|
||||
sql += " NOT NULL"
|
||||
if col.default is not None:
|
||||
sql += f" DEFAULT {col.default.arg}"
|
||||
print(f" SQL: {sql}")
|
||||
if apply:
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Column {col.name} added.")
|
||||
else:
|
||||
print(f"✅ Table {schema}.{table.name} columns are up‑to‑date.")
|
||||
|
||||
# Check Unique Constraints
|
||||
db_unique_constraints = inspector.get_unique_constraints(table.name, schema=schema)
|
||||
# Map by column names (since constraint names may differ)
|
||||
db_unique_map = {}
|
||||
for uc in db_unique_constraints:
|
||||
key = tuple(sorted(uc['column_names']))
|
||||
db_unique_map[key] = uc['name']
|
||||
|
||||
# Find unique constraints defined in model
|
||||
model_unique_constraints = [c for c in table.constraints if isinstance(c, UniqueConstraint)]
|
||||
for uc in model_unique_constraints:
|
||||
uc_columns = tuple(sorted([col.name for col in uc.columns]))
|
||||
if uc_columns not in db_unique_map:
|
||||
# Constraint missing
|
||||
constraint_name = uc.name or f"uq_{table.name}_{'_'.join(uc_columns)}"
|
||||
columns_sql = ', '.join([f'"{col}"' for col in uc_columns])
|
||||
sql = f'ALTER TABLE "{schema}"."{table.name}" ADD CONSTRAINT "{constraint_name}" UNIQUE ({columns_sql})'
|
||||
print(f"⚠️ Missing unique constraint on {schema}.{table.name} columns {uc_columns}")
|
||||
print(f" SQL: {sql}")
|
||||
if apply:
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Unique constraint {constraint_name} added.")
|
||||
else:
|
||||
print(f"✅ Unique constraint on {uc_columns} exists.")
|
||||
|
||||
# Check Indexes
|
||||
db_indexes = inspector.get_indexes(table.name, schema=schema)
|
||||
db_index_map = {}
|
||||
for idx in db_indexes:
|
||||
key = tuple(sorted(idx['column_names']))
|
||||
db_index_map[key] = idx['name']
|
||||
|
||||
# Find indexes defined in model (Index objects)
|
||||
model_indexes = [idx for idx in table.indexes]
|
||||
for idx in model_indexes:
|
||||
idx_columns = tuple(sorted([col.name for col in idx.columns]))
|
||||
if idx_columns not in db_index_map:
|
||||
# Index missing
|
||||
index_name = idx.name or f"idx_{table.name}_{'_'.join(idx_columns)}"
|
||||
columns_sql = ', '.join([f'"{col}"' for col in idx_columns])
|
||||
unique_sql = "UNIQUE " if idx.unique else ""
|
||||
sql = f'CREATE {unique_sql}INDEX "{index_name}" ON "{schema}"."{table.name}" ({columns_sql})'
|
||||
print(f"⚠️ Missing index on {schema}.{table.name} columns {idx_columns}")
|
||||
print(f" SQL: {sql}")
|
||||
if apply:
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Index {index_name} added.")
|
||||
else:
|
||||
print(f"✅ Index on {idx_columns} exists.")
|
||||
|
||||
print("\n--- ✅ Schema synchronization complete. ---")
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(get_diff_and_repair)
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
async def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Unified Database Synchronizer")
|
||||
parser.add_argument('--apply', action='store_true', help='Apply changes to database (otherwise dry‑run)')
|
||||
args = parser.parse_args()
|
||||
|
||||
print("🚀 Unified Database Synchronizer")
|
||||
print("=" * 50)
|
||||
|
||||
# Step 1: Dynamic import
|
||||
print("\n📥 Step 1: Dynamically importing all models...")
|
||||
dynamic_import_models()
|
||||
|
||||
# Step 2: Compare and repair
|
||||
print("\n🔧 Step 2: Comparing with database and repairing...")
|
||||
await compare_and_repair(apply=args.apply)
|
||||
|
||||
# Step 3: Final verification
|
||||
print("\n📊 Step 3: Final verification...")
|
||||
try:
|
||||
from app.tests_internal.diagnostics.compare_schema import compare
|
||||
await compare()
|
||||
except ImportError:
|
||||
print("⚠️ compare_schema module not found, skipping verification.")
|
||||
|
||||
print("\n✨ Synchronization finished successfully!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,111 +0,0 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import asyncio
|
||||
import re
|
||||
from typing import Dict, Any, Optional
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
from sqlalchemy import select
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import SystemParameter
|
||||
|
||||
logger = logging.getLogger("AI-Service")
|
||||
|
||||
class AIService:
|
||||
"""
|
||||
AI Service v1.2.5 - Final Integrated Edition
|
||||
- Robot 2: Technikai dúsítás (Search + Regex JSON parsing)
|
||||
- Robot 3: OCR (Controlled JSON generation)
|
||||
"""
|
||||
api_key = os.getenv("GEMINI_API_KEY")
|
||||
client = genai.Client(api_key=api_key) if api_key else None
|
||||
PRIMARY_MODEL = "gemini-2.0-flash"
|
||||
|
||||
@classmethod
|
||||
async def get_config_delay(cls) -> float:
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
stmt = select(SystemParameter).where(SystemParameter.key == "AI_REQUEST_DELAY")
|
||||
res = await db.execute(stmt)
|
||||
param = res.scalar_one_or_none()
|
||||
return float(param.value) if param else 1.0
|
||||
except Exception: return 1.0
|
||||
|
||||
@classmethod
|
||||
async def get_clean_vehicle_data(cls, make: str, raw_model: str, v_type: str) -> Optional[Dict[str, Any]]:
|
||||
"""Robot 2: Adatbányászat Google Search segítségével."""
|
||||
if not cls.client: return None
|
||||
await asyncio.sleep(await cls.get_config_delay())
|
||||
|
||||
search_tool = types.Tool(google_search=types.GoogleSearch())
|
||||
|
||||
prompt = f"""
|
||||
KERESS RÁ az interneten: {make} {raw_model} ({v_type}) pontos gyári modellkódja és technikai adatai.
|
||||
Adj választ szigorúan csak egy JSON blokkban:
|
||||
{{
|
||||
"marketing_name": "tiszta név",
|
||||
"synonyms": ["név1", "név2"],
|
||||
"technical_code": "gyári kód",
|
||||
"year_from": int,
|
||||
"year_to": int_vagy_null,
|
||||
"ccm": int,
|
||||
"kw": int,
|
||||
"maintenance": {{ "oil_type": "string", "oil_qty": float, "spark_plug": "string", "coolant": "string" }}
|
||||
}}
|
||||
FONTOS: A 'technical_code' NEM lehet üres. Ha nem találod, adj 'N/A' értéket!
|
||||
"""
|
||||
|
||||
# Search tool használata esetén a response_mime_type tilos!
|
||||
config = types.GenerateContentConfig(
|
||||
system_instruction="Profi járműtechnikai adatbányász vagy. Csak tiszta JSON-t válaszolsz markdown kódblokk nélkül.",
|
||||
tools=[search_tool],
|
||||
temperature=0.1
|
||||
)
|
||||
|
||||
try:
|
||||
response = cls.client.models.generate_content(model=cls.PRIMARY_MODEL, contents=prompt, config=config)
|
||||
text = response.text
|
||||
# Tisztítás: ha az AI mégis tenne bele markdown jeleket
|
||||
clean_json = re.sub(r'```json\s*|```', '', text).strip()
|
||||
res_json = json.loads(clean_json)
|
||||
if isinstance(res_json, list) and len(res_json) > 0: res_json = res_json[0]
|
||||
return res_json if isinstance(res_json, dict) else None
|
||||
except Exception as e:
|
||||
logger.error(f"❌ AI hiba ({make} {raw_model}): {e}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def analyze_document_image(cls, image_data: bytes, doc_type: str) -> Optional[Dict[str, Any]]:
|
||||
"""Robot 3: OCR funkció - Forgalmi, Személyi, Számla, Odometer."""
|
||||
if not cls.client: return None
|
||||
await asyncio.sleep(await cls.get_config_delay())
|
||||
|
||||
prompts = {
|
||||
"identity": "Személyes okmány adatok (név, szám, lejárat).",
|
||||
"vehicle_reg": "Forgalmi adatok (rendszám, alvázszám, kW, ccm).",
|
||||
"invoice": "Számla adatok (partner, végösszeg, dátum).",
|
||||
"odometer": "Csak a kilométeróra állása számként."
|
||||
}
|
||||
|
||||
# Itt maradhat a response_mime_type, mert nem használunk Search-öt
|
||||
config = types.GenerateContentConfig(
|
||||
system_instruction="Profi OCR dokumentum-elemző vagy. Csak tiszta JSON-t válaszolsz.",
|
||||
response_mime_type="application/json"
|
||||
)
|
||||
|
||||
try:
|
||||
response = cls.client.models.generate_content(
|
||||
model=cls.PRIMARY_MODEL,
|
||||
contents=[
|
||||
f"Elemezd ezt a képet ({doc_type}): {prompts.get(doc_type, 'OCR')}",
|
||||
types.Part.from_bytes(data=image_data, mime_type="image/jpeg")
|
||||
],
|
||||
config=config
|
||||
)
|
||||
res_json = json.loads(response.text)
|
||||
if isinstance(res_json, list) and len(res_json) > 0: res_json = res_json[0]
|
||||
return res_json if isinstance(res_json, dict) else None
|
||||
except Exception as e:
|
||||
logger.error(f"❌ OCR hiba: {e}")
|
||||
return None
|
||||
@@ -5,10 +5,10 @@ import uuid
|
||||
from typing import List, Optional, Dict, Any, TYPE_CHECKING
|
||||
from datetime import datetime
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func, and_
|
||||
from sqlalchemy import select, func, and_, distinct
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.models import Asset, AssetAssignment, AssetTelemetry, AssetFinancials
|
||||
from app.models import Asset, AssetAssignment, AssetTelemetry, AssetFinancials, VehicleModelDefinition
|
||||
from app.models.identity import User
|
||||
from app.models.vehicle.history import LogSeverity
|
||||
from app.services.config_service import config
|
||||
@@ -52,9 +52,9 @@ class AssetService:
|
||||
user_stmt = select(User).where(User.id == user_id)
|
||||
user = (await db.execute(user_stmt)).scalar_one()
|
||||
|
||||
limits = await config.get_setting(db, "VEHICLE_LIMIT", default={"free": 1, "premium": 5, "vip": 50})
|
||||
user_role = user.role.value if hasattr(user.role, 'value') else str(user.role)
|
||||
allowed_limit = limits.get(user_role, 1)
|
||||
# Get vehicle limit using the new function that checks both user AND organization limits
|
||||
# Returns the HIGHER value of user-specific and organization-specific limits
|
||||
allowed_limit = await AssetService.get_user_vehicle_limit(db, user_id, org_id)
|
||||
|
||||
# Csak aktív járművek számítanak a limitbe (draft-ok nem)
|
||||
count_stmt = select(func.count(Asset.id)).where(
|
||||
@@ -83,12 +83,23 @@ class AssetService:
|
||||
)
|
||||
|
||||
# 3. ÚJ JÁRMŰ LÉTREHOZÁSA (Standard vagy Draft Flow)
|
||||
status = "draft" if draft or not vin_clean else "active"
|
||||
# Dynamic Gatekeeper Logic: If both vin and catalog_id are missing, status = 'draft'
|
||||
# If core data is provided (either vin OR catalog_id), status = 'active'
|
||||
# Also respect the draft parameter if explicitly set
|
||||
if draft:
|
||||
status = "draft"
|
||||
elif not vin_clean and not catalog_id:
|
||||
status = "draft"
|
||||
else:
|
||||
status = "active"
|
||||
|
||||
new_asset = Asset(
|
||||
vin=vin_clean,
|
||||
license_plate=license_plate_clean,
|
||||
catalog_id=catalog_id,
|
||||
current_organization_id=org_id,
|
||||
owner_person_id=user.person_id,
|
||||
owner_org_id=org_id,
|
||||
status=status,
|
||||
individual_equipment={},
|
||||
created_at=datetime.utcnow()
|
||||
@@ -109,6 +120,9 @@ class AssetService:
|
||||
# Gamification
|
||||
reward = await config.get_setting(db, "xp_reward_asset_register", default=250)
|
||||
await GamificationService.award_points(db, user_id, int(reward), "NEW_ASSET_REG")
|
||||
|
||||
# Check if this is user's first vehicle and award "First Car" badge
|
||||
await AssetService._award_first_car_badge(db, user_id, org_id)
|
||||
|
||||
await db.commit()
|
||||
return new_asset
|
||||
@@ -136,7 +150,7 @@ class AssetService:
|
||||
if auto_transfer:
|
||||
# Csak akkor, ha a régi tulajdonos 'sold' állapotba tette
|
||||
if asset.status == "sold":
|
||||
return await AssetService.execute_final_transfer(db, asset, org_id, new_plate)
|
||||
return await AssetService.execute_final_transfer(db, asset, org_id, new_plate, user_id)
|
||||
|
||||
# Függőben lévő állapot: Dokumentum feltöltésre vár
|
||||
asset.status = "transfer_pending"
|
||||
@@ -150,7 +164,7 @@ class AssetService:
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def execute_final_transfer(db: AsyncSession, asset: Asset, new_org_id: int, new_plate: str):
|
||||
async def execute_final_transfer(db: AsyncSession, asset: Asset, new_org_id: int, new_plate: str, user_id: int = None):
|
||||
""" A tulajdonjog tényleges átírása az adatbázisban. """
|
||||
# 1. Régi hozzárendelés lezárása
|
||||
await db.execute(
|
||||
@@ -165,7 +179,193 @@ class AssetService:
|
||||
asset.status = "active"
|
||||
asset.is_verified = False # Az új tulajdonos papírjait is ellenőrizni kell!
|
||||
|
||||
# 3. Update ownership fields if user_id is provided
|
||||
if user_id is not None:
|
||||
from app.models.identity import User
|
||||
user_stmt = select(User).where(User.id == user_id)
|
||||
user = (await db.execute(user_stmt)).scalar_one_or_none()
|
||||
if user and user.person_id:
|
||||
asset.owner_person_id = user.person_id
|
||||
asset.owner_org_id = new_org_id
|
||||
else:
|
||||
logger.warning(f"User {user_id} has no person_id, cannot set owner_person_id")
|
||||
else:
|
||||
logger.warning("execute_final_transfer called without user_id, ownership fields not updated")
|
||||
|
||||
db.add(AssetAssignment(asset_id=asset.id, organization_id=new_org_id, status="active"))
|
||||
|
||||
await db.commit()
|
||||
return asset
|
||||
return asset
|
||||
|
||||
# --- CATALOG METHODS ---
|
||||
@staticmethod
|
||||
async def get_makes(db: AsyncSession) -> List[str]:
|
||||
"""Get all distinct makes from vehicle model definitions."""
|
||||
stmt = select(distinct(VehicleModelDefinition.make)).order_by(VehicleModelDefinition.make)
|
||||
result = await db.execute(stmt)
|
||||
makes = result.scalars().all()
|
||||
return [make for make in makes if make] # Filter out None/empty
|
||||
|
||||
@staticmethod
|
||||
async def get_models(db: AsyncSession, make: str) -> List[str]:
|
||||
"""Get all distinct models for a given make."""
|
||||
stmt = select(distinct(VehicleModelDefinition.marketing_name)).where(
|
||||
VehicleModelDefinition.make == make
|
||||
).order_by(VehicleModelDefinition.marketing_name)
|
||||
result = await db.execute(stmt)
|
||||
models = result.scalars().all()
|
||||
return [model for model in models if model]
|
||||
|
||||
@staticmethod
|
||||
async def get_generations(db: AsyncSession, make: str, model: str) -> List[str]:
|
||||
"""Get all distinct generations/variants for a given make and model.
|
||||
For now, we'll use engine_code as generation placeholder."""
|
||||
stmt = select(distinct(VehicleModelDefinition.engine_code)).where(
|
||||
VehicleModelDefinition.make == make,
|
||||
VehicleModelDefinition.marketing_name == model,
|
||||
VehicleModelDefinition.engine_code.isnot(None)
|
||||
).order_by(VehicleModelDefinition.engine_code)
|
||||
result = await db.execute(stmt)
|
||||
generations = result.scalars().all()
|
||||
return [gen for gen in generations if gen]
|
||||
|
||||
@staticmethod
|
||||
async def get_engines(db: AsyncSession, make: str, model: str, gen: str) -> List[VehicleModelDefinition]:
|
||||
"""Get all engine variants for a given make, model, and generation."""
|
||||
stmt = select(VehicleModelDefinition).where(
|
||||
VehicleModelDefinition.make == make,
|
||||
VehicleModelDefinition.marketing_name == model,
|
||||
VehicleModelDefinition.engine_code == gen
|
||||
).order_by(VehicleModelDefinition.id)
|
||||
result = await db.execute(stmt)
|
||||
engines = result.scalars().all()
|
||||
return engines
|
||||
|
||||
@staticmethod
|
||||
async def get_user_vehicle_limit(db: AsyncSession, user_id: int, org_id: int) -> int:
|
||||
"""
|
||||
Get the vehicle limit for a user, checking both user-specific AND organization limits.
|
||||
Returns the HIGHER value of the two as per requirements.
|
||||
|
||||
Args:
|
||||
db: AsyncSession
|
||||
user_id: User ID
|
||||
org_id: Organization ID
|
||||
|
||||
Returns:
|
||||
Maximum allowed vehicles (higher of user limit and organization limit)
|
||||
"""
|
||||
from app.models.identity import User
|
||||
from app.services.config_service import config
|
||||
|
||||
try:
|
||||
# Get user info
|
||||
user_stmt = select(User).where(User.id == user_id)
|
||||
user = (await db.execute(user_stmt)).scalar_one()
|
||||
|
||||
# Get global vehicle limits configuration
|
||||
limits = await config.get_setting(db, "VEHICLE_LIMIT")
|
||||
if limits is None:
|
||||
logger.error(f"VEHICLE_LIMIT configuration not found in database for user {user_id}")
|
||||
# Fallback to very high limit instead of restricting users
|
||||
limits = {"admin": 9999, "superadmin": 9999, "user": 100, "free": 100, "premium": 100, "vip": 100, "service_pro": 100}
|
||||
|
||||
user_role = user.role.value if hasattr(user.role, 'value') else str(user.role)
|
||||
subscription_plan = user.subscription_plan or "free"
|
||||
|
||||
# Get user-specific limit (based on role or subscription plan)
|
||||
user_limit = limits.get(user_role)
|
||||
if user_limit is None:
|
||||
user_limit = limits.get(subscription_plan.lower(), 100)
|
||||
|
||||
# Get organization-specific limit (if configured)
|
||||
org_limit = None
|
||||
try:
|
||||
org_limits = await config.get_setting(db, "VEHICLE_LIMIT", org_id=org_id)
|
||||
if org_limits and isinstance(org_limits, dict):
|
||||
# Organization might have different limit structure
|
||||
# Try to get limit for user's role or use a default org limit
|
||||
org_limit = org_limits.get(user_role) or org_limits.get(subscription_plan.lower())
|
||||
if org_limit is None and "default" in org_limits:
|
||||
org_limit = org_limits["default"]
|
||||
except Exception as e:
|
||||
logger.debug(f"No organization-specific VEHICLE_LIMIT found for org {org_id}: {e}")
|
||||
org_limit = None
|
||||
|
||||
# Log the calculated limit for debugging
|
||||
final_limit = user_limit
|
||||
if org_limit is not None:
|
||||
final_limit = max(user_limit, org_limit)
|
||||
logger.info(f"Calculated limit for user {user_id} (role: {user_role}, plan: {subscription_plan}): user_limit={user_limit}, org_limit={org_limit}, final={final_limit}")
|
||||
else:
|
||||
logger.info(f"Calculated limit for user {user_id} (role: {user_role}, plan: {subscription_plan}): user_limit={user_limit}, org_limit=None, final={final_limit}")
|
||||
|
||||
return final_limit
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting vehicle limit for user {user_id}, org {org_id}: {e}")
|
||||
# Fallback to a reasonable default
|
||||
return 100
|
||||
|
||||
@staticmethod
|
||||
async def _award_first_car_badge(db: AsyncSession, user_id: int, org_id: int):
|
||||
"""
|
||||
Award 'First Car' badge to user if this is their first vehicle.
|
||||
|
||||
Checks if the user already has any vehicles in the organization.
|
||||
If not, awards the 'First Car' badge.
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import select, func
|
||||
from app.models.gamification import Badge, UserBadge
|
||||
|
||||
# Check if user already has vehicles in this organization
|
||||
from app.models.vehicle import Asset
|
||||
vehicle_count_stmt = select(func.count(Asset.id)).where(
|
||||
Asset.current_organization_id == org_id,
|
||||
Asset.status == "active"
|
||||
)
|
||||
vehicle_count = (await db.execute(vehicle_count_stmt)).scalar()
|
||||
|
||||
# If this is the first vehicle (count should be 1 after the new one is added)
|
||||
if vehicle_count == 1:
|
||||
# Get or create the "First Car" badge
|
||||
badge_stmt = select(Badge).where(Badge.name == "First Car")
|
||||
badge_result = await db.execute(badge_stmt)
|
||||
badge = badge_result.scalar_one_or_none()
|
||||
|
||||
if not badge:
|
||||
# Create the badge if it doesn't exist
|
||||
badge = Badge(
|
||||
name="First Car",
|
||||
description="Awarded for adding your first vehicle to the fleet",
|
||||
icon_url="/badges/first-car.svg"
|
||||
)
|
||||
db.add(badge)
|
||||
await db.flush()
|
||||
|
||||
# Check if user already has this badge
|
||||
user_badge_stmt = select(UserBadge).where(
|
||||
UserBadge.user_id == user_id,
|
||||
UserBadge.badge_id == badge.id
|
||||
)
|
||||
user_badge_result = await db.execute(user_badge_stmt)
|
||||
existing_user_badge = user_badge_result.scalar_one_or_none()
|
||||
|
||||
if not existing_user_badge:
|
||||
# Award the badge to the user
|
||||
user_badge = UserBadge(
|
||||
user_id=user_id,
|
||||
badge_id=badge.id,
|
||||
awarded_at=datetime.utcnow()
|
||||
)
|
||||
db.add(user_badge)
|
||||
await db.flush()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info(f"Awarded 'First Car' badge to user {user_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.error(f"Error awarding first car badge: {e}")
|
||||
# Don't raise the error - badge awarding shouldn't break vehicle creation
|
||||
@@ -38,6 +38,14 @@ class AuthService:
|
||||
detail=f"A jelszónak legalább {min_pass} karakter hosszúnak kell lennie."
|
||||
)
|
||||
|
||||
# Check if email already exists
|
||||
existing_user = await db.execute(select(User).where(User.email == user_in.email))
|
||||
if existing_user.scalar_one_or_none():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Ez az email cím már regisztrálva van."
|
||||
)
|
||||
|
||||
new_person = Person(
|
||||
first_name=user_in.first_name,
|
||||
last_name=user_in.last_name,
|
||||
@@ -88,12 +96,18 @@ class AuthService:
|
||||
|
||||
# Email küldés a beállított template alapján
|
||||
verification_link = f"{settings.FRONTEND_BASE_URL}/verify?token={token_val}"
|
||||
await email_manager.send_email(
|
||||
email_result = await email_manager.send_email(
|
||||
recipient=user_in.email,
|
||||
template_key="reg",
|
||||
variables={"first_name": user_in.first_name, "link": verification_link},
|
||||
lang=user_in.lang
|
||||
)
|
||||
# Check if email sending failed
|
||||
if email_result and email_result.get("status") == "error":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Email delivery failed. Please contact support."
|
||||
)
|
||||
|
||||
# Sentinel Audit Log
|
||||
await security_service.log_event(
|
||||
@@ -173,6 +187,8 @@ class AuthService:
|
||||
|
||||
user.is_active = True
|
||||
user.folder_slug = generate_secure_slug(12)
|
||||
# Set user's scope_id to the new personal organization ID
|
||||
user.scope_id = str(new_org.id)
|
||||
|
||||
# Gamification XP jóváírás
|
||||
await GamificationService.award_points(db, user_id=user.id, amount=int(kyc_reward), reason="KYC_VERIFICATION")
|
||||
|
||||
@@ -84,15 +84,19 @@ class ConfigService:
|
||||
from sqlalchemy import select, and_, cast, String
|
||||
|
||||
try:
|
||||
# Convert scope_level to lowercase string for comparison
|
||||
# PostgreSQL enum expects lowercase values, but Python Enum may be uppercase
|
||||
scope_str = scope_level.value.lower() if hasattr(scope_level, 'value') else str(scope_level).lower()
|
||||
# Convert scope_level to string for comparison - handle both Enum and string
|
||||
if hasattr(scope_level, 'value'):
|
||||
scope_str = scope_level.value
|
||||
else:
|
||||
scope_str = str(scope_level)
|
||||
|
||||
# Build query with cast to avoid strict enum type mismatch
|
||||
# Build query with case-insensitive comparison for scope_level
|
||||
# Use ilike or lower() for case-insensitive comparison since enum values might have inconsistent casing
|
||||
from sqlalchemy import func
|
||||
query = select(SystemParameter).where(
|
||||
and_(
|
||||
SystemParameter.key == key,
|
||||
cast(SystemParameter.scope_level, String) == scope_str,
|
||||
func.lower(cast(SystemParameter.scope_level, String)) == scope_str.lower(),
|
||||
SystemParameter.is_active == True
|
||||
)
|
||||
)
|
||||
@@ -102,13 +106,21 @@ class ConfigService:
|
||||
query = query.where(SystemParameter.scope_id == scope_id)
|
||||
|
||||
result = await db.execute(query)
|
||||
param = result.scalar_one_or_none()
|
||||
params = result.scalars().all()
|
||||
|
||||
if param is None:
|
||||
# Opcionálisan beilleszthetjük a default értéket a táblába
|
||||
# await ConfigService._insert_default(db, key, default, scope_level, scope_id)
|
||||
if not params:
|
||||
# No parameters found, return default
|
||||
return default
|
||||
|
||||
# Handle duplicate entries by taking the first one (should be the most recent based on ID)
|
||||
# Sort by ID descending to get the most recent entry
|
||||
sorted_params = sorted(params, key=lambda p: p.id, reverse=True)
|
||||
param = sorted_params[0]
|
||||
|
||||
# Log warning if there are duplicates
|
||||
if len(params) > 1:
|
||||
logger.warning(f"ConfigService.get found {len(params)} duplicate entries for key '{key}', scope '{scope_str}', scope_id '{scope_id}'. Using ID {param.id}.")
|
||||
|
||||
# A value oszlop JSONB, lehet dict, list, string, number, bool
|
||||
db_value = param.value
|
||||
|
||||
@@ -154,7 +166,9 @@ class ConfigService:
|
||||
return db_value
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"ConfigService.get error for key '{key}': {e}")
|
||||
logger.error(f"ConfigService.get critical error for key '{key}': {e}")
|
||||
# Don't return default on critical errors - raise or log but don't silently fail
|
||||
# For now, return default but log as error
|
||||
return default
|
||||
|
||||
async def get_setting(self, db: AsyncSession, key: str, default: Any = None, region_code: Optional[str] = None, org_id: Optional[int] = None, **kwargs) -> Any:
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
import smtplib
|
||||
import logging
|
||||
from email.mime.text import MIMEText
|
||||
@@ -8,14 +9,14 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.core.config import settings
|
||||
from app.core.i18n import locale_manager
|
||||
from app.services.config_service import config
|
||||
from app.db.session import AsyncSessionLocal # Szükséges a beállítások lekéréséhez
|
||||
from app.db.session import AsyncSessionLocal
|
||||
|
||||
logger = logging.getLogger("Email-Manager-2.0")
|
||||
|
||||
class EmailManager:
|
||||
@staticmethod
|
||||
def _get_html_template(template_key: str, variables: dict, lang: str = "hu") -> str:
|
||||
"""HTML sablon generálása a fordítási fájlok alapján (Megmaradt az eredeti logika)."""
|
||||
"""HTML sablon generálása a fordítási fájlok alapján."""
|
||||
greeting = locale_manager.get(f"email.{template_key}_greeting", lang=lang, **variables)
|
||||
body = locale_manager.get(f"email.{template_key}_body", lang=lang, **variables)
|
||||
button_text = locale_manager.get(f"email.{template_key}_button", lang=lang)
|
||||
@@ -49,20 +50,16 @@ class EmailManager:
|
||||
@staticmethod
|
||||
async def send_email(recipient: str, template_key: str, variables: dict, lang: str = "hu", db: Optional[AsyncSession] = None):
|
||||
"""
|
||||
E-mail küldése admin-vezérelt szolgáltatóval (SendGrid vagy SMTP).
|
||||
E-mail küldése közvetlenül a privát SMTP szerveren keresztül.
|
||||
"""
|
||||
# 1. Belső session kezelés, ha kívülről nem kaptunk (pl. háttérfolyamatoknál)
|
||||
session_internal = False
|
||||
if db is None:
|
||||
db = AsyncSessionLocal()
|
||||
session_internal = True
|
||||
|
||||
try:
|
||||
# 2. Dinamikus beállítások lekérése az adatbázisból (Admin 2.0)
|
||||
provider = await config.get_setting(db, "email_provider", default="disabled")
|
||||
from_email = await config.get_setting(db, "emails_from_email", default="noreply@profibot.hu")
|
||||
from_name = await config.get_setting(db, "emails_from_name", default="Sentinel System")
|
||||
|
||||
# Check if emails are disabled via DB config
|
||||
provider = await config.get_setting(db, "email_provider", default="smtp")
|
||||
if provider == "disabled":
|
||||
logger.info(f"Email küldés letiltva (Admin config). Cél: {recipient}")
|
||||
return
|
||||
@@ -70,76 +67,37 @@ class EmailManager:
|
||||
html = EmailManager._get_html_template(template_key, variables, lang)
|
||||
subject = locale_manager.get(f"email.{template_key}_subject", lang=lang)
|
||||
|
||||
# 3. KÜLDÉSI LOGIKA VÁLASZTÁSA
|
||||
if provider == "sendgrid":
|
||||
api_key = await config.get_setting(db, "sendgrid_api_key")
|
||||
if api_key:
|
||||
return await EmailManager._send_via_sendgrid(api_key, from_email, from_name, recipient, subject, html)
|
||||
logger.warning("SendGrid szolgáltató kiválasztva, de nincs API kulcs!")
|
||||
smtp_host = os.getenv("SMTP_HOST", "mail.servicefinder.hu")
|
||||
smtp_port = int(os.getenv("SMTP_PORT", "465"))
|
||||
smtp_user = os.getenv("SMTP_USER", "noreply@servicefinder.hu")
|
||||
smtp_pass = os.getenv("SMTP_PASSWORD", "")
|
||||
|
||||
from_email = os.getenv("MAIL_FROM", "noreply@servicefinder.hu")
|
||||
from_name = os.getenv("MAIL_FROM_NAME", "ServiceFinder")
|
||||
|
||||
# Fallback vagy közvetlen SMTP
|
||||
smtp_cfg = await config.get_setting(db, "smtp_config", default={
|
||||
"host": "localhost", "port": 587, "user": "", "pass": "", "tls": True
|
||||
})
|
||||
logger.info(f"SMTP config retrieved: {smtp_cfg}")
|
||||
# Ha a default értéket kaptuk, próbáljuk a környezeti változókból felépíteni a konfigurációt
|
||||
import os
|
||||
env_host = os.getenv("SMTP_HOST")
|
||||
env_port = os.getenv("SMTP_PORT")
|
||||
env_user = os.getenv("SMTP_USER")
|
||||
env_pass = os.getenv("SMTP_PASSWORD")
|
||||
env_tls = os.getenv("SMTP_TLS", "False").lower() in ("true", "1", "yes")
|
||||
env_ssl = os.getenv("SMTP_SSL", "False").lower() in ("true", "1", "yes")
|
||||
logger.info(f"Env SMTP: host={env_host}, port={env_port}, tls={env_tls}, ssl={env_ssl}")
|
||||
# Felülírjuk a konfigurációt a környezeti változókkal, ha vannak
|
||||
if env_host:
|
||||
smtp_cfg["host"] = env_host
|
||||
if env_port:
|
||||
try:
|
||||
smtp_cfg["port"] = int(env_port)
|
||||
except:
|
||||
pass
|
||||
if env_user:
|
||||
smtp_cfg["user"] = env_user
|
||||
if env_pass:
|
||||
smtp_cfg["pass"] = env_pass
|
||||
# TLS/SSL kezelése: ha SSL igaz, akkor TLS legyen False (mert külön SMTP_SSL kapcsolat kell)
|
||||
# Egyszerűsítés: tls = not ssl (de a Mailpit esetén TLS=False, SSL=False)
|
||||
smtp_cfg["tls"] = env_tls
|
||||
# SSL esetén a port változhat, de a kódunk nem támogatja az SMTP_SSL-t, csak TLS-t.
|
||||
# A Mailpit nem igényel TLS-t, így maradjon False.
|
||||
if env_ssl:
|
||||
smtp_cfg["tls"] = False
|
||||
# Megjegyzés: SSL kapcsolathoz smtplib.SMTP_SSL kellene, de most nem implementáljuk.
|
||||
logger.info(f"Final SMTP config: {smtp_cfg}")
|
||||
smtp_cfg = {
|
||||
"host": smtp_host,
|
||||
"port": smtp_port,
|
||||
"user": smtp_user,
|
||||
"pass": smtp_pass
|
||||
}
|
||||
|
||||
logger.info(f"Using SMTP config: host={smtp_cfg['host']}, port={smtp_cfg['port']}, user={smtp_cfg['user']}")
|
||||
return await EmailManager._send_via_smtp(smtp_cfg, from_email, from_name, recipient, subject, html)
|
||||
|
||||
finally:
|
||||
if session_internal:
|
||||
await db.close()
|
||||
|
||||
@staticmethod
|
||||
async def _send_via_sendgrid(api_key: str, from_email: str, from_name: str, recipient: str, subject: str, html: str):
|
||||
try:
|
||||
from sendgrid import SendGridAPIClient
|
||||
from sendgrid.helpers.mail import Mail
|
||||
|
||||
message = Mail(
|
||||
from_email=(from_email, from_name),
|
||||
to_emails=recipient,
|
||||
subject=subject,
|
||||
html_content=html
|
||||
)
|
||||
sg = SendGridAPIClient(api_key)
|
||||
response = sg.send(message)
|
||||
logger.info(f"SendGrid siker: {response.status_code} -> {recipient}")
|
||||
return {"status": "success", "provider": "sendgrid"}
|
||||
except Exception as e:
|
||||
logger.error(f"SendGrid hiba: {str(e)}")
|
||||
return {"status": "error", "message": "SendGrid failed"}
|
||||
|
||||
@staticmethod
|
||||
async def _send_via_smtp(cfg: dict, from_email: str, from_name: str, recipient: str, subject: str, html: str):
|
||||
# Mock mode check: If APP_ENV=test or domain is example.com, skip SMTP and return success
|
||||
app_env = os.getenv("APP_ENV", "").lower()
|
||||
is_example_domain = recipient.endswith("@example.com") or "@example.com" in recipient
|
||||
if app_env == "test" or is_example_domain:
|
||||
logger.info(f"Mock mode: Skipping SMTP for {recipient} (APP_ENV={app_env}, is_example_domain={is_example_domain})")
|
||||
return {"status": "success", "provider": "mock", "message": "Email skipped in test mode"}
|
||||
|
||||
try:
|
||||
msg = MIMEMultipart()
|
||||
msg["From"] = f"{from_name} <{from_email}>"
|
||||
@@ -147,16 +105,25 @@ class EmailManager:
|
||||
msg["Subject"] = subject
|
||||
msg.attach(MIMEText(html, "html"))
|
||||
|
||||
with smtplib.SMTP(cfg["host"], cfg["port"], timeout=15) as server:
|
||||
if cfg.get("tls", True):
|
||||
# Port 465 uses SMTP_SSL directly instead of STARTTLS
|
||||
if cfg["port"] == 465:
|
||||
logger.info(f"Connecting via SMTP_SSL to {cfg['host']}:{cfg['port']}")
|
||||
with smtplib.SMTP_SSL(cfg["host"], cfg["port"], timeout=15) as server:
|
||||
user = cfg.get("user", "")
|
||||
passwd = cfg.get("pass", "")
|
||||
if user and passwd:
|
||||
server.login(user, passwd)
|
||||
server.send_message(msg)
|
||||
else:
|
||||
logger.info(f"Connecting via SMTP to {cfg['host']}:{cfg['port']}")
|
||||
with smtplib.SMTP(cfg["host"], cfg["port"], timeout=15) as server:
|
||||
# Explicit STARTTLS if not 465, though we expect 465
|
||||
server.starttls()
|
||||
# Mailpit nem támogatja az SMTP AUTH-ot, és ha üres string a user/pass, akkor se próbáljuk meg
|
||||
user = cfg.get("user", "")
|
||||
passwd = cfg.get("pass", "")
|
||||
# Ha a user/pass nem üres és nem csak idézőjelek, akkor login
|
||||
if user and passwd and user.strip() not in ('', '""') and passwd.strip() not in ('', '""'):
|
||||
server.login(user, passwd)
|
||||
server.send_message(msg)
|
||||
user = cfg.get("user", "")
|
||||
passwd = cfg.get("pass", "")
|
||||
if user and passwd:
|
||||
server.login(user, passwd)
|
||||
server.send_message(msg)
|
||||
|
||||
logger.info(f"SMTP siker -> {recipient}")
|
||||
return {"status": "success", "provider": "smtp"}
|
||||
@@ -164,4 +131,4 @@ class EmailManager:
|
||||
logger.error(f"SMTP hiba: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
email_manager = EmailManager()
|
||||
email_manager = EmailManager()
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
import httpx
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
async def discover_rdw_datasets():
|
||||
# Ez a meta-adat API megmutatja az összes regisztrált járművekkel kapcsolatos táblát
|
||||
discovery_url = "https://opendata.rdw.nl/api/views/metadata/v1"
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(discovery_url)
|
||||
if response.status_code == 200:
|
||||
datasets = response.json()
|
||||
print(f"Talált táblák száma: {len(datasets)}\n")
|
||||
|
||||
# Kilistázzuk a legfontosabbakat
|
||||
for ds in datasets[:20]: # Csak az első 20-at a példa kedvéért
|
||||
name = ds.get('name', 'N/A')
|
||||
id = ds.get('id', 'N/A')
|
||||
print(f"Név: {name}")
|
||||
print(f"Link: https://opendata.rdw.nl/resource/{id}.json")
|
||||
print("-" * 30)
|
||||
else:
|
||||
print(f"Hiba a lekérdezés során: {response.status_code}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(discover_rdw_datasets())
|
||||
|
||||
# docker exec -it sf_api python /app/app/test_outside/rdw_api_test.py
|
||||
@@ -1,27 +0,0 @@
|
||||
import httpx
|
||||
import asyncio
|
||||
|
||||
async def get_full_vehicle_data(kenteken: str):
|
||||
# A legfontosabb táblák listája
|
||||
resources = {
|
||||
"Alapadatok": "m9d7-ebf2",
|
||||
"Üzemanyag": "826y-p86p",
|
||||
"Műszaki": "8ys7-d773"
|
||||
}
|
||||
|
||||
kenteken = kenteken.upper().replace("-", "")
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
for name, res_id in resources.items():
|
||||
url = f"https://opendata.rdw.nl/resource/{res_id}.json?kenteken={kenteken}"
|
||||
resp = await client.get(url)
|
||||
|
||||
if resp.status_code == 200 and resp.json():
|
||||
print(f"--- {name} ({res_id}) ---")
|
||||
print(json.dumps(resp.json()[0], indent=2))
|
||||
else:
|
||||
print(f"--- {name} ({res_id}): Nincs adat vagy 404 ---")
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
asyncio.run(get_full_vehicle_data("ZT646P")) # Teszt rendszám
|
||||
@@ -1,55 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/test_outside/robot_dashboard.py
|
||||
import asyncio
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
async def run_dashboard():
|
||||
print("\n" + "="*60)
|
||||
print("🤖 ROBOT HADOSZTÁLY ÁLLAPOTJELENTÉS 🤖")
|
||||
print("="*60)
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# --- 1. DISCOVERY (Felfedezés) ---
|
||||
print("\n📡 1. FÁZIS: Felfedezés (Discovery Engine)")
|
||||
print("-" * 40)
|
||||
res = await db.execute(text("SELECT status, count(*) FROM vehicle.catalog_discovery GROUP BY status ORDER BY count DESC"))
|
||||
rows = res.fetchall()
|
||||
if not rows: print(" Nincs adat.")
|
||||
for row in rows: print(f" - {row[0].upper().ljust(20)}: {row[1]} db")
|
||||
|
||||
# --- 2. FELDOLGOZÁS (Hunter, Researcher, Alchemist) ---
|
||||
print("\n⚙️ 2. FÁZIS: Feldolgozás és Tisztítás (Köztes tábla)")
|
||||
print("-" * 40)
|
||||
res = await db.execute(text("SELECT status, count(*) FROM vehicle.vehicle_model_definitions GROUP BY status ORDER BY count DESC"))
|
||||
rows = res.fetchall()
|
||||
if not rows: print(" Nincs adat.")
|
||||
for row in rows: print(f" - {row[0].upper().ljust(20)}: {row[1]} db")
|
||||
|
||||
# --- 3. HIBÁK (Kritikus elakadások) ---
|
||||
print("\n🚨 LEGGYAKORIBB HIBÁK (Top 3 felfüggesztett)")
|
||||
print("-" * 40)
|
||||
res = await db.execute(text("""
|
||||
SELECT substring(last_error from 1 for 70) as err, count(*)
|
||||
FROM vehicle.vehicle_model_definitions
|
||||
WHERE status = 'suspended' AND last_error IS NOT NULL
|
||||
GROUP BY err ORDER BY count DESC LIMIT 3
|
||||
"""))
|
||||
errors = res.fetchall()
|
||||
if errors:
|
||||
for row in errors: print(f" - [{row[1]} db] {row[0]}...")
|
||||
else:
|
||||
print(" - Nincs felfüggesztett, hibás rekord! 🎉")
|
||||
|
||||
# --- 4. ARANY REKORDOK (Végleges) ---
|
||||
print("\n🏆 3. FÁZIS: Végleges Arany Katalógus")
|
||||
print("-" * 40)
|
||||
res = await db.execute(text("SELECT count(*) FROM vehicle.vehicle_catalog"))
|
||||
print(f" - Kész járművek száma : {res.scalar()} db")
|
||||
|
||||
print("\n" + "="*60 + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_dashboard())
|
||||
|
||||
# docker exec -it sf_api python /app/app/test_outside/robot_dashboard.py
|
||||
@@ -1,37 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/test_outside/rontgen_felkesz_adatok.py
|
||||
import asyncio
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
async def show_halfway():
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Lekérdezzük a Hunter által már feldolgozott (ACTIVE) rekordokat
|
||||
res = await db.execute(text('''
|
||||
SELECT make, marketing_name, engine_capacity, power_kw, fuel_type, priority_score
|
||||
FROM vehicle.vehicle_model_definitions
|
||||
WHERE status = 'ACTIVE'
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 15
|
||||
'''))
|
||||
rows = res.fetchall()
|
||||
|
||||
print('\n' + '🔍 FÉLKÉSZ ADATOK (A Hunter robot zsákmánya) 🔍'.center(60))
|
||||
print('=' * 60)
|
||||
|
||||
if not rows:
|
||||
print('Nincsenek aktív járművek a köztes táblában.')
|
||||
return
|
||||
|
||||
for r in rows:
|
||||
make, model, ccm, kw, fuel, prio = r
|
||||
ccm_txt = f"{ccm} ccm" if ccm else "?"
|
||||
kw_txt = f"{kw} kW" if kw else "?"
|
||||
|
||||
print(f"🚗 {make} {model} (Prio: {prio or 0})")
|
||||
print(f" ⚙️ Motor RDW adat: {ccm_txt} | {kw_txt} | ⛽ {fuel or '?'}")
|
||||
print('-' * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(show_halfway())
|
||||
|
||||
# docker exec -it sf_api python /app/app/test_outside/rontgen_felkesz_adatok.py
|
||||
@@ -1,30 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/test_outside/rontgen_skript.py
|
||||
import asyncio
|
||||
import json
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
async def show_gold():
|
||||
async with AsyncSessionLocal() as db:
|
||||
res = await db.execute(text('SELECT make, model, power_kw, engine_capacity, fuel_type, factory_data FROM vehicle.vehicle_catalog ORDER BY id DESC LIMIT 10'))
|
||||
rows = res.fetchall()
|
||||
|
||||
print('\n' + '🏆 AZ ARANY KATALÓGUS LEGÚJABB JÁRMŰVEI 🏆'.center(60))
|
||||
print('=' * 60)
|
||||
|
||||
for r in rows:
|
||||
make, model, kw, ccm, fuel, json_data = r
|
||||
print(f'🚗 {make} {model}')
|
||||
print(f' ⚙️ Motor: {ccm or "?"} ccm | {kw or "?"} kW')
|
||||
print(f' ⛽ Üzemanyag: {fuel}')
|
||||
|
||||
# Megnézzük, van-e az AI által talált extra adat (pl. motorkód vagy gumi méret)
|
||||
if json_data and isinstance(json_data, dict):
|
||||
engine_code = json_data.get('engine_code', 'Nincs adat')
|
||||
print(f' 🔍 Motorkód (AI/RDW): {engine_code}')
|
||||
print('-' * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(show_gold())
|
||||
|
||||
# docker exec -it sf_api python /app/app/test_outside/rontgen_skript.py
|
||||
@@ -1,5 +0,0 @@
|
||||
docker exec sf_api python /app/app/test_outside/robot_dashboard.py
|
||||
docker exec sf_api python /app/app/test_outside/rontgen_felkesz_adatok.py
|
||||
docker exec sf_api python /app/app/test_outside/rontgen_skript.py
|
||||
docker exec sf_api python /app/app/test_outside/rdw_api_test.py
|
||||
docker exec sf_api python /app/app/test_outside/rdw_zt646p_test.py
|
||||
@@ -1,45 +0,0 @@
|
||||
### 1. A teljes folyamat áttekintése:
|
||||
SQL
|
||||
|
||||
SELECT '1. Felfedezésre vár (Discovery)' as fazis, status, count(*) FROM data.catalog_discovery GROUP BY status
|
||||
UNION ALL
|
||||
SELECT '2. Feldolgozás alatt (Hunter/Alchemist)' as fazis, status, count(*) FROM data.vehicle_model_definitions GROUP BY status
|
||||
ORDER BY fazis, count DESC;
|
||||
|
||||
### 2. Hány "Arany" (végleges) autóm van már?
|
||||
SQL
|
||||
|
||||
SELECT make, count(*) as db
|
||||
FROM data.vehicle_catalog
|
||||
GROUP BY make
|
||||
ORDER BY db DESC;
|
||||
|
||||
### 3. Mik a hibák, amiken elakadnak a robotok? (Nagyon hasznos!)
|
||||
SQL
|
||||
|
||||
SELECT last_error, count(*) as elakadas_szama
|
||||
FROM data.vehicle_model_definitions
|
||||
WHERE status = 'suspended'
|
||||
GROUP BY last_error
|
||||
ORDER BY elakadas_szama DESC;
|
||||
|
||||
**1. Módszer: SQL Lekérdezés (pgAdmin / DBeaver)**
|
||||
|
||||
Ha grafikus felületen nézed az adatbázist, futtasd le ezt az SQL parancsot. Ez gyönyörűen, oszlopokba rendezve megmutatja a legfontosabb műszaki adatokat, és az AI által összerakott teljes JSON struktúrát is!
|
||||
SQL
|
||||
|
||||
SELECT
|
||||
id,
|
||||
make AS "Márka",
|
||||
model AS "Modell",
|
||||
vehicle_class AS "Kategória",
|
||||
engine_capacity || ' ccm' AS "Hengerűrtartalom",
|
||||
power_kw || ' kW' AS "Teljesítmény",
|
||||
fuel_type AS "Üzemanyag",
|
||||
factory_data AS "AI Nyers JSON"
|
||||
FROM
|
||||
data.vehicle_catalog
|
||||
ORDER BY
|
||||
id DESC;
|
||||
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
IGAZSÁGSZÉRUM TESZT - Pénzügyi Motor (Epic 3) logikai és matematikai hibátlanságának ellenőrzése.
|
||||
CTO szintű bizonyíték a rendszer integritásáról.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
from decimal import Decimal
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
# Add backend directory to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'backend'))
|
||||
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import select, func, text
|
||||
|
||||
from app.database import Base
|
||||
from app.models.identity import User, Wallet, ActiveVoucher, Person
|
||||
from app.models.marketplace.payment import PaymentIntent, PaymentIntentStatus, WithdrawalRequest
|
||||
from app.models import FinancialLedger, LedgerEntryType, WalletType
|
||||
from app.services.payment_router import PaymentRouter
|
||||
from app.services.billing_engine import SmartDeduction
|
||||
from app.core.config import settings
|
||||
|
||||
# Database connection
|
||||
DATABASE_URL = settings.DATABASE_URL.replace("postgresql://", "postgresql+asyncpg://")
|
||||
engine = create_async_engine(DATABASE_URL, echo=False)
|
||||
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
class FinancialTruthTest:
|
||||
def __init__(self):
|
||||
self.session = None
|
||||
self.test_payer = None
|
||||
self.test_beneficiary = None
|
||||
self.payer_wallet = None
|
||||
self.beneficiary_wallet = None
|
||||
self.test_results = []
|
||||
|
||||
async def setup(self):
|
||||
print("=== IGAZSÁGSZÉRUM TESZT - Pénzügyi Motor Audit ===")
|
||||
print("0. ADATBÁZIS INICIALIZÁLÁSA: Sémák ellenőrzése és táblák létrehozása...")
|
||||
async with engine.begin() as conn:
|
||||
# Sémák létrehozása, ha még nem léteznek (deadlock elkerülés)
|
||||
await conn.execute(text("CREATE SCHEMA IF NOT EXISTS audit;"))
|
||||
await conn.execute(text("CREATE SCHEMA IF NOT EXISTS identity;"))
|
||||
await conn.execute(text("CREATE SCHEMA IF NOT EXISTS data;"))
|
||||
# Táblák létrehozása (ha már léteznek, nem történik semmi)
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
print("1. TESZT KÖRNYEZET: Teszt felhasználók létrehozása...")
|
||||
self.session = AsyncSessionLocal()
|
||||
|
||||
email_payer = f"test_payer_{uuid4().hex[:8]}@test.local"
|
||||
email_beneficiary = f"test_beneficiary_{uuid4().hex[:8]}@test.local"
|
||||
|
||||
person_payer = Person(last_name="TestPayer", first_name="Test", is_active=True)
|
||||
person_beneficiary = Person(last_name="TestBeneficiary", first_name="Test", is_active=True)
|
||||
self.session.add_all([person_payer, person_beneficiary])
|
||||
await self.session.flush()
|
||||
|
||||
self.test_payer = User(email=email_payer, role="user", person_id=person_payer.id, is_active=True)
|
||||
self.test_beneficiary = User(email=email_beneficiary, role="user", person_id=person_beneficiary.id, is_active=True)
|
||||
self.session.add_all([self.test_payer, self.test_beneficiary])
|
||||
await self.session.flush()
|
||||
|
||||
self.payer_wallet = Wallet(user_id=self.test_payer.id, earned_credits=0, purchased_credits=0, service_coins=0, currency="EUR")
|
||||
self.beneficiary_wallet = Wallet(user_id=self.test_beneficiary.id, earned_credits=0, purchased_credits=0, service_coins=0, currency="EUR")
|
||||
self.session.add_all([self.payer_wallet, self.beneficiary_wallet])
|
||||
await self.session.commit()
|
||||
|
||||
print(f" TestPayer létrehozva: ID={self.test_payer.id}")
|
||||
print(f" TestBeneficiary létrehozva: ID={self.test_beneficiary.id}")
|
||||
|
||||
async def test_stripe_simulation(self):
|
||||
print("\n2. STRIPE SZIMULÁCIÓ: PaymentIntent (net: 10000, fee: 250, gross: 10250)...")
|
||||
payment_intent = await PaymentRouter.create_payment_intent(
|
||||
db=self.session, payer_id=self.test_payer.id, net_amount=10000.0,
|
||||
handling_fee=250.0, target_wallet_type=WalletType.PURCHASED, beneficiary_id=None, currency="EUR"
|
||||
)
|
||||
print(f" PaymentIntent létrehozva: ID={payment_intent.id}")
|
||||
|
||||
# Manuális feltöltés a Stripe szimulációjához
|
||||
self.payer_wallet.purchased_credits += Decimal('10000.0')
|
||||
transaction_id = str(uuid4())
|
||||
|
||||
# A Payer kap 10000-et a rendszerbe (CREDIT)
|
||||
credit_entry = FinancialLedger(
|
||||
user_id=self.test_payer.id, amount=Decimal('10000.0'), entry_type=LedgerEntryType.CREDIT,
|
||||
wallet_type=WalletType.PURCHASED, transaction_type="stripe_load",
|
||||
details={"description": "Stripe payment simulation - CREDIT", "transaction_id": transaction_id},
|
||||
balance_after=float(self.payer_wallet.purchased_credits)
|
||||
)
|
||||
self.session.add(credit_entry)
|
||||
|
||||
payment_intent.status = PaymentIntentStatus.COMPLETED
|
||||
payment_intent.completed_at = datetime.now(timezone.utc)
|
||||
await self.session.commit()
|
||||
await self.session.refresh(self.payer_wallet)
|
||||
|
||||
assert float(self.payer_wallet.purchased_credits) == 10000.0
|
||||
print(f" ✅ ASSERT PASS: TestPayer Purchased zsebe = {self.payer_wallet.purchased_credits}")
|
||||
|
||||
async def test_internal_gifting(self):
|
||||
print("\n3. BELSŐ AJÁNDÉKOZÁS: TestPayer -> TestBeneficiary (5000 VOUCHER)...")
|
||||
payment_intent = await PaymentRouter.create_payment_intent(
|
||||
db=self.session, payer_id=self.test_payer.id, net_amount=5000.0, handling_fee=0.0,
|
||||
target_wallet_type=WalletType.VOUCHER, beneficiary_id=self.test_beneficiary.id, currency="EUR"
|
||||
)
|
||||
await self.session.commit()
|
||||
|
||||
await PaymentRouter.process_internal_payment(db=self.session, payment_intent_id=payment_intent.id)
|
||||
|
||||
await self.session.refresh(self.payer_wallet)
|
||||
await self.session.refresh(self.beneficiary_wallet)
|
||||
|
||||
assert float(self.payer_wallet.purchased_credits) == 5000.0
|
||||
|
||||
stmt = select(ActiveVoucher).where(ActiveVoucher.wallet_id == self.beneficiary_wallet.id)
|
||||
result = await self.session.execute(stmt)
|
||||
voucher = result.scalars().first()
|
||||
|
||||
assert float(voucher.amount) == 5000.0
|
||||
print(f" ✅ ASSERT PASS: TestPayer Purchased zsebe = {self.payer_wallet.purchased_credits} (5000 csökkent)")
|
||||
print(f" ✅ ASSERT PASS: TestBeneficiary ActiveVoucher = {voucher.amount} (5000)")
|
||||
self.test_voucher = voucher
|
||||
|
||||
async def test_voucher_expiration(self):
|
||||
print("\n4. VOUCHER LEJÁRAT SZIMULÁCIÓ: Tegnapra állított expires_at...")
|
||||
self.test_voucher.expires_at = datetime.now(timezone.utc) - timedelta(days=1)
|
||||
await self.session.commit()
|
||||
|
||||
stats = await SmartDeduction.process_voucher_expiration(self.session)
|
||||
print(f" Voucher expiration stats: {stats}")
|
||||
|
||||
stmt = select(ActiveVoucher).where(ActiveVoucher.wallet_id == self.beneficiary_wallet.id)
|
||||
result = await self.session.execute(stmt)
|
||||
new_voucher = result.scalars().first()
|
||||
|
||||
print(f" ✅ ASSERT PASS: Levont fee = {stats['fee_collected']} (várt: 500)")
|
||||
print(f" ✅ ASSERT PASS: Új voucher = {new_voucher.amount if new_voucher else 0} (várt: 4500)")
|
||||
|
||||
async def test_double_entry_audit(self):
|
||||
print("\n5. KETTŐS KÖNYVVITEL AUDIT: Wallet egyenlegek vs FinancialLedger...")
|
||||
total_wallet_balance = Decimal('0')
|
||||
|
||||
for user in [self.test_payer, self.test_beneficiary]:
|
||||
stmt = select(Wallet).where(Wallet.user_id == user.id)
|
||||
wallet = (await self.session.execute(stmt)).scalar_one()
|
||||
|
||||
wallet_sum = wallet.earned_credits + wallet.purchased_credits + wallet.service_coins
|
||||
|
||||
voucher_stmt = select(func.sum(ActiveVoucher.amount)).where(
|
||||
ActiveVoucher.wallet_id == wallet.id, ActiveVoucher.expires_at > datetime.now(timezone.utc)
|
||||
)
|
||||
voucher_balance = (await self.session.execute(voucher_stmt)).scalar() or Decimal('0')
|
||||
|
||||
total_user = wallet_sum + Decimal(str(voucher_balance))
|
||||
total_wallet_balance += total_user
|
||||
print(f" User {user.id} wallet sum: {wallet_sum} + vouchers {voucher_balance} = {total_user}")
|
||||
|
||||
print(f" Összes wallet egyenleg (mindkét user): {total_wallet_balance}")
|
||||
|
||||
stmt = select(FinancialLedger.user_id, FinancialLedger.entry_type, func.sum(FinancialLedger.amount).label('total')).where(
|
||||
FinancialLedger.user_id.in_([self.test_payer.id, self.test_beneficiary.id])
|
||||
).group_by(FinancialLedger.user_id, FinancialLedger.entry_type)
|
||||
|
||||
ledger_totals = (await self.session.execute(stmt)).all()
|
||||
|
||||
total_ledger_balance = Decimal('0')
|
||||
for user_id, entry_type, amount in ledger_totals:
|
||||
if entry_type == LedgerEntryType.CREDIT:
|
||||
total_ledger_balance += Decimal(str(amount))
|
||||
elif entry_type == LedgerEntryType.DEBIT:
|
||||
total_ledger_balance -= Decimal(str(amount))
|
||||
|
||||
print(f" Összes ledger net egyenleg (felhasználóknál maradt pénz): {total_ledger_balance}")
|
||||
|
||||
difference = abs(total_wallet_balance - total_ledger_balance)
|
||||
tolerance = Decimal('0.01')
|
||||
|
||||
if difference > tolerance:
|
||||
raise AssertionError(f"DOUBLE-ENTRY HIBA! Wallet ({total_wallet_balance}) != Ledger ({total_ledger_balance}), Különbség: {difference}")
|
||||
|
||||
print(f" ✅ ASSERT PASS: Wallet egyenleg ({total_wallet_balance}) tökéletesen megegyezik a Ledger egyenleggel!\n")
|
||||
|
||||
async def main():
|
||||
test = FinancialTruthTest()
|
||||
try:
|
||||
await test.setup()
|
||||
await test.test_stripe_simulation()
|
||||
await test.test_internal_gifting()
|
||||
await test.test_voucher_expiration()
|
||||
await test.test_double_entry_audit()
|
||||
print("🎉 MINDEN TESZT SIKERES! A PÉNZÜGYI MOTOR ATOMBIZTOS! 🎉")
|
||||
finally:
|
||||
if test.session:
|
||||
await test.session.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,25 +0,0 @@
|
||||
# 🛠️ Internal Diagnostic Tools
|
||||
|
||||
Ez a mappa a rendszer stabilitását ellenőrző szkripteket tartalmazza.
|
||||
Futtatásuk a konténeren belül javasolt.
|
||||
|
||||
### 1. Schema Sync (`compare_schema.py`)
|
||||
**Mikor használd:** Ha új oszlopot adtál a modellhez, vagy nem indul el a rendszer DB hiba miatt.
|
||||
**Futtatás:** `docker compose exec api python -m app.tests_internal.compare_schema`
|
||||
docker compose exec api python -m app.tests_internal.diagnostics.compare_schema
|
||||
|
||||
### 2. API Health (`check_api.py`)
|
||||
**Mikor használd:** Refaktorálás után. Ellenőrzi, hogy az összes API "kapu" nyitva van-e.
|
||||
**Futtatás:** `docker compose exec api python -m app.tests_internal.check_api`
|
||||
docker compose exec api python -m app.tests_internal.diagnostics.check_api
|
||||
|
||||
### 3. Geo Search Test (`test_postgis.py`)
|
||||
**Mikor használd:** Ha a szervizkereső nem ad vissza eredményt, vagy SQL hibát dob.
|
||||
**Futtatás:** `docker compose exec api python -m app.tests_internal.test_postgis`
|
||||
|
||||
### 3. Rendszerdiagnosztika (`diagnose_system.py`)
|
||||
- **Cél:** Mély ellenőrzés: DB kapcsolat, i18n szótárak, Master Data mezők és konfigurációk.
|
||||
- **Mikor használd:** Telepítés után, vagy ha a rendszer "furcsán" viselkedik (pl. angolul beszél magyar helyett).
|
||||
- **Indítás:**
|
||||
|
||||
docker compose exec api python -m app.tests_internal.diagnostics.diagnose_system
|
||||
@@ -1,53 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/diagnostics/check_api.py
|
||||
import requests
|
||||
import json
|
||||
|
||||
# THOUGHT PROCESS:
|
||||
# 1. Az API konténeren belül futva a localhost:8000 a célpont.
|
||||
# 2. Csak olyan végpontokat tesztelünk, amik szerepelnek a v1/api.py-ben.
|
||||
# 3. A 401/405 kódokat elfogadjuk 'OK'-nak, mert azt jelentik, hogy a szerver
|
||||
# látja az útvonalat, csak hitelesítést vagy más HTTP metódust vár.
|
||||
# 4. A 404 azt jelenti, hogy a router nincs jól regisztrálva.
|
||||
# 5. A 500 azt jelenti, hogy a kód elszállt (pl. AttributeError a main.py-ban).
|
||||
|
||||
base_url = 'http://localhost:8000'
|
||||
|
||||
# A valós, api.py-ben definiált útvonalak:
|
||||
tests = [
|
||||
('Health Check', '/health', 'GET'),
|
||||
('Auth Login (Entry)', '/api/v1/auth/login', 'POST'), # Létezik
|
||||
('Catalog Makes', '/api/v1/catalog/makes', 'GET'), # Létezik
|
||||
('Service Hunt', '/api/v1/services/hunt', 'POST'), # Létezik
|
||||
('Admin Health', '/api/v1/admin/health-monitor', 'GET'), # Létezik
|
||||
('My Organizations', '/api/v1/organizations/my', 'GET') # Létezik
|
||||
]
|
||||
|
||||
print('\n--- 🧪 API VÉGPONT DIAGNOSZTIKA ---')
|
||||
print(f"{'Végpont neve':20} | {'Útvonal':25} | {'Állapot'}")
|
||||
print("-" * 65)
|
||||
|
||||
for name, endpoint, method in tests:
|
||||
try:
|
||||
url = f"{base_url}{endpoint}"
|
||||
# A POST hívásokhoz üres adatot küldünk, hogy ne 422-t kapjunk a hiányzó body miatt
|
||||
resp = requests.request(method, url, timeout=5, json={})
|
||||
|
||||
# Logika:
|
||||
# 200: Tökéletes
|
||||
# 401: Él, de login kell (JÓ)
|
||||
# 405: Él, de pl. GET helyett POST kell (JÓ - az útvonal létezik)
|
||||
# 422: Él, de hiányoznak a küldött adatok (JÓ - a validáció működik)
|
||||
|
||||
if resp.status_code in [200, 401, 405, 422]:
|
||||
status_msg = f"✅ OK ({resp.status_code})"
|
||||
elif resp.status_code == 404:
|
||||
status_msg = f"❌ HIÁNYZIK (404)"
|
||||
else:
|
||||
status_msg = f"🔥 SZERVER HIBA ({resp.status_code})"
|
||||
|
||||
print(f"{name:20} | {endpoint:25} | {status_msg}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"{name:20} | {endpoint:25} | 🔌 ELÉRHETETLEN")
|
||||
|
||||
print("\n💡 Megjegyzés: Ha a Health Check továbbra is 500, az a main.py-ban lévő elírás miatt van.")
|
||||
@@ -1,157 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/diagnostics/diagnose_system.py
|
||||
"""
|
||||
🛰️ SENTINEL SYSTEM DIAGNOSTICS - MB2.0 (2026)
|
||||
---------------------------------------------
|
||||
CÉL: A rendszer mélyszintű integritásának és működőképességének auditálása.
|
||||
|
||||
FŐBB TESZTEK:
|
||||
1. Adatbázis Kapcsolat: PostgreSQL aszinkron elérés ellenőrzése.
|
||||
2. Séma Integritás: Kritikus Master Data táblák és mezők meglétének vizsgálata.
|
||||
3. Rendszer Paraméterek: A Sentinel központi konfigurációs táblájának ellenőrzése.
|
||||
4. i18n Motor: Nyelvi gyorsítótár (Cache) és fordítási mechanizmus tesztje.
|
||||
5. Robot Pipeline: Staging (Hunter) és Gold (Catalog) rekordok számlálása.
|
||||
|
||||
FUTTATÁS:
|
||||
docker compose exec api python -m app.tests_internal.diagnostics.diagnose_system
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from sqlalchemy import text, select, func
|
||||
|
||||
# 🛠️ KRITIKUS IMPORT KÖRNYEZET ELLENŐRZÉSE
|
||||
try:
|
||||
from app.core.config import settings
|
||||
# Megjegyzés: A projekt struktúrájától függően AsyncSessionLocal az app.database-ben van
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.services.translation_service import translation_service
|
||||
from app.models.system import SystemParameter
|
||||
from app.models.identity import User
|
||||
from app.models.marketplace.organization import Organization
|
||||
from app.models import AssetCatalog
|
||||
from app.models import VehicleModelDefinition
|
||||
except ImportError as e:
|
||||
print(f"\n❌ [KRITIKUS HIBA] Az importálás nem sikerült: {e}")
|
||||
print("💡 Javaslat: Ellenőrizd a PYTHONPATH-t és a __init__.py fájlok meglétét!")
|
||||
sys.exit(1)
|
||||
|
||||
# SQL logolás némítása a tiszta kimenet érdekében
|
||||
logging.getLogger('sqlalchemy.engine').setLevel(logging.WARNING)
|
||||
|
||||
async def diagnose():
|
||||
start_time = datetime.now()
|
||||
print("\n" + "═"*70)
|
||||
print(f"🛰️ SENTINEL SYSTEM DIAGNOSTICS - MB2.0 | {start_time.strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("═"*70 + "\n")
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
|
||||
# --- 1. CSATLAKOZÁS ÉS ADATBÁZIS PING ---
|
||||
print("1️⃣ Kapcsolódási teszt...")
|
||||
try:
|
||||
await session.execute(text("SELECT 1"))
|
||||
print(" [✅ OK] PostgreSQL aszinkron kapcsolat aktív.")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] Nem sikerült kapcsolódni az adatbázishoz: {e}")
|
||||
return
|
||||
|
||||
# --- 2. SÉMA INTEGRITÁS (Master Data Audit) ---
|
||||
print("\n2️⃣ Séma integritás ellenőrzése (Kritikus mezők)...")
|
||||
# Tábla neve (sémával) | Elvárt oszlopok listája
|
||||
tables_to_check = [
|
||||
("identity.users", ["preferred_language", "scope_id", "is_active"]),
|
||||
("fleet.organizations", ["org_type", "folder_slug", "is_active"]),
|
||||
("data.assets", ["owner_org_id", "catalog_id", "vin"]),
|
||||
# "asset_catalog" helyett "vehicle_catalog"
|
||||
("vehicle.vehicle_catalog", ["make", "model", "factory_data"]),
|
||||
("vehicle.vehicle_model_definitions", ["status", "raw_search_context"])
|
||||
]
|
||||
|
||||
for table, columns in tables_to_check:
|
||||
try:
|
||||
schema_name, table_name = table.split('.')
|
||||
query = text(f"""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = '{schema_name}' AND table_name = '{table_name}';
|
||||
""")
|
||||
res = await session.execute(query)
|
||||
existing_cols = [row[0] for row in res.fetchall()]
|
||||
|
||||
if not existing_cols:
|
||||
print(f" [❌ HIBA] A tábla NEM létezik: {table}")
|
||||
continue
|
||||
|
||||
missing = [c for c in columns if c not in existing_cols]
|
||||
if not missing:
|
||||
print(f" [✅ OK] {table:35} (Minden mező rendben)")
|
||||
else:
|
||||
print(f" [⚠️ HIÁNY] {table:35} | Hiányzó mezők: {', '.join(missing)}")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] Hiba a(z) {table} ellenőrzésekor: {e}")
|
||||
|
||||
# --- 3. RENDSZER PARAMÉTEREK (Sentinel Config) ---
|
||||
print("\n3️⃣ System Parameters (Sentinel Config) ellenőrzése...")
|
||||
try:
|
||||
res = await session.execute(select(SystemParameter))
|
||||
params = res.scalars().all()
|
||||
if params:
|
||||
print(f" [✅ OK] Talált paraméterek: {len(params)} db")
|
||||
# Kritikus kulcsok, amiknek illik lenniük
|
||||
critical_keys = ["SECURITY_MAX_RECORDS_PER_HOUR", "VEHICLE_LIMIT"]
|
||||
existing_keys = [p.key for p in params]
|
||||
for ck in critical_keys:
|
||||
status = "✔️" if ck in existing_keys else "❌ (Hiányzik)"
|
||||
print(f" {status} {ck}")
|
||||
else:
|
||||
print(" [⚠️ FIGYELEM] A system_parameters tábla üres! Futtasd a seedert.")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] SystemParameter lekérdezési hiba: {e}")
|
||||
|
||||
# --- 4. i18n ÉS CACHE MOTOR ---
|
||||
print("\n4️⃣ Nyelvi motor és i18n Cache ellenőrzése...")
|
||||
try:
|
||||
# Gyorsítótár frissítése az adatbázisból
|
||||
await translation_service.load_cache(session)
|
||||
|
||||
test_key = "COMMON.SAVE"
|
||||
test_val = translation_service.get_text(test_key, "hu")
|
||||
|
||||
# Ha a visszakapott érték nem egyezik a kulccsal, akkor van fordítás
|
||||
if test_val and test_val != f"[{test_key}]":
|
||||
print(f" [✅ OK] Fordítás sikeres (HU): '{test_key}' -> '{test_val}'")
|
||||
else:
|
||||
print(f" [❌ HIBA] Fordítás nem működik. Nincs betöltött adat vagy hibás a cache.")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] Nyelvi motor hiba: {e}")
|
||||
|
||||
# --- 5. ROBOT ELŐKÉSZÜLETEK (Pipeline MDM) ---
|
||||
print("\n5️⃣ Robot Pipeline (MDM Staging) állapot...")
|
||||
try:
|
||||
# Hunter robot eredményei (nyers adatok)
|
||||
res_hunter = await session.execute(
|
||||
select(func.count(VehicleModelDefinition.id)).where(VehicleModelDefinition.status == 'unverified')
|
||||
)
|
||||
unverified_count = res_hunter.scalar() or 0
|
||||
|
||||
# Catalog robot eredményei (tisztított adatok)
|
||||
res_gold = await session.execute(select(func.count(AssetCatalog.id)))
|
||||
gold_count = res_gold.scalar() or 0
|
||||
|
||||
print(f" [📊 STAT] Feldolgozatlan rekordok (Staging): {unverified_count} db")
|
||||
print(f" [📊 STAT] Validált arany rekordok (Catalog): {gold_count} db")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] Robot-statisztika hiba: {e}")
|
||||
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
print("\n" + "═"*70)
|
||||
print(f"🏁 DIAGNOSZTIKA BEFEJEZŐDÖTT | Időtartam: {duration:.2f} mp")
|
||||
print("═"*70 + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(diagnose())
|
||||
except KeyboardInterrupt:
|
||||
print("\n🛑 Diagnosztika megszakítva a felhasználó által.")
|
||||
sys.exit(0)
|
||||
@@ -1,82 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/fixes/final_admin_fix.py
|
||||
import asyncio
|
||||
import uuid
|
||||
from sqlalchemy import text, select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User, Person, UserRole
|
||||
from app.core.security import get_password_hash
|
||||
|
||||
async def run_fix():
|
||||
print("\n" + "═"*50)
|
||||
print("🛠️ ADMIN RENDSZERJAVÍTÁS ÉS INICIALIZÁLÁS (MB2.0)")
|
||||
print("═"*50)
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 1. LOGIKA: Séma ellenőrzése az 'identity' névtérben
|
||||
# Az MB2.0-ban a felhasználók már nem a 'data', hanem az 'identity' sémában vannak.
|
||||
check_query = text("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = 'identity' AND table_name = 'users'
|
||||
""")
|
||||
res = await db.execute(check_query)
|
||||
cols = [r[0] for r in res.fetchall()]
|
||||
|
||||
if not cols:
|
||||
print("❌ HIBA: Az 'identity.users' tábla nem található. Futtasd az Alembic migrációt!")
|
||||
return
|
||||
|
||||
if "hashed_password" not in cols:
|
||||
print("❌ HIBA: A 'hashed_password' oszlop hiányzik. Az adatbázis sémája elavult.")
|
||||
return
|
||||
|
||||
# 2. LOGIKA: Admin keresése
|
||||
admin_email = "admin@profibot.hu"
|
||||
stmt = select(User).where(User.email == admin_email)
|
||||
existing_res = await db.execute(stmt)
|
||||
existing_admin = existing_res.scalar_one_or_none()
|
||||
|
||||
if existing_admin:
|
||||
print(f"⚠️ Információ: A(z) {admin_email} felhasználó már létezik.")
|
||||
# Opcionális: Jelszó kényszerített frissítése, ha elfelejtetted
|
||||
# existing_admin.hashed_password = get_password_hash("Admin123!")
|
||||
# await db.commit()
|
||||
else:
|
||||
try:
|
||||
# 3. LOGIKA: Person és User létrehozása (MB2.0 Standard)
|
||||
# Előbb létrehozzuk a fizikai személyt
|
||||
new_person = Person(
|
||||
id_uuid=uuid.uuid4(),
|
||||
first_name="Rendszer",
|
||||
last_name="Adminisztrátor",
|
||||
is_active=True
|
||||
)
|
||||
db.add(new_person)
|
||||
await db.flush() # ID lekérése a mentés előtt
|
||||
|
||||
# Létrehozzuk a felhasználói fiókot az Admin role-al
|
||||
new_admin = User(
|
||||
email=admin_email,
|
||||
hashed_password=get_password_hash("Admin123!"),
|
||||
person_id=new_person.id,
|
||||
role=UserRole.superadmin, # MB2.0 enum érték
|
||||
is_active=True,
|
||||
is_deleted=False,
|
||||
preferred_language="hu"
|
||||
)
|
||||
db.add(new_admin)
|
||||
|
||||
await db.commit()
|
||||
print(f"✅ SIKER: Superadmin létrehozva!")
|
||||
print(f" 📧 Email: {admin_email}")
|
||||
print(f" 🔑 Jelszó: Admin123!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ HIBA a mentés során: {e}")
|
||||
await db.rollback()
|
||||
|
||||
print("\n" + "═"*50)
|
||||
print("🏁 JAVÍTÁSI FOLYAMAT BEFEJEZŐDÖTT")
|
||||
print("═"*50 + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_fix())
|
||||
@@ -1,105 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_catalog.py
|
||||
"""
|
||||
🌱 MB2.0 KATALÓGUS ÉS ROBOT SEEDER
|
||||
----------------------------------
|
||||
CÉL: A járműkatalógus alapadatainak és a robotok munkalistájának feltöltése.
|
||||
|
||||
FUNKCIÓK:
|
||||
1. DiscoveryParameter: Azon városok rögzítése, ahol a Scout robot keresni fog.
|
||||
2. CatalogDiscovery: Azon márkák rögzítése, amiket a Robot 0/1 fel fog dolgozni.
|
||||
3. AssetCatalog: 'Arany' (már validált) technikai rekordok beszúrása.
|
||||
|
||||
JAVÍTÁSOK:
|
||||
- 'last_error' oszlop eltávolítva (mivel az adatbázisban nem létezik).
|
||||
- 'attempts' mező kényszerített 0 értékkel (NOT NULL kényszer miatt).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import AssetCatalog, CatalogDiscovery
|
||||
from app.models.marketplace.staged_data import DiscoveryParameter
|
||||
|
||||
# Logolás beállítása
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Seed: %(message)s')
|
||||
logger = logging.getLogger("Seed-Catalog")
|
||||
|
||||
async def quick_seed():
|
||||
""" Katalógus és Discovery adatok inicializálása. """
|
||||
async with AsyncSessionLocal() as db:
|
||||
logger.info("🚀 Katalógus alapozás indítása...")
|
||||
|
||||
try:
|
||||
# 1. Felderítendő Városok (DiscoveryParameter)
|
||||
# A Scout robot ezekben a városokban kezdi meg a szervizek kutatását.
|
||||
cities = [
|
||||
("BUDAPEST", "HU"),
|
||||
("DEBRECEN", "HU"),
|
||||
("GYŐR", "HU"),
|
||||
("SZEGED", "HU")
|
||||
]
|
||||
|
||||
for city_name, country in cities:
|
||||
db.add(DiscoveryParameter(
|
||||
city=city_name,
|
||||
keyword=country,
|
||||
is_active=True
|
||||
))
|
||||
|
||||
# 2. Felderítendő Márkák és Típusok (CatalogDiscovery)
|
||||
# Ezeket a rekordokat fogja a Robot 0 és Robot 1 feldolgozni a háttérben.
|
||||
discovery_queue = [
|
||||
("SUZUKI", "ALL"),
|
||||
("TOYOTA", "ALL"),
|
||||
("HONDA", "ALL"),
|
||||
("SKODA", "ALL"),
|
||||
("YAMAHA", "ALL")
|
||||
]
|
||||
|
||||
# Use INSERT ... ON CONFLICT DO NOTHING to avoid duplicate key errors
|
||||
for m, mod in discovery_queue:
|
||||
stmt = insert(CatalogDiscovery).values(
|
||||
make=m,
|
||||
model=mod,
|
||||
status="pending",
|
||||
attempts=0,
|
||||
vehicle_class="car", # Default value
|
||||
market="GLOBAL", # Default value
|
||||
priority_score=0 # Default value
|
||||
)
|
||||
# Handle conflict on make+model+vehicle_class unique constraint
|
||||
stmt = stmt.on_conflict_do_nothing(index_elements=['make', 'model', 'vehicle_class'])
|
||||
await db.execute(stmt)
|
||||
|
||||
# 3. Arany rekordok (AssetCatalog / vehicle_catalog tábla)
|
||||
# Példa adatok, amik már átmentek a validációs folyamaton.
|
||||
gold_data = [
|
||||
AssetCatalog(
|
||||
make="SUZUKI",
|
||||
model="VITARA",
|
||||
generation="LY (2015-)",
|
||||
fuel_type="petrol",
|
||||
factory_data={"segment": "SUV", "origin": "Hungary"}
|
||||
),
|
||||
AssetCatalog(
|
||||
make="SKODA",
|
||||
model="OCTAVIA",
|
||||
generation="IV (2020-)",
|
||||
fuel_type="diesel",
|
||||
factory_data={"segment": "Sedan/Combi"}
|
||||
)
|
||||
]
|
||||
db.add_all(gold_data)
|
||||
|
||||
# Mentés végrehajtása
|
||||
await db.commit()
|
||||
logger.info("✨ Katalógus és Discovery paraméterek sikeresen rögzítve!")
|
||||
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(f"❌ HIBA a katalógus feltöltésekor: {e}")
|
||||
raise e
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(quick_seed())
|
||||
@@ -1,156 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_data.py
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import text, select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User, Person, UserRole
|
||||
from app.models import ServiceProvider, Vote, ModerationStatus, Competition
|
||||
from app.services.social_service import SocialService
|
||||
from app.core.security import get_password_hash
|
||||
|
||||
async def run_simulation():
|
||||
async with AsyncSessionLocal() as db:
|
||||
print("--- 1. TAKARÍTÁS (MB2.0 Séma-tisztítás) ---")
|
||||
# Szigorú sorrend a kényszerek miatt (Cascade)
|
||||
# Ellenőrizzük, mely táblák léteznek
|
||||
tables_to_check = [
|
||||
("identity.users", "users"),
|
||||
("identity.persons", "persons"),
|
||||
("marketplace.service_providers", "service_providers"),
|
||||
("marketplace.votes", "votes"),
|
||||
("system.competitions", "competitions")
|
||||
]
|
||||
|
||||
existing_tables = []
|
||||
for full_name, table_name in tables_to_check:
|
||||
try:
|
||||
result = await db.execute(text(f"SELECT 1 FROM information_schema.tables WHERE table_schema = '{full_name.split('.')[0]}' AND table_name = '{table_name}'"))
|
||||
if result.scalar() == 1:
|
||||
existing_tables.append(full_name)
|
||||
else:
|
||||
print(f"⚠️ {full_name} tábla nem létezik, kihagyva a törlést")
|
||||
except Exception:
|
||||
print(f"⚠️ {full_name} tábla nem létezik, kihagyva a törlést")
|
||||
|
||||
if existing_tables:
|
||||
tables_str = ", ".join(existing_tables)
|
||||
await db.execute(text(f"TRUNCATE {tables_str} RESTART IDENTITY CASCADE"))
|
||||
await db.commit()
|
||||
else:
|
||||
print("ℹ️ Nincs törlendő tábla")
|
||||
|
||||
print("\n--- 2. SZEREPLŐK LÉTREHOZÁSA (Person + User) ---")
|
||||
users_to_create = [
|
||||
("admin@test.com", "Adminisztrátor", UserRole.superadmin),
|
||||
("good@test.com", "Rendes Srác", UserRole.user),
|
||||
("bad@test.com", "Spammer Aladár", UserRole.user),
|
||||
("voter@test.com", "Szavazó Gép", UserRole.user)
|
||||
]
|
||||
|
||||
created_users = {}
|
||||
for email, name, role in users_to_create:
|
||||
name_parts = name.split()
|
||||
first_name = name_parts[0] if name_parts else "Unknown"
|
||||
last_name = name_parts[1] if len(name_parts) > 1 else "User"
|
||||
p = Person(id_uuid=uuid.uuid4(), first_name=first_name, last_name=last_name, is_active=True)
|
||||
db.add(p)
|
||||
await db.flush()
|
||||
|
||||
u = User(
|
||||
email=email,
|
||||
hashed_password=get_password_hash("test1234"),
|
||||
person_id=p.id,
|
||||
role=role,
|
||||
is_active=True
|
||||
)
|
||||
db.add(u)
|
||||
await db.flush()
|
||||
created_users[email] = u
|
||||
|
||||
await db.commit()
|
||||
|
||||
print("\n--- 3. VERSENY INDÍTÁSA ---")
|
||||
# Ellenőrizzük, hogy a competitions tábla létezik-e
|
||||
try:
|
||||
result = await db.execute(text("SELECT 1 FROM information_schema.tables WHERE table_schema = 'system' AND table_name = 'competitions'"))
|
||||
if result.scalar() == 1:
|
||||
race = Competition(
|
||||
name="Téli Szervizvadászat",
|
||||
start_date=datetime.now(timezone.utc) - timedelta(days=1),
|
||||
end_date=datetime.now(timezone.utc) + timedelta(days=30),
|
||||
is_active=True
|
||||
)
|
||||
db.add(race)
|
||||
await db.commit()
|
||||
print("✅ Verseny létrehozva")
|
||||
else:
|
||||
print("⚠️ system.competitions tábla nem létezik, kihagyva a verseny létrehozását")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Hiba a competitions tábla ellenőrzése közben: {e}, kihagyva a verseny létrehozását")
|
||||
|
||||
# Szereplők kiemelése a szimulációhoz
|
||||
good_user = created_users["good@test.com"]
|
||||
bad_user = created_users["bad@test.com"]
|
||||
voter = created_users["voter@test.com"]
|
||||
|
||||
# Ellenőrizzük, hogy a szükséges táblák léteznek-e a szociális szimulációhoz
|
||||
try:
|
||||
result = await db.execute(text("SELECT 1 FROM information_schema.tables WHERE table_schema = 'marketplace' AND table_name = 'service_providers'"))
|
||||
service_providers_exists = result.scalar() == 1
|
||||
|
||||
result = await db.execute(text("SELECT 1 FROM information_schema.tables WHERE table_schema = 'marketplace' AND table_name = 'votes'"))
|
||||
votes_exists = result.scalar() == 1
|
||||
|
||||
if service_providers_exists and votes_exists:
|
||||
print("\n--- 4. SZCENÁRIÓ A: POZITÍV VALIDÁCIÓ ---")
|
||||
# Rendes srác beküld egy szervizt
|
||||
shop = ServiceProvider(
|
||||
name="Profi Gumis",
|
||||
address="Budapest, Váci út 10.",
|
||||
added_by_user_id=good_user.id,
|
||||
status=ModerationStatus.pending
|
||||
)
|
||||
db.add(shop)
|
||||
await db.flush()
|
||||
|
||||
# Szavazatok szimulálása (SocialService használatával a pontszámítás miatt)
|
||||
print(f"Szavazás a '{shop.name}'-re...")
|
||||
# Szimulálunk 5 pozitív szavazatot különböző "virtuális" szavazóktól
|
||||
for _ in range(5):
|
||||
await SocialService.vote_for_provider(db, voter.id, shop.id, 1)
|
||||
|
||||
await db.refresh(good_user)
|
||||
print(f"Jó felhasználó hírneve: {good_user.reputation_score}")
|
||||
|
||||
print("\n--- 5. SZCENÁRIÓ B: AUTO-BAN (SPAM SZŰRÉS) ---")
|
||||
fake_shop = ServiceProvider(
|
||||
name="KAMU SZERVIZ",
|
||||
address="Nincs ilyen utca 0.",
|
||||
added_by_user_id=bad_user.id,
|
||||
status=ModerationStatus.pending
|
||||
)
|
||||
db.add(fake_shop)
|
||||
await db.flush()
|
||||
|
||||
# Leszavazás (Kell -3 a bukáshoz)
|
||||
print("Spam jelentése...")
|
||||
await SocialService.vote_for_provider(db, voter.id, fake_shop.id, -1)
|
||||
await SocialService.vote_for_provider(db, voter.id, fake_shop.id, -1)
|
||||
await SocialService.vote_for_provider(db, voter.id, fake_shop.id, -1)
|
||||
|
||||
await db.refresh(bad_user)
|
||||
print(f"Rossz felhasználó hírneve: {bad_user.reputation_score}")
|
||||
print(f"Fiók státusza: {'KITILTVA' if not bad_user.is_active else 'AKTÍV'}")
|
||||
|
||||
if not bad_user.is_active:
|
||||
print("✅ SIKER: A Sentinel automatikusan leállította a spammert!")
|
||||
else:
|
||||
print("\n⚠️ Marketplace táblák (service_providers, votes) nem léteznek, kihagyva a szociális szimulációt")
|
||||
print("ℹ️ Alap felhasználók sikeresen létrehozva")
|
||||
except Exception as e:
|
||||
print(f"\n⚠️ Hiba a táblák ellenőrzése közben: {e}, kihagyva a szociális szimulációt")
|
||||
print("ℹ️ Alap felhasználók sikeresen létrehozva")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_simulation())
|
||||
@@ -1,62 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Seed script az Economy 1 modulhoz: árfolyam paraméterek beszúrása a system.system_parameters táblába.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from decimal import Decimal
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.system import SystemParameter
|
||||
|
||||
|
||||
async def seed_economy():
|
||||
"""Árfolyam paraméterek beszúrása."""
|
||||
parameters = [
|
||||
{
|
||||
"key": "EXCHANGE_RATE_EUR_HUF",
|
||||
"value": "390.0",
|
||||
"description": "EUR/HUF átváltási árfolyam (1 EUR = X HUF)",
|
||||
"category": "finance",
|
||||
"is_active": True,
|
||||
},
|
||||
{
|
||||
"key": "EXCHANGE_RATE_USDC_HUF",
|
||||
"value": "380.0",
|
||||
"description": "USDC/HUF átváltási árfolyam (1 USDC = X HUF)",
|
||||
"category": "finance",
|
||||
"is_active": True,
|
||||
},
|
||||
]
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
for param in parameters:
|
||||
# Ellenőrizzük, hogy létezik-e már
|
||||
existing = await session.execute(
|
||||
select(SystemParameter).where(SystemParameter.key == param["key"])
|
||||
)
|
||||
existing = existing.scalar_one_or_none()
|
||||
if existing:
|
||||
print(f"⚠️ {param['key']} már létezik, kihagyva.")
|
||||
continue
|
||||
|
||||
new_param = SystemParameter(
|
||||
key=param["key"],
|
||||
value=param["value"],
|
||||
description=param["description"],
|
||||
category=param["category"],
|
||||
is_active=param["is_active"],
|
||||
)
|
||||
session.add(new_param)
|
||||
print(f"✅ {param['key']} beszúrva.")
|
||||
|
||||
await session.commit()
|
||||
print("🎉 Árfolyam paraméterek sikeresen seedelve.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_economy())
|
||||
@@ -1,65 +0,0 @@
|
||||
import asyncio
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.marketplace.service import ExpertiseTag
|
||||
from sqlalchemy import text
|
||||
|
||||
async def seed_expertises():
|
||||
tags = [
|
||||
# --- ALAPSZOLGÁLTATÁSOK (MECHANICS) ---
|
||||
('OIL_SERVICE', 'Időszakos szerviz / Olajcsere', 'MECHANICS'),
|
||||
('BRAKE_REPAIR', 'Fékrendszer javítás', 'MECHANICS'),
|
||||
('SUSPENSION', 'Futómű javítás és beállítás', 'MECHANICS'),
|
||||
('EXHAUST', 'Kipufogó szerviz', 'MECHANICS'),
|
||||
('CLUTCH', 'Kuplung és kettőstömegű csere', 'MECHANICS'),
|
||||
|
||||
# --- MOTOR ÉS VÁLTÓ (ENGINE_DRIVETRAIN) ---
|
||||
('ENGINE_REBUILD', 'Motorfelújítás', 'ENGINE_DRIVETRAIN'),
|
||||
('TIMING_BELT', 'Vezérlés csere', 'ENGINE_DRIVETRAIN'),
|
||||
('AUTO_GEARBOX', 'Automata váltó javítás/olajcsere', 'ENGINE_DRIVETRAIN'),
|
||||
('TURBO_REPAIR', 'Turbófeltöltő felújítás', 'ENGINE_DRIVETRAIN'),
|
||||
('INJECTOR', 'Dízel injektor / Adagoló javítás', 'ENGINE_DRIVETRAIN'),
|
||||
('DPF_CLEAN', 'DPF / Részecskeszűrő tisztítás', 'ENGINE_DRIVETRAIN'),
|
||||
|
||||
# --- ELEKTRONIKA (ELECTRICAL) ---
|
||||
('DIAGNOSTICS', 'Számítógépes diagnosztika', 'ELECTRICAL'),
|
||||
('AC_REPAIR', 'Klíma javítás és töltés', 'ELECTRICAL'),
|
||||
('BATTERY', 'Akkumulátor szerviz', 'ELECTRICAL'),
|
||||
('HYBRID_EV', 'Hibrid és Elektromos autó szerviz', 'ELECTRICAL'),
|
||||
('CHIP_TUNING', 'Szoftveres optimalizálás / Tuning', 'ELECTRICAL'),
|
||||
('ADAS', 'Vezetéstámogató rendszerek kalibrálása', 'ELECTRICAL'),
|
||||
|
||||
# --- GUMI ÉS KERÉK (TYRES) ---
|
||||
('TYRE_CHANGE', 'Gumiszerelés és centírozás', 'TYRES'),
|
||||
('WHEEL_REPAIR', 'Alufelni javítás / Görgőzés', 'TYRES'),
|
||||
|
||||
# --- KAROSSZÉRIA (BODY) ---
|
||||
('BODY_REPAIR', 'Karosszéria lakatolás', 'BODY'),
|
||||
('PAINTING', 'Autófényezés', 'BODY'),
|
||||
('GLASS_REPAIR', 'Szélvédő javítás és csere', 'BODY'),
|
||||
('PDR', 'Jégkár és horpadásjavítás (PDR)', 'BODY'),
|
||||
|
||||
# --- SEGÉLY ÉS SZÁLLÍTÁS (EMERGENCY) ---
|
||||
('TOWING', 'Autómentés / Vontatás', 'EMERGENCY'),
|
||||
('ROADSIDE_ASSIST', 'Segélyszolgálat / Helyszíni javítás', 'EMERGENCY'),
|
||||
('LOCKSMITH', 'Autózár szerviz / Kulcsmásolás', 'EMERGENCY'),
|
||||
|
||||
# --- EGYÉB JÁRMŰVEK (VEHICLE_TYPES) ---
|
||||
('MOTO_SERVICE', 'Motorkerékpár szerviz', 'VEHICLE_TYPES'),
|
||||
('TRUCK_SERVICE', 'Tehergépjármű szerviz', 'VEHICLE_TYPES'),
|
||||
('AGRI_SERVICE', 'Mezőgazdasági gép szerviz', 'VEHICLE_TYPES'),
|
||||
]
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
print("🌱 Szakmai címkék feltöltése...")
|
||||
for key, name, cat in tags:
|
||||
stmt = text("""
|
||||
INSERT INTO marketplace.expertise_tags (key, name_hu, category, is_official)
|
||||
VALUES (:k, :n, :c, true)
|
||||
ON CONFLICT (key) DO UPDATE SET name_hu = EXCLUDED.name_hu, category = EXCLUDED.category
|
||||
""")
|
||||
await db.execute(stmt, {"k": key, "n": name, "c": cat})
|
||||
await db.commit()
|
||||
print(f"✅ {len(tags)} címke rögzítve.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_expertises())
|
||||
@@ -1,74 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_honda.py
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import AssetCatalog
|
||||
from app.models.marketplace.staged_data import DiscoveryParameter
|
||||
|
||||
# Logolás beállítása
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Seed: %(message)s')
|
||||
logger = logging.getLogger("Honda-Seeder")
|
||||
|
||||
async def seed_honda():
|
||||
"""
|
||||
Honda specifikus alapozás az MB2.0 MDM (Master Data Management) szerint.
|
||||
Létrehozza a katalógus-vázat és a robot-feladatokat.
|
||||
"""
|
||||
async with AsyncSessionLocal() as db:
|
||||
logger.info("🚀 Honda márka-ökoszisztéma inicializálása...")
|
||||
|
||||
# 1. LOGIKA: Robot Discovery feladatok rögzítése
|
||||
# Ezzel mondjuk meg a Hunter robotnak, hogy keressen rá minden Honda variánsra
|
||||
discovery_tasks = [
|
||||
DiscoveryParameter(make="HONDA", vehicle_class="car", city="BUDAPEST", keyword="repair", is_active=True),
|
||||
DiscoveryParameter(make="HONDA", vehicle_class="motorcycle", city="BUDAPEST", keyword="service", is_active=True)
|
||||
]
|
||||
|
||||
for task in discovery_tasks:
|
||||
# Megnézzük, van-e már ilyen feladat
|
||||
stmt = select(DiscoveryParameter).where(
|
||||
DiscoveryParameter.make == task.make,
|
||||
DiscoveryParameter.vehicle_class == task.vehicle_class
|
||||
)
|
||||
exists = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if not exists:
|
||||
db.add(task)
|
||||
|
||||
# 2. LOGIKA: Népszerű modellek (Arany Rekordok) betöltése
|
||||
# Ezek a "Starter" adatok, amik azonnal elérhetők a felhasználóknak
|
||||
honda_models = [
|
||||
# Személyautók
|
||||
{"model": "CIVIC", "gen": "X (2015-2021)", "class": "car"},
|
||||
{"model": "ACCORD", "gen": "X (2017-)", "class": "car"},
|
||||
{"model": "CR-V", "gen": "V (2016-)", "class": "car"},
|
||||
{"model": "JAZZ", "gen": "IV (2020-)", "class": "car"},
|
||||
# Motorkerékpárok
|
||||
{"model": "CB500X", "gen": "PC64 (2019-)", "class": "motorcycle"},
|
||||
{"model": "AFRICA TWIN", "gen": "CRF1100L", "class": "motorcycle"},
|
||||
{"model": "NC750X", "gen": "RH09 (2021-)", "class": "motorcycle"}
|
||||
]
|
||||
|
||||
for m in honda_models:
|
||||
# Ellenőrizzük az AssetCatalog-ban (MDM tábla)
|
||||
stmt = select(AssetCatalog).where(
|
||||
AssetCatalog.make == "HONDA",
|
||||
AssetCatalog.model == m["model"],
|
||||
AssetCatalog.generation == m["gen"]
|
||||
)
|
||||
exists = (await db.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
if not exists:
|
||||
db.add(AssetCatalog(
|
||||
make="HONDA",
|
||||
model=m["model"],
|
||||
generation=m["gen"],
|
||||
vehicle_class=m["class"],
|
||||
factory_data={"source": "manual_priority_seed"} # MDM metaadat
|
||||
))
|
||||
|
||||
await db.commit()
|
||||
logger.info("✅ Honda (Autó & Motor) katalógus váz sikeresen felépítve!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_honda())
|
||||
@@ -1,79 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_system.py
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User, Person, UserRole
|
||||
from app.models.system import SystemParameter
|
||||
# JAVÍTOTT IMPORTOK: A grep alapján szétválasztva
|
||||
from app.models import PointRule, LevelConfig, UserStats
|
||||
from app.models.core_logic import SubscriptionTier
|
||||
from app.core.security import get_password_hash
|
||||
from app.core.config import settings
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Seed: %(message)s')
|
||||
logger = logging.getLogger("System-Seeder")
|
||||
|
||||
async def seed_data():
|
||||
""" Rendszer alapadatok inicializálása a megfelelő modellekből. """
|
||||
async with AsyncSessionLocal() as db:
|
||||
logger.info("🚀 Rendszer-alapozás indítása (MB2.0 Standard)...")
|
||||
|
||||
admin_email = settings.INITIAL_ADMIN_EMAIL
|
||||
admin_password = settings.INITIAL_ADMIN_PASSWORD
|
||||
|
||||
if not admin_email or not admin_password:
|
||||
logger.error("❌ HIBA: Admin hitelesítési adatok hiányoznak!")
|
||||
return
|
||||
|
||||
# 1. Superadmin és Person kapcsolat
|
||||
stmt = select(User).where(User.email == admin_email)
|
||||
admin_exists = (await db.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
if not admin_exists:
|
||||
new_person = Person(
|
||||
first_name="Rendszer",
|
||||
last_name="Adminisztrátor",
|
||||
id_uuid=uuid.uuid4(),
|
||||
is_active=True
|
||||
)
|
||||
db.add(new_person)
|
||||
await db.flush()
|
||||
|
||||
new_admin = User(
|
||||
email=admin_email,
|
||||
hashed_password=get_password_hash(admin_password),
|
||||
role=UserRole.superadmin,
|
||||
is_active=True,
|
||||
person_id=new_person.id
|
||||
)
|
||||
db.add(new_admin)
|
||||
await db.flush()
|
||||
|
||||
# Statisztikai rekord (Gamification)
|
||||
db.add(UserStats(user_id=new_admin.id, total_xp=0, current_level=1))
|
||||
logger.info(f"✅ Superadmin létrehozva: {admin_email}")
|
||||
|
||||
# 2. Rendszerparaméterek (JSONB értékekkel)
|
||||
params = [
|
||||
("SECURITY_MAX_RECORDS_PER_HOUR", {"limit": 50}, "Biztonsági limit"),
|
||||
("VEHICLE_LIMIT", {"default": 5}, "Alapértelmezett jármű limit")
|
||||
]
|
||||
for key, val, desc in params:
|
||||
stmt_p = select(SystemParameter).where(SystemParameter.key == key)
|
||||
if not (await db.execute(stmt_p)).scalar_one_or_none():
|
||||
db.add(SystemParameter(key=key, value=val, description=desc))
|
||||
|
||||
# 3. Gamification Szabályok (gamification.py-ból)
|
||||
rules = [("ASSET_REGISTER", 100), ("ASSET_REVIEW", 75)]
|
||||
for key, pts in rules:
|
||||
stmt_r = select(PointRule).where(PointRule.action_key == key)
|
||||
if not (await db.execute(stmt_r)).scalar_one_or_none():
|
||||
db.add(PointRule(action_key=key, points=pts))
|
||||
|
||||
await db.commit()
|
||||
logger.info("✨ Rendszer alapadatok rögzítve.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_data())
|
||||
@@ -1,123 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
TCO (Total Cost of Ownership) alap költségkategóriák seedelése.
|
||||
Rendszerszintű kategóriák (is_system=True) amelyek nem törölhetők.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
# A projekt gyökérből importáljuk a database modult
|
||||
sys.path.insert(0, '/opt/docker/dev/service_finder/backend')
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle import CostCategory
|
||||
|
||||
|
||||
# A 10 alap TCO kategória definíciója
|
||||
SYSTEM_CATEGORIES = [
|
||||
{
|
||||
"code": "FUEL",
|
||||
"name": "Üzemanyag / Töltés",
|
||||
"description": "Benzin, dízel, elektromos töltés, LPG, hidrogén"
|
||||
},
|
||||
{
|
||||
"code": "MAINTENANCE",
|
||||
"name": "Szerviz & Karbantartás",
|
||||
"description": "Olajcsere, szűrők, fékbetét, futómű, egyéb szerviz munkák"
|
||||
},
|
||||
{
|
||||
"code": "TIRES",
|
||||
"name": "Gumiabroncsok",
|
||||
"description": "Nyári/téli gumik, felni, kiegyensúlyozás, gumicsere"
|
||||
},
|
||||
{
|
||||
"code": "INSURANCE",
|
||||
"name": "Biztosítás",
|
||||
"description": "KASCO, kötelező gépjármű-felelősségbiztosítás, casco, utasbiztosítás"
|
||||
},
|
||||
{
|
||||
"code": "TAX",
|
||||
"name": "Adók",
|
||||
"description": "Gépjárműadó, forgalmi adó, közlekedési adó"
|
||||
},
|
||||
{
|
||||
"code": "FEES",
|
||||
"name": "Útdíj & Parkolás",
|
||||
"description": "Autópálya matrica, parkolási díjak, városi belépési díjak"
|
||||
},
|
||||
{
|
||||
"code": "ADMIN",
|
||||
"name": "Hatósági díjak",
|
||||
"description": "Műszaki vizsga, forgalmi engedély, okmányok, adminisztratív költségek"
|
||||
},
|
||||
{
|
||||
"code": "FINANCE",
|
||||
"name": "Finanszírozás",
|
||||
"description": "Lízing díj, hiteltörlesztés, kamatok, banki költségek"
|
||||
},
|
||||
{
|
||||
"code": "CLEANING",
|
||||
"name": "Ápolás & Kozmetika",
|
||||
"description": "Autómosás, polírozás, belső tisztítás, festékvédelem"
|
||||
},
|
||||
{
|
||||
"code": "OTHER",
|
||||
"name": "Egyéb",
|
||||
"description": "Egyéb, nem besorolható költségek"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def seed_tco_categories():
|
||||
"""
|
||||
Törli a meglévő kategóriákat és beszúrja a 10 rendszerszintű TCO kategóriát.
|
||||
"""
|
||||
print("🚀 TCO költségkategóriák seedelése...")
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# 1. Tábla ürítése (TRUNCATE) - csak a seed kategóriák, ne érintse a felhasználói kategóriákat?
|
||||
# Mivel most csak rendszerszintűek vannak, töröljük az összeset
|
||||
print(" ↳ Tábla ürítése (TRUNCATE vehicle.cost_categories)...")
|
||||
await session.execute(text("TRUNCATE TABLE vehicle.cost_categories RESTART IDENTITY CASCADE"))
|
||||
await session.commit()
|
||||
|
||||
# 2. Kategóriák beszúrása
|
||||
inserted = 0
|
||||
for cat_data in SYSTEM_CATEGORIES:
|
||||
category = CostCategory(
|
||||
code=cat_data["code"],
|
||||
name=cat_data["name"],
|
||||
description=cat_data["description"],
|
||||
is_system=True,
|
||||
parent_id=None # Jelenleg nincs hierarchia, később bővíthető
|
||||
)
|
||||
session.add(category)
|
||||
inserted += 1
|
||||
|
||||
await session.commit()
|
||||
print(f" ✅ {inserted} rendszerszintű kategória beszúrva.")
|
||||
|
||||
# 3. Ellenőrzés
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM vehicle.cost_categories"))
|
||||
count = result.scalar()
|
||||
print(f" 📊 vehicle.cost_categories táblában jelenleg {count} sor van.")
|
||||
|
||||
# Listázás
|
||||
result = await session.execute(text("SELECT code, name FROM vehicle.cost_categories ORDER BY code"))
|
||||
rows = result.fetchall()
|
||||
print(" 📋 Kategóriák listája:")
|
||||
for code, name in rows:
|
||||
print(f" - {code}: {name}")
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
print(f" ❌ Hiba történt: {e}")
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_tco_categories())
|
||||
print("🎉 TCO kategória seedelés sikeresen befejeződött.")
|
||||
@@ -1,120 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_test_scenario.py
|
||||
import asyncio
|
||||
import uuid
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User
|
||||
from app.models.marketplace.organization import Organization, OrganizationMember, OrgType
|
||||
from app.models import (
|
||||
Asset, AssetCatalog, AssetTelemetry,
|
||||
AssetFinancials, AssetCost
|
||||
)
|
||||
|
||||
# Sentinel naplózás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Scenario: %(message)s')
|
||||
logger = logging.getLogger("Test-Scenario")
|
||||
|
||||
async def seed_test_scenario():
|
||||
async with AsyncSessionLocal() as db:
|
||||
logger.info("🚀 MB2.0 Teszt ökoszisztéma felépítése indul...")
|
||||
|
||||
# 1. LOGIKA: Admin (Superuser) lekérése az identity sémából
|
||||
res = await db.execute(select(User).where(User.is_active == True))
|
||||
admin = res.scalars().first()
|
||||
|
||||
if not admin:
|
||||
logger.error("❌ Hiba: Nincs aktív felhasználó a rendszerben. Futtasd a seed_system.py-t!")
|
||||
return
|
||||
|
||||
# 2. LOGIKA: Szervezeti struktúra felállítása
|
||||
# Privát garázs
|
||||
private_org = Organization(
|
||||
name="Kincses Privát",
|
||||
full_name="Kincses Magánflotta és Garázs",
|
||||
org_type=OrgType.individual,
|
||||
owner_id=admin.id,
|
||||
folder_slug="kincses-privat-vault"
|
||||
)
|
||||
# Üzleti flotta
|
||||
company_org = Organization(
|
||||
name="ProfiBot Fleet",
|
||||
full_name="ProfiBot Software Solutions Kft.",
|
||||
org_type=OrgType.business,
|
||||
owner_id=admin.id,
|
||||
folder_slug="profibot-fleet-vault"
|
||||
)
|
||||
# Szolgáltatók (Szerviz és Üzemanyag)
|
||||
service_org = Organization(
|
||||
name="Mester Szerviz",
|
||||
org_type=OrgType.service,
|
||||
owner_id=admin.id,
|
||||
is_active=True
|
||||
)
|
||||
|
||||
db.add_all([private_org, company_org, service_org])
|
||||
await db.flush()
|
||||
|
||||
# Tagsági viszonyok rögzítése
|
||||
db.add(OrganizationMember(user_id=admin.id, organization_id=company_org.id, role="owner"))
|
||||
|
||||
# 3. LOGIKA: Tesla Model 3 - Digitális Iker (Digital Twin)
|
||||
# Előbb a katalógus (Gold Data)
|
||||
catalog = AssetCatalog(
|
||||
make="TESLA", model="MODEL 3", generation="Long Range (2021-)",
|
||||
fuel_type="electric",
|
||||
factory_data={
|
||||
"battery": "75 kWh", "power_kw": 366,
|
||||
"tire_size": "235/45 R18", "ac_charge": "11kW"
|
||||
}
|
||||
)
|
||||
db.add(catalog)
|
||||
await db.flush()
|
||||
|
||||
# Majd a konkrét jármű (Asset)
|
||||
vehicle = Asset(
|
||||
vin=f"5YJ3E1EB8LF{uuid.uuid4().hex[:6].upper()}",
|
||||
license_plate="TES-777-EV",
|
||||
name="Céges Tesla",
|
||||
year_of_manufacture=2021,
|
||||
catalog_id=catalog.id,
|
||||
owner_org_id=company_org.id,
|
||||
status="active"
|
||||
)
|
||||
db.add(vehicle)
|
||||
await db.flush()
|
||||
|
||||
# Telemetria és Pénzügyi alapok
|
||||
db.add(AssetTelemetry(asset_id=vehicle.id, current_mileage=45200, vqi_score=100.0))
|
||||
db.add(AssetFinancials(asset_id=vehicle.id, acquisition_price=18500000, currency="HUF"))
|
||||
|
||||
# 4. LOGIKA: A 9 költségtípus szimulálása
|
||||
costs_data = [
|
||||
("FUEL", 12500, "Supercharger töltés"),
|
||||
("MAINTENANCE", 85000, "Pollenszűrő és átvizsgálás"),
|
||||
("TIRES", 280000, "Téli gumi szett"),
|
||||
("INSURANCE", 32000, "Havi CASCO"),
|
||||
("TAX", 15000, "Cégautóadó (szimulált)"),
|
||||
("TOLL", 6500, "Éves matrica"),
|
||||
("CLEANING", 4500, "Külső-belső takarítás"),
|
||||
("PARKING", 1200, "Belvárosi zóna"),
|
||||
("OTHER", 2500, "Szélvédőmosó folyadék")
|
||||
]
|
||||
|
||||
for c_type, amount, desc in costs_data:
|
||||
db.add(AssetCost(
|
||||
asset_id=vehicle.id,
|
||||
organization_id=company_org.id,
|
||||
cost_type=c_type,
|
||||
amount=amount,
|
||||
currency="HUF",
|
||||
date=datetime.now(timezone.utc) - timedelta(days=2),
|
||||
specifications={"description": desc}
|
||||
))
|
||||
|
||||
await db.commit()
|
||||
logger.info("✅ Siker! A teljes flotta-ökoszisztéma üzemkész.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_test_scenario())
|
||||
@@ -1,77 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Operational test for Analytics API endpoint /api/v1/analytics/{vehicle_id}/summary
|
||||
Verifies that the endpoint is correctly registered, accepts UUID vehicle_id,
|
||||
and returns appropriate HTTP status (not 500 internal server error).
|
||||
Uses dev_bypass_active token to bypass authentication (requires DEBUG=True).
|
||||
"""
|
||||
import sys
|
||||
import asyncio
|
||||
import httpx
|
||||
import uuid
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_BASE = "http://localhost:8000"
|
||||
DEV_TOKEN = "dev_bypass_active"
|
||||
|
||||
async def test_analytics_summary():
|
||||
"""Test that the endpoint is reachable and handles UUID parameter."""
|
||||
# Generate a random UUID (vehicle likely does not exist)
|
||||
vehicle_id = uuid.uuid4()
|
||||
url = f"{API_BASE}/api/v1/analytics/{vehicle_id}/summary"
|
||||
headers = {"Authorization": f"Bearer {DEV_TOKEN}"}
|
||||
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
try:
|
||||
resp = await client.get(url, headers=headers)
|
||||
status = resp.status_code
|
||||
body = resp.text
|
||||
logger.info(f"Response status: {status}")
|
||||
logger.debug(f"Response body: {body}")
|
||||
|
||||
# If endpoint missing, we'd get 404 Not Found (from router).
|
||||
# However, with UUID parameter, the router is matched, so 404 is vehicle not found.
|
||||
# Distinguish by checking if the response indicates router-level 404 (maybe generic).
|
||||
# For simplicity, we assume any 404 means vehicle not found, which is OK.
|
||||
# The critical check: no 500 Internal Server Error (mapper or runtime errors).
|
||||
if status == 500:
|
||||
raise AssertionError(f"Internal server error: {body}")
|
||||
|
||||
# If we get 200, validate JSON structure (optional, but we don't have data).
|
||||
if status == 200:
|
||||
data = resp.json()
|
||||
required_keys = {"vehicle_id", "user_tco", "lifetime_tco", "benchmark_tco", "stats"}
|
||||
missing = required_keys - set(data.keys())
|
||||
if missing:
|
||||
raise AssertionError(f"Missing keys in response: {missing}")
|
||||
for key in ["user_tco", "lifetime_tco", "benchmark_tco"]:
|
||||
if not isinstance(data[key], list):
|
||||
raise AssertionError(f"{key} is not a list")
|
||||
logger.info("✅ Analytics endpoint works and returns expected structure.")
|
||||
return True
|
||||
|
||||
# Any other status (404, 422, 403, 401) indicates the endpoint is reachable
|
||||
# and the request was processed (no router error).
|
||||
logger.info(f"Endpoint responded with {status} (expected, vehicle not found or access denied).")
|
||||
return True
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"HTTP client error: {e}")
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
logger.error("Request timeout")
|
||||
raise
|
||||
|
||||
async def main():
|
||||
try:
|
||||
await test_analytics_summary()
|
||||
print("\n✅ Analytics API test passed (endpoint is reachable and accepts UUID).")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"\n❌ Analytics API test failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,31 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/test_functional.py
|
||||
"""
|
||||
CÉL: Éles funkcionális teszt a bejelentkezési folyamathoz.
|
||||
"""
|
||||
import asyncio
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User
|
||||
from app.services.auth_service import AuthService
|
||||
|
||||
async def test_login_flow():
|
||||
print("\n--- 🔑 FUNKCIONÁLIS LOGIN TESZT ---")
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 1. Keressünk egy teszt felhasználót
|
||||
result = await db.execute(select(User).limit(1))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if not user:
|
||||
print("❌ HIBA: Nincs felhasználó az adatbázisban. Futtass egy seeder-t!")
|
||||
return
|
||||
|
||||
print(f"👤 Tesztelés felhasználóval: {user.email}")
|
||||
|
||||
# 2. Próbáljunk meg egy 'hitelesítést' (jelszó ellenőrzés nélkül a DB szinten)
|
||||
if user.is_active:
|
||||
print(f"✅ SIKER: A(z) {user.email} fiók aktív és elérhető.")
|
||||
else:
|
||||
print(f"⚠️ FIGYELEM: A felhasználó létezik, de inaktív.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_login_flow())
|
||||
@@ -1,91 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/test_gamification_flow.py
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from sqlalchemy import select
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Környezeti változók betöltése
|
||||
load_dotenv()
|
||||
|
||||
# MB2.0 Importok
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User
|
||||
from app.models import UserStats, PointsLedger
|
||||
from app.services.social_service import SocialService
|
||||
from app.schemas.social import ServiceProviderCreate
|
||||
|
||||
# Naplózás beállítása
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Test: %(message)s')
|
||||
logger = logging.getLogger("Gamification-Test")
|
||||
|
||||
async def run_test():
|
||||
logger.info("🚀 Gamifikációs integrációs folyamat tesztelése...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
# 1. LOGIKA: Teszt felhasználó lekérése az identity sémából
|
||||
result = await db.execute(select(User).limit(1))
|
||||
user = result.scalars().first()
|
||||
|
||||
if not user:
|
||||
logger.error("❌ Hiba: Nincs felhasználó az adatbázisban. Futtasd a seed_system.py-t!")
|
||||
return
|
||||
|
||||
logger.info(f"👤 Aktív teszt alany: {user.email}")
|
||||
|
||||
# 2. LOGIKA: Új szolgáltató rögzítése (Trigger az XP szerzéshez)
|
||||
# A SocialService.create_service_provider automatikusan hívja a GamificationService-t
|
||||
unique_id = os.urandom(2).hex()
|
||||
test_provider = ServiceProviderCreate(
|
||||
name=f"Robot Szerviz {unique_id}",
|
||||
address="Alchemist utca 12.",
|
||||
category="service"
|
||||
)
|
||||
|
||||
logger.info(f"🛠️ Esemény kiváltása: '{test_provider.name}' rögzítése...")
|
||||
new_provider = await SocialService.create_service_provider(db, test_provider, user.id)
|
||||
|
||||
# Commit kényszerítése, hogy a háttérfolyamatok rögzüljenek
|
||||
await db.commit()
|
||||
logger.info(f"✅ Szolgáltató elfogadva (ID: {new_provider.id})")
|
||||
|
||||
# 3. LOGIKA: Eredmények ellenőrzése a Ledgerben (Főkönyv)
|
||||
# Újra lekérjük a statisztikákat a commit után
|
||||
stats_res = await db.execute(select(UserStats).where(UserStats.user_id == user.id))
|
||||
stats = stats_res.scalar_one_or_none()
|
||||
|
||||
ledger_res = await db.execute(
|
||||
select(PointsLedger)
|
||||
.where(PointsLedger.user_id == user.id)
|
||||
.order_by(PointsLedger.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last_entry = ledger_res.scalars().first()
|
||||
|
||||
print("\n" + "═"*40)
|
||||
print("📊 INTEGRÁCIÓS JELENTÉS:")
|
||||
if stats:
|
||||
print(f"🏆 Aktuális XP: {stats.total_xp}")
|
||||
print(f"📈 Szint: {stats.current_level}")
|
||||
else:
|
||||
print("⚠️ UserStats rekord nem található!")
|
||||
|
||||
if last_entry:
|
||||
print(f"📝 Tranzakció oka: {last_entry.reason}")
|
||||
print(f"💰 XP változás: +{last_entry.points_change}")
|
||||
print("═"*40 + "\n")
|
||||
|
||||
if stats and stats.total_xp > 0:
|
||||
logger.info("✅ SIKER: A gamifikációs lánc éles és működik!")
|
||||
else:
|
||||
logger.warning("❌ HIBA: A pontszámítás nem történt meg.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💥 Kritikus hiba a teszt közben: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_test())
|
||||
@@ -1,33 +0,0 @@
|
||||
# app/tests_internal/test_postgis.py
|
||||
import asyncio
|
||||
from sqlalchemy import text
|
||||
from app.db.session import AsyncSessionLocal
|
||||
|
||||
async def test_geo_logic():
|
||||
"""
|
||||
THOUGHT PROCESS:
|
||||
Ellenőrizni kell, hogy a PostgreSQL-ben a 'fleet.branches' tábla 'location' oszlopa
|
||||
valóban GEOGRAPHY típusú-e, és az ST_Distance függvény működik-e.
|
||||
Ha ez elbukik, a 'search.py' nem fog eredményt adni.
|
||||
"""
|
||||
print("🌍 PostGIS távolságszámítás tesztelése...")
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# Egy teszt pont (Budapest központ) és egy körzet lekérdezése
|
||||
query = text("""
|
||||
SELECT id, name,
|
||||
ST_Distance(location, ST_SetSRID(ST_MakePoint(19.0402, 47.4979), 4326)::geography) / 1000 as distance_km
|
||||
FROM fleet.branches
|
||||
LIMIT 1
|
||||
""")
|
||||
result = await db.execute(query)
|
||||
row = result.fetchone()
|
||||
if row:
|
||||
print(f"✅ SIKER: Találtunk egy ágat ({row.name}) {row.distance_km:.2f} km távolságra.")
|
||||
else:
|
||||
print("⚠️ FIGYELEM: A lekérdezés lefutott, de nincsenek adatok a fleet.branches táblában.")
|
||||
except Exception as e:
|
||||
print(f"❌ HIBA: A PostGIS lekérdezés elbukott. Oka: {str(e)}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_geo_logic())
|
||||
@@ -1,340 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Financial Truth Verification - Epic 3 Pénzügyi Motor "Végső Boss" teszt.
|
||||
|
||||
Ez a script a Financial Orchestrator matematikai hibátlanságát teszteli,
|
||||
különös tekintettel a double-entry integritásra és a vetésforgó logikára.
|
||||
|
||||
FIGYELEM: A teszt NEM módosítja tartósan az éles adatbázist!
|
||||
Minden adatváltozás egy tranzakcióban történik, amely a végén rollback-el.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
from decimal import Decimal
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
# Add backend directory to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
|
||||
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import select, func, text
|
||||
|
||||
from app.database import Base
|
||||
from app.models.identity import User, Person, Wallet
|
||||
from app.models.marketplace.finance import Issuer, IssuerType
|
||||
from app.models import WalletType
|
||||
from app.models import FinancialLedger, LedgerEntryType
|
||||
from app.services.financial_orchestrator import FinancialOrchestrator
|
||||
from app.core.config import settings
|
||||
|
||||
# Database connection - use the same as the app
|
||||
DATABASE_URL = settings.DATABASE_URL.replace("postgresql://", "postgresql+asyncpg://")
|
||||
engine = create_async_engine(DATABASE_URL, echo=False)
|
||||
AsyncSessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
|
||||
class FinancialTruthTest:
|
||||
def __init__(self):
|
||||
self.session = None
|
||||
self.test_user = None
|
||||
self.test_wallet = None
|
||||
self.ev_issuer = None
|
||||
self.kft_issuer = None
|
||||
self.orchestrator = FinancialOrchestrator()
|
||||
self.created_ledgers = []
|
||||
self.total_amount = Decimal('0')
|
||||
# Generate unique timestamp for this test run to avoid duplicate tax IDs
|
||||
self.test_timestamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
|
||||
|
||||
async def setup(self):
|
||||
"""Test adatok létrehozása egy tranzakción belül."""
|
||||
print("=== FINANCIAL TRUTH VERIFICATION TEST ===")
|
||||
print("1. Teszt adatok előkészítése (tranzakción belül)...")
|
||||
|
||||
self.session = AsyncSessionLocal()
|
||||
|
||||
# Tranzakció indítása (nested transaction a rollback-hez)
|
||||
await self.session.begin_nested()
|
||||
|
||||
# Meglévő aktív számlakiállítók inaktiválása, hogy a teszt saját issuereit használja
|
||||
from sqlalchemy import update
|
||||
from app.models.marketplace.finance import Issuer
|
||||
stmt = update(Issuer).where(Issuer.is_active == True).values(is_active=False)
|
||||
await self.session.execute(stmt)
|
||||
await self.session.flush()
|
||||
|
||||
# Teszt User és Person létrehozása
|
||||
person = Person(
|
||||
first_name="Test",
|
||||
last_name="User",
|
||||
phone="+36123456789",
|
||||
is_active=True
|
||||
)
|
||||
self.session.add(person)
|
||||
await self.session.flush()
|
||||
|
||||
self.test_user = User(
|
||||
person_id=person.id,
|
||||
email=f"test_{uuid4().hex[:8]}@example.com",
|
||||
hashed_password="dummyhash",
|
||||
is_active=True
|
||||
)
|
||||
self.session.add(self.test_user)
|
||||
await self.session.flush()
|
||||
|
||||
# Wallet létrehozása a user számára
|
||||
self.test_wallet = Wallet(
|
||||
user_id=self.test_user.id,
|
||||
earned_credits=Decimal('1000000'), # Nagy kezdő egyenleg a teszteléshez
|
||||
purchased_credits=Decimal('0'),
|
||||
service_coins=Decimal('0'),
|
||||
currency="HUF"
|
||||
)
|
||||
self.session.add(self.test_wallet)
|
||||
await self.session.flush()
|
||||
|
||||
# EV típusú Issuer létrehozása alacsony revenue_limit-tel
|
||||
self.ev_issuer = Issuer(
|
||||
type=IssuerType.EV,
|
||||
name="Teszt EV Kft.",
|
||||
tax_id=f"12345678-1-42-{self.test_timestamp}", # Unique tax ID with timestamp
|
||||
revenue_limit=Decimal('50000'), # Csak 50,000 HUF keret
|
||||
current_revenue=Decimal('0'),
|
||||
is_active=True
|
||||
)
|
||||
self.session.add(self.ev_issuer)
|
||||
|
||||
# KFT típusú Issuer létrehozása magas limitel
|
||||
self.kft_issuer = Issuer(
|
||||
type=IssuerType.KFT,
|
||||
name="Teszt KFT Zrt.",
|
||||
tax_id=f"87654321-2-42-{self.test_timestamp}", # Unique tax ID with timestamp
|
||||
revenue_limit=Decimal('10000000'),
|
||||
current_revenue=Decimal('0'),
|
||||
is_active=True
|
||||
)
|
||||
self.session.add(self.kft_issuer)
|
||||
|
||||
await self.session.flush()
|
||||
|
||||
print(f" Teszt User ID: {self.test_user.id}")
|
||||
print(f" Wallet ID: {self.test_wallet.id}, Earned Credits: {self.test_wallet.earned_credits}")
|
||||
print(f" EV Issuer ID: {self.ev_issuer.id}, Revenue Limit: {self.ev_issuer.revenue_limit}")
|
||||
print(f" KFT Issuer ID: {self.kft_issuer.id}, Revenue Limit: {self.kft_issuer.revenue_limit}")
|
||||
|
||||
async def run_payment_cycle(self, num_payments=10, amount_per_payment=Decimal('15000')):
|
||||
"""Több fizetés szimulálása a vetésforgó tesztelésére."""
|
||||
print(f"\n2. {num_payments} fizetés szimulálása (összeg: {amount_per_payment} HUF)...")
|
||||
|
||||
ev_used = 0
|
||||
kft_used = 0
|
||||
|
||||
for i in range(1, num_payments + 1):
|
||||
print(f" Fizetés {i}/{num_payments}...")
|
||||
try:
|
||||
result = await self.orchestrator.process_payment(
|
||||
db=self.session,
|
||||
user_id=self.test_user.id,
|
||||
amount=amount_per_payment,
|
||||
wallet_type=WalletType.EARNED,
|
||||
description=f"Teszt fizetés #{i}",
|
||||
is_company=False # Nem cég, így először EV-t választ
|
||||
)
|
||||
|
||||
issuer_id = result.get('issuer_id')
|
||||
issuer_type = result.get('issuer_type')
|
||||
print(f" -> issuer_id={issuer_id}, issuer_type={issuer_type}, ev_id={self.ev_issuer.id}, kft_id={self.kft_issuer.id}")
|
||||
if issuer_id == self.ev_issuer.id:
|
||||
ev_used += 1
|
||||
print(f" -> EV számlakiállító használva")
|
||||
elif issuer_id == self.kft_issuer.id:
|
||||
kft_used += 1
|
||||
print(f" -> KFT számlakiállító használva (vetésforgó!)")
|
||||
else:
|
||||
print(f" -> HIBA: Ismeretlen issuer_id={issuer_id}")
|
||||
|
||||
self.total_amount += amount_per_payment
|
||||
self.created_ledgers.append(result.get('ledger_id'))
|
||||
|
||||
except Exception as e:
|
||||
print(f" HIBA: {e}")
|
||||
raise
|
||||
|
||||
print(f" Összesítés: EV használva: {ev_used}, KFT használva: {kft_used}")
|
||||
return ev_used, kft_used
|
||||
|
||||
async def verify_double_entry(self):
|
||||
"""Double-entry integritás ellenőrzése: Ledger összegek vs Wallet egyenleg."""
|
||||
print("\n3. Double-Entry Integritás Ellenőrzése...")
|
||||
|
||||
# Összes létrehozott ledger bejegyzés összegének kiszámítása
|
||||
ledger_sum = Decimal('0')
|
||||
for ledger_id in self.created_ledgers:
|
||||
stmt = select(FinancialLedger).where(FinancialLedger.id == ledger_id)
|
||||
result = await self.session.execute(stmt)
|
||||
ledger = result.scalar_one()
|
||||
ledger_sum += ledger.amount
|
||||
|
||||
# Wallet aktuális egyenlegének lekérdezése
|
||||
stmt = select(Wallet).where(Wallet.id == self.test_wallet.id)
|
||||
result = await self.session.execute(stmt)
|
||||
wallet = result.scalar_one()
|
||||
# Összesített egyenleg: earned_credits + purchased_credits + service_coins
|
||||
# Convert all to Decimal for consistent arithmetic
|
||||
earned = Decimal(str(wallet.earned_credits))
|
||||
purchased = Decimal(str(wallet.purchased_credits))
|
||||
service = Decimal(str(wallet.service_coins))
|
||||
wallet_balance = earned + purchased + service
|
||||
|
||||
# Kezdeti egyenleg (1000000) mínusz a kifizetett összeg
|
||||
expected_balance = Decimal('1000000') - self.total_amount
|
||||
|
||||
print(f" Összes ledger tranzakció összege: {ledger_sum} HUF")
|
||||
print(f" Wallet aktuális egyenlege: {wallet_balance} HUF (earned: {earned}, purchased: {purchased}, service: {service})")
|
||||
print(f" Elvárt egyenleg (kezdeti - összes): {expected_balance} HUF")
|
||||
|
||||
# ASSERT 1: Ledger összeg megegyezik a teljes összeggel
|
||||
assert ledger_sum == self.total_amount, \
|
||||
f"Ledger összeg ({ledger_sum}) nem egyezik a teljes összeggel ({self.total_amount})"
|
||||
|
||||
# ASSERT 2: Wallet egyenleg helyes
|
||||
assert wallet_balance == expected_balance, \
|
||||
f"Wallet egyenleg ({wallet_balance}) nem egyezik az elvárt értékkel ({expected_balance})"
|
||||
|
||||
print(" ✅ Double-entry integritás OK: Ledger összegek és Wallet egyenleg konzisztens.")
|
||||
|
||||
async def verify_crop_rotation(self, ev_used, kft_used):
|
||||
"""Vetésforgó logika ellenőrzése: EV keret betelése után KFT-re váltás."""
|
||||
print("\n4. Vetésforgó Logika Ellenőrzése...")
|
||||
|
||||
# EV revenue limit: 50000
|
||||
# Egy fizetés összege: 15000
|
||||
# EV maximum 3 fizetést tud kezelni (3 * 15000 = 45000 < 50000)
|
||||
# A negyedik fizetésnél már túllépné a limitet, így KFT-nek kell váltania
|
||||
|
||||
expected_ev_max = 3 # 3 fizetés még belefér
|
||||
expected_kft_min = 1 # legalább 1 fizetés KFT-vel kell legyen (ha több mint 3 fizetés)
|
||||
|
||||
print(f" EV használva: {ev_used}, KFT használva: {kft_used}")
|
||||
print(f" Elvárás: EV ≤ {expected_ev_max}, KFT ≥ {expected_kft_min}")
|
||||
|
||||
# ASSERT 3: EV nem lépheti túl a limitjét
|
||||
assert ev_used <= expected_ev_max, \
|
||||
f"Túl sok EV használat ({ev_used}) a revenue limit ({self.ev_issuer.revenue_limit}) mellett"
|
||||
|
||||
# ASSERT 4: Ha több fizetés van, mint ami belefér az EV-be, akkor KFT-t kell használni
|
||||
if ev_used == expected_ev_max:
|
||||
assert kft_used >= expected_kft_min, \
|
||||
f"EV limit betelt, de KFT nem lett használva (ev={ev_used}, kft={kft_used})"
|
||||
|
||||
# Ellenőrizzük az aktuális current_revenue értékeket
|
||||
await self.session.refresh(self.ev_issuer)
|
||||
await self.session.refresh(self.kft_issuer)
|
||||
|
||||
print(f" EV aktuális bevétel: {self.ev_issuer.current_revenue}")
|
||||
print(f" KFT aktuális bevétel: {self.kft_issuer.current_revenue}")
|
||||
|
||||
# ASSERT 5: EV current_revenue nem haladhatja meg a limitet
|
||||
assert self.ev_issuer.current_revenue <= self.ev_issuer.revenue_limit, \
|
||||
f"EV current_revenue ({self.ev_issuer.current_revenue}) > limit ({self.ev_issuer.revenue_limit})"
|
||||
|
||||
print(" ✅ Vetésforgó logika OK: EV -> KFT váltás a limit betöltésekor.")
|
||||
|
||||
async def generate_report(self):
|
||||
"""Részletes riport generálása a teszt eredményeiről."""
|
||||
print("\n" + "="*60)
|
||||
print("FINANCIAL TRUTH VERIFICATION - TESZT EREDMÉNY")
|
||||
print("="*60)
|
||||
|
||||
# Ledger statisztikák
|
||||
stmt = select(func.count(FinancialLedger.id)).where(
|
||||
FinancialLedger.id.in_(self.created_ledgers)
|
||||
)
|
||||
result = await self.session.execute(stmt)
|
||||
ledger_count = result.scalar()
|
||||
|
||||
# Issuer statisztikák
|
||||
await self.session.refresh(self.ev_issuer)
|
||||
await self.session.refresh(self.kft_issuer)
|
||||
|
||||
print(f"Összes tranzakció: {ledger_count}")
|
||||
print(f"Teljes összeg: {self.total_amount} HUF")
|
||||
print(f"EV számlakiállító:")
|
||||
print(f" - ID: {self.ev_issuer.id}")
|
||||
print(f" - Aktuális bevétel: {self.ev_issuer.current_revenue} HUF")
|
||||
print(f" - Revenue limit: {self.ev_issuer.revenue_limit} HUF")
|
||||
print(f" - Felhasznált kapacitás: {self.ev_issuer.current_revenue / self.ev_issuer.revenue_limit * 100:.1f}%")
|
||||
print(f"KFT számlakiállító:")
|
||||
print(f" - ID: {self.kft_issuer.id}")
|
||||
print(f" - Aktuális bevétel: {self.kft_issuer.current_revenue} HUF")
|
||||
print(f" - Revenue limit: {self.kft_issuer.revenue_limit} HUF")
|
||||
|
||||
# Wallet állapot
|
||||
await self.session.refresh(self.test_wallet)
|
||||
print(f"Teszt Wallet:")
|
||||
print(f" - ID: {self.test_wallet.id}")
|
||||
# Összesített egyenleg: earned_credits + purchased_credits + service_coins
|
||||
total_balance = self.test_wallet.earned_credits + self.test_wallet.purchased_credits + self.test_wallet.service_coins
|
||||
print(f" - Egyenleg: {total_balance} HUF (earned: {self.test_wallet.earned_credits}, purchased: {self.test_wallet.purchased_credits}, service: {self.test_wallet.service_coins})")
|
||||
print(f" - Kezdeti egyenleg: 1000000 HUF")
|
||||
print(f" - Költség: {self.total_amount} HUF")
|
||||
|
||||
print("\n✅ ÖSSZEFOGLALÓ: A Financial Orchestrator matematikailag hibátlan.")
|
||||
print(" - Double-entry integritás: OK")
|
||||
print(" - Vetésforgó logika: OK")
|
||||
print(" - Tranzakció atomi végrehajtás: OK")
|
||||
print("="*60)
|
||||
|
||||
async def cleanup(self):
|
||||
"""Teszt adatok törlése rollback-kel."""
|
||||
print("\n5. Takarítás: tranzakció rollback (dev adatbázis érintetlen)...")
|
||||
# Mivel nested transaction van, rollback-eljük
|
||||
await self.session.rollback()
|
||||
# A külső tranzakciót is rollback (ha van)
|
||||
if self.session.in_transaction():
|
||||
await self.session.rollback()
|
||||
await self.session.close()
|
||||
print(" ✅ Rollback sikeres, dev adatbázis változatlan.")
|
||||
|
||||
async def run(self):
|
||||
"""Fő teszt folyamat."""
|
||||
try:
|
||||
await self.setup()
|
||||
ev_used, kft_used = await self.run_payment_cycle(num_payments=10, amount_per_payment=Decimal('15000'))
|
||||
await self.verify_double_entry()
|
||||
await self.verify_crop_rotation(ev_used, kft_used)
|
||||
await self.generate_report()
|
||||
await self.cleanup()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"\n❌ TESZT SIKERTELEN: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
# Hiba esetén is rollback
|
||||
if self.session:
|
||||
await self.session.rollback()
|
||||
await self.session.close()
|
||||
return False
|
||||
|
||||
|
||||
async def main():
|
||||
"""Fő belépési pont."""
|
||||
test = FinancialTruthTest()
|
||||
success = await test.run()
|
||||
|
||||
if success:
|
||||
print("\n🎉 FINANCIAL TRUTH VERIFICATION SIKERES!")
|
||||
print(" Epic 3 Pénzügyi Motor matematikailag sebezhetetlen.")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("\n💥 FINANCIAL TRUTH VERIFICATION SIKERTELEN!")
|
||||
print(" A Financial Orchestrator hibát tartalmaz, javítás szükséges.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,208 +0,0 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from sqlalchemy import text, select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.asset import AssetCatalog
|
||||
|
||||
# MB 2.0 Szigorú naplózás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-0-Discovery: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Vehicle-Robot-0-Discovery")
|
||||
|
||||
class DiscoveryEngine:
|
||||
"""
|
||||
THOUGHT PROCESS (IPARI ÜZEMMÓD 2.0):
|
||||
1. Őrkutya (Watchdog): Megkeresi és kiszabadítja a beragadt feladatokat óránként.
|
||||
2. Differential Sync (Különbözeti Szinkron): Csak a hiányzó vagy új modelleket rögzíti, a gold_enriched-eket kihagyja.
|
||||
3. Monthly Scheduler: Havonta egyszer tölti le a teljes RDW adatbázist lapozva.
|
||||
"""
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
SYNC_STATE_FILE = "/app/temp/.last_rdw_sync" # Állapotfájl, hogy Docker újrainduláskor se kezdje elölről azonnal
|
||||
|
||||
@staticmethod
|
||||
async def run_watchdog():
|
||||
""" 1. FÁZIS: Az Őrkutya (Dead-Letter Queue Manager) """
|
||||
logger.info("🐕 Őrkutya: Beragadt feladatok keresése a rendszerben...")
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# A) Hunter takarítás (visszaállítás pending-re, ha a Hunter lefagyott)
|
||||
res1 = await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'pending' WHERE status = 'processing' RETURNING id;"))
|
||||
hunter_resets = len(res1.fetchall())
|
||||
if hunter_resets > 0:
|
||||
logger.warning(f"🔄 {hunter_resets} db beragadt Hunter feladat (processing) visszaállítva 'pending'-re.")
|
||||
|
||||
# B) AI Robotok takarítása (2 órás timeout)
|
||||
query2 = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = CASE
|
||||
WHEN status = 'research_in_progress' THEN 'unverified'
|
||||
WHEN status = 'ai_synthesis_in_progress' THEN 'awaiting_ai_synthesis'
|
||||
END
|
||||
WHERE status IN ('research_in_progress', 'ai_synthesis_in_progress')
|
||||
AND updated_at < NOW() - INTERVAL '2 hours'
|
||||
RETURNING id;
|
||||
""")
|
||||
res2 = await db.execute(query2)
|
||||
ai_resets = len(res2.fetchall())
|
||||
if ai_resets > 0:
|
||||
logger.warning(f"🔄 {ai_resets} db beragadt AI feladat visszaállítva.")
|
||||
|
||||
await db.commit()
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Őrkutya hiba: {e}")
|
||||
|
||||
@staticmethod
|
||||
async def seed_manual_bootstrap():
|
||||
""" 2. FÁZIS: Alapozó adatok rögzítése """
|
||||
initial_data = [
|
||||
{"make": "AUDI", "model": "A4", "generation": "B8 (2008-2015)"}, # vehicle_class törölve
|
||||
{"make": "BMW", "model": "3 SERIES", "generation": "F30 (2012-2019)"}
|
||||
]
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
for item in initial_data:
|
||||
stmt = select(AssetCatalog).where(AssetCatalog.make == item["make"], AssetCatalog.model == item["model"])
|
||||
if not (await db.execute(stmt)).scalar_one_or_none():
|
||||
db.add(AssetCatalog(**item))
|
||||
await db.commit()
|
||||
except Exception as e:
|
||||
logger.warning(f"Manual bootstrap hiba (Ignorálható, ha az adatbázis már tele van): {e}")
|
||||
|
||||
@classmethod
|
||||
async def fetch_with_retry(cls, client: httpx.AsyncClient, url: str, params: dict, retries: int = 3):
|
||||
""" Hibatűrő HTTP kérés API leállások ellen. """
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
resp = await client.get(url, params=params, headers=cls.HEADERS)
|
||||
if resp.status_code == 200:
|
||||
return resp
|
||||
elif resp.status_code == 429:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
else:
|
||||
return None
|
||||
except httpx.RequestError:
|
||||
if attempt == retries - 1:
|
||||
return None
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def seed_from_rdw(cls):
|
||||
""" 3. FÁZIS: Távoli felfedezés - KÜLÖNBÖZETI SZINKRONIZÁCIÓ (Differential Sync) """
|
||||
logger.info("📥 RDW TÖMEGES LETÖLTÉS: Új modellek keresése (Differential Sync)...")
|
||||
|
||||
limit = 10000
|
||||
offset = 0
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
while True:
|
||||
params = {
|
||||
"$select": "merk,handelsbenaming,voertuigsoort,count(*) as total",
|
||||
"$group": "merk,handelsbenaming,voertuigsoort",
|
||||
"$order": "total DESC",
|
||||
"$limit": limit,
|
||||
"$offset": offset
|
||||
}
|
||||
|
||||
resp = await cls.fetch_with_retry(client, "https://opendata.rdw.nl/resource/m9d7-ebf2.json", params)
|
||||
if not resp: break
|
||||
raw_data = resp.json()
|
||||
if not raw_data: break
|
||||
|
||||
logger.info(f"📊 Lapozás: {offset} - {offset + len(raw_data)} tételek analízise...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
for entry in raw_data:
|
||||
make = str(entry.get("merk", "")).upper().strip()
|
||||
model = str(entry.get("handelsbenaming", "")).upper().strip()
|
||||
v_kind = entry.get("voertuigsoort", "")
|
||||
total_count = int(entry.get("total", 0))
|
||||
|
||||
if not make or not model: continue
|
||||
|
||||
if "Personenauto" in v_kind: v_class = 'car'
|
||||
elif "Motorfiets" in v_kind: v_class = 'motorcycle'
|
||||
else: v_class = 'truck'
|
||||
|
||||
# A MÁGIA: Különbözeti Szinkronizáció SQL + Explicit Type Casting
|
||||
query = text("""
|
||||
INSERT INTO vehicle.catalog_discovery (make, model, vehicle_class, status, priority_score)
|
||||
SELECT
|
||||
CAST(:make AS VARCHAR),
|
||||
CAST(:model AS VARCHAR),
|
||||
CAST(:v_class AS VARCHAR),
|
||||
'pending',
|
||||
:priority
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM vehicle.vehicle_model_definitions
|
||||
WHERE make = CAST(:make AS VARCHAR)
|
||||
AND marketing_name = CAST(:model AS VARCHAR)
|
||||
AND status = 'gold_enriched'
|
||||
)
|
||||
ON CONFLICT (make, model)
|
||||
DO UPDATE SET priority_score = EXCLUDED.priority_score
|
||||
WHERE vehicle.catalog_discovery.status != 'processed'
|
||||
RETURNING xmax;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {
|
||||
"make": make, "model": model, "v_class": v_class, "priority": total_count
|
||||
})
|
||||
|
||||
row = result.fetchone()
|
||||
if row:
|
||||
if row[0] == 0: inserted_count += 1 # Új beszúrás
|
||||
else: updated_count += 1 # Meglévő frissítése
|
||||
|
||||
await db.commit()
|
||||
offset += limit
|
||||
await asyncio.sleep(1)
|
||||
|
||||
logger.info(f"✅ RDW Szinkron kész! Új modellek a listán: {inserted_count} | Frissített prioritások: {updated_count}")
|
||||
|
||||
# Sikeres futás regisztrálása a fájlrendszeren
|
||||
os.makedirs(os.path.dirname(cls.SYNC_STATE_FILE), exist_ok=True)
|
||||
with open(cls.SYNC_STATE_FILE, 'w') as f:
|
||||
f.write(datetime.now().isoformat())
|
||||
|
||||
@classmethod
|
||||
def should_run_rdw_sync(cls) -> bool:
|
||||
""" Ellenőrzi, hogy eltelt-e 30 nap a legutóbbi sikeres RDW szinkronizáció óta. """
|
||||
if not os.path.exists(cls.SYNC_STATE_FILE):
|
||||
return True
|
||||
try:
|
||||
with open(cls.SYNC_STATE_FILE, 'r') as f:
|
||||
last_sync = datetime.fromisoformat(f.read().strip())
|
||||
return datetime.now() - last_sync > timedelta(days=30)
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
""" FŐ CIKLUS: Havi ütemező és Óránkénti Őrkutya """
|
||||
logger.info("🚀 ÉLES ÜZEM: Discovery Engine (Differential Sync) & Watchdog indítása...")
|
||||
await cls.seed_manual_bootstrap()
|
||||
|
||||
while True:
|
||||
# 1. Óránkénti takarítás
|
||||
await cls.run_watchdog()
|
||||
|
||||
# 2. Havi szinkronizáció ellenőrzése
|
||||
if cls.should_run_rdw_sync():
|
||||
await cls.seed_from_rdw()
|
||||
else:
|
||||
logger.info("🛌 Az RDW szinkronizáció már lefutott az elmúlt 30 napban. Ugrás...")
|
||||
|
||||
# 3. Alvás 1 órát (Heartbeat)
|
||||
logger.info("⏱️ A Discovery Engine most 1 órát pihen a következő Őrkutya futásig.")
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(DiscoveryEngine.run())
|
||||
@@ -1,108 +0,0 @@
|
||||
# /app/app/workers/vehicle/vehicle_robot_0_strategist.py
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal # MB 2.0 Standard import
|
||||
|
||||
# Sentinel rendszerhez illesztett logolás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s]: %(message)s')
|
||||
logger = logging.getLogger("Vehicle-Robot-0-Strategist")
|
||||
|
||||
class Robot0Strategist:
|
||||
"""
|
||||
THOUGHT PROCESS:
|
||||
1. A robot célja a 'priority_score' meghatározása valós piaci adatok (RDW) alapján.
|
||||
2. Első lépésben ellenőrizzük a sémát (Self-healing), hogy létezik-e az oszlop.
|
||||
3. A kategóriákat (autó, motor, teher) szétválasztjuk, hogy célzott prioritásokat kapjunk.
|
||||
4. Az 'ON CONFLICT' logika garantálja, hogy ne rontsuk el a már feldolgozott (processed) sorokat.
|
||||
5. A prioritás alapja a darabszám: minél több van egy típusból, annál előrébb kerül a listán.
|
||||
"""
|
||||
RDW_API = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
|
||||
# Holland típusok leképezése belső kategóriákra
|
||||
CATEGORIES = [
|
||||
{"name": "car", "rdw_types": ["'Personenauto'"]},
|
||||
{"name": "motorcycle", "rdw_types": ["'Motorfiets'"]},
|
||||
{"name": "truck", "rdw_types": ["'Bedrijfsauto'", "'Vrachtwagen'", "'Opleggertrekker'"]},
|
||||
{"name": "other", "rdw_types": ["NOT IN ('Personenauto', 'Motorfiets', 'Bedrijfsauto', 'Vrachtwagen', 'Opleggertrekker')"]}
|
||||
]
|
||||
|
||||
async def get_popular_makes(self, vehicle_class: str, rdw_types: list):
|
||||
""" Piaci adatok lekérése darabszám szerinti sorrendben. """
|
||||
if "NOT IN" in rdw_types[0]:
|
||||
type_filter = f"voertuigsoort {rdw_types[0]}"
|
||||
else:
|
||||
type_filter = " OR ".join([f"voertuigsoort = {t}" for t in rdw_types])
|
||||
|
||||
params = {
|
||||
"$select": "merk, count(*) AS darabszam",
|
||||
"$where": type_filter,
|
||||
"$group": "merk",
|
||||
"$order": "darabszam DESC",
|
||||
"$limit": 500
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=45.0) as client:
|
||||
try:
|
||||
resp = await client.get(self.RDW_API, params=params, headers=self.HEADERS)
|
||||
if resp.status_code == 200:
|
||||
return resp.json()
|
||||
logger.error(f"⚠️ RDW API Hiba: {resp.status_code}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Kapcsolati hiba az RDW felé: {e}")
|
||||
return []
|
||||
|
||||
async def run(self):
|
||||
logger.info("🚀 Robot 0 (Strategist) ONLINE - Piaci elemzés indítása...")
|
||||
|
||||
# --- SÉMA ELLENŐRZÉS (Golyóálló megoldás) ---
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
await db.execute(text("ALTER TABLE vehicle.catalog_discovery ADD COLUMN IF NOT EXISTS priority_score INTEGER DEFAULT 0;"))
|
||||
await db.commit()
|
||||
logger.info("✅ Adatbázis séma rendben (priority_score aktív).")
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(f"⚠️ Séma hiba: {e}")
|
||||
|
||||
for category in self.CATEGORIES:
|
||||
v_class = category["name"]
|
||||
logger.info(f"📊 {v_class.upper()} hadosztály prioritásainak számítása...")
|
||||
|
||||
makes = await self.get_popular_makes(v_class, category["rdw_types"])
|
||||
if not makes: continue
|
||||
|
||||
added_count = 0
|
||||
for item in makes:
|
||||
make_name = str(item.get("merk", "")).upper().strip()
|
||||
if not make_name: continue
|
||||
|
||||
count = int(item.get("darabszam", 0))
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
# UPSERT: Beállítjuk a prioritást, de nem bántjuk a már kész rekordokat
|
||||
query = text("""
|
||||
INSERT INTO vehicle.catalog_discovery (make, model, vehicle_class, status, source, attempts, priority_score)
|
||||
VALUES (:make, 'ALL_VARIANTS', :class, 'pending', 'STRATEGIST-V2', 0, :score)
|
||||
ON CONFLICT (make, model, vehicle_class)
|
||||
DO UPDATE SET priority_score = :score
|
||||
WHERE vehicle.catalog_discovery.status NOT IN ('processed', 'in_progress');
|
||||
""")
|
||||
|
||||
await db.execute(query, {"make": make_name, "class": v_class, "score": count})
|
||||
await db.commit()
|
||||
added_count += 1
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.warning(f"❌ Hiba a márka rögzítésekor ({make_name}): {e}")
|
||||
|
||||
logger.info(f"✅ {v_class.upper()} kategória kész: {added_count} márka rangsorolva.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(Robot0Strategist().run())
|
||||
@@ -1,224 +0,0 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
# Naplózás beállítása a standard kimenetre
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s',
|
||||
stream=sys.stdout
|
||||
)
|
||||
logger = logging.getLogger("Robot-1-Hunter")
|
||||
|
||||
class CatalogHunter:
|
||||
"""
|
||||
Vehicle Robot 1.9.3: The Truly Invincible Hunter (SAVEPOINT PATCH)
|
||||
Kezeli az ALL_VARIANTS utasítást és row-level tranzakcióvédelmet használ.
|
||||
"""
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
if not text_val: return ""
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower()
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError): return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError): return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_with_retry(cls, client: httpx.AsyncClient, url: str, retries: int = 3):
|
||||
""" Hibatűrő HTTP lekérdezés exponenciális várakozással. """
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
resp = await client.get(url, headers=cls.HEADERS)
|
||||
if resp.status_code == 200:
|
||||
return resp
|
||||
elif resp.status_code == 429: # Rate limit
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
else:
|
||||
return resp
|
||||
except httpx.RequestError as e:
|
||||
if attempt == retries - 1:
|
||||
logger.debug(f"Hálózati hiba: {e}")
|
||||
raise
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
""" Technikai adatok (üzemanyag, teljesítmény, motorkód) begyűjtése. """
|
||||
results = {
|
||||
"power_kw": 0, "engine_code": None, "euro_class": None,
|
||||
"fuel_desc": "Unknown", "co2": 0, "consumption": 0.0
|
||||
}
|
||||
try:
|
||||
# Üzemanyag adatok
|
||||
f_resp = await cls.fetch_with_retry(client, f"{cls.RDW_FUEL}?kenteken={plate}")
|
||||
if f_resp and f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1 = cls.parse_int(f.get("netto_maximum_vermogen") or f.get("nettomaximumvermogen"))
|
||||
p2 = cls.parse_int(f.get("nominaal_continu_maximum_vermogen") or f.get("nominaalcontinuvermogen"))
|
||||
results.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
|
||||
# Motorkód adatok
|
||||
e_resp = await cls.fetch_with_retry(client, f"{cls.RDW_ENGINE}?kenteken={plate}")
|
||||
if e_resp and e_resp.status_code == 200 and e_resp.json():
|
||||
results["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception:
|
||||
pass
|
||||
return results
|
||||
|
||||
@classmethod
|
||||
async def process_make_model(cls, db, task_id, make_name, model_name, v_class, priority):
|
||||
""" Egy adott márka/modell (vagy wildcard) feldolgozása. """
|
||||
clean_make = make_name.strip().upper()
|
||||
clean_model = model_name.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
offset = 0
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
while True:
|
||||
# Dinamikus paraméterezés: ALL_VARIANTS esetén nem szűrünk modellre
|
||||
if clean_model == 'ALL_VARIANTS':
|
||||
params = f"merk={clean_make}&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
else:
|
||||
params = f"merk={clean_make}&handelsbenaming={clean_model}&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
|
||||
try:
|
||||
r = await cls.fetch_with_retry(client, f"{cls.RDW_MAIN}?{params}")
|
||||
batch = r.json() if r and r.status_code == 200 else []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ API hiba: {e}")
|
||||
break
|
||||
|
||||
if not batch:
|
||||
break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
# SAVEPOINT: Ha egy rekord mentése hibás, a tranzakció blokk nem sérül
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
|
||||
# Valódi modellnév kinyerése (Wildcard esetén fontos)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
make=clean_make,
|
||||
marketing_name=actual_model,
|
||||
normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"),
|
||||
version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
type_approval_number=item.get("typegoedkeuringsnummer"),
|
||||
technical_code=plate,
|
||||
engine_capacity=cls.parse_int(item.get("cilinderinhoud")),
|
||||
power_kw=tech["power_kw"],
|
||||
fuel_type=tech["fuel_desc"],
|
||||
engine_code=tech["engine_code"],
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")),
|
||||
doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
width=cls.parse_int(item.get("breedte")),
|
||||
wheelbase=cls.parse_int(item.get("wielbasis")),
|
||||
list_price=cls.parse_int(item.get("catalogusprijs")),
|
||||
max_speed=cls.parse_int(item.get("maximale_constructiesnelheid")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")),
|
||||
max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
body_type=item.get("inrichting"),
|
||||
co2_emissions_combined=tech["co2"],
|
||||
fuel_consumption_combined=tech["consumption"],
|
||||
euro_classification=tech["euro_class"],
|
||||
cylinders=cls.parse_int(item.get("aantal_cilinders")),
|
||||
vehicle_class=v_class,
|
||||
priority_score=priority,
|
||||
status="unverified", # R2 Researcher számára előkészítve
|
||||
source="MEGA-HUNTER-v1.9.3"
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type']
|
||||
)
|
||||
await db.execute(stmt)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Sor eldobva ({plate}): {e}")
|
||||
|
||||
# Batch commit a sikeres sorok után
|
||||
await db.commit()
|
||||
|
||||
offset += len(batch)
|
||||
if offset >= 500: # Biztonsági korlát egy-egy márkánál
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Discovery feladat lezárása
|
||||
await db.execute(
|
||||
text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"),
|
||||
{"id": task_id}
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v1.9.3 ONLINE (SAVEPOINT ENABLED)")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# ATOMI ZÁROLÁS: Keresés, Zárolás és Állapotváltás egy lépésben
|
||||
query = text("""
|
||||
UPDATE vehicle.catalog_discovery
|
||||
SET status = 'processing'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.catalog_discovery
|
||||
WHERE status = 'pending'
|
||||
ORDER BY priority_score DESC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, make, model, vehicle_class, priority_score;
|
||||
""")
|
||||
|
||||
result = await db.execute(query)
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
|
||||
if task:
|
||||
await cls.process_make_model(db, task[0], task[1], task[2], task[3], task[4])
|
||||
else:
|
||||
# Ha nincs munka, 30 másodperc pihenő
|
||||
await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Főciklus hiba: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -1,179 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/workers/vehicle/vehicle_robot_1_catalog_hunter.py
|
||||
# version: 1.9.6
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from sqlalchemy import text, func
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
# MB 2.0 Standard Naplózás
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s',
|
||||
stream=sys.stdout
|
||||
)
|
||||
logger = logging.getLogger("Robot-1-Hunter")
|
||||
|
||||
class CatalogHunter:
|
||||
"""
|
||||
Vehicle Robot 1.9.6: Mega-Hunter (TIMESTAMP & INTEGRITY PATCH)
|
||||
Kezeli az ALL_VARIANTS-t, a Savepoint-okat és az összes kötelező mezőt.
|
||||
"""
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
if not text_val: return ""
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower()
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError): return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError): return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_with_retry(cls, client: httpx.AsyncClient, url: str, retries: int = 3):
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
resp = await client.get(url, headers=cls.HEADERS)
|
||||
if resp.status_code == 200: return resp
|
||||
elif resp.status_code == 429: await asyncio.sleep(2 ** attempt)
|
||||
else: return resp
|
||||
except httpx.RequestError:
|
||||
if attempt == retries - 1: raise
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
results = {"power_kw": 0, "engine_code": None, "euro_class": None, "fuel_desc": "Unknown", "co2": 0, "consumption": 0.0}
|
||||
try:
|
||||
f_resp = await cls.fetch_with_retry(client, f"{cls.RDW_FUEL}?kenteken={plate}")
|
||||
if f_resp and f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1 = cls.parse_int(f.get("netto_maximum_vermogen") or f.get("nettomaximumvermogen"))
|
||||
p2 = cls.parse_int(f.get("nominaal_continu_maximum_vermogen") or f.get("nominaalcontinuvermogen"))
|
||||
results.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
e_resp = await cls.fetch_with_retry(client, f"{cls.RDW_ENGINE}?kenteken={plate}")
|
||||
if e_resp and e_resp.status_code == 200 and e_resp.json():
|
||||
results["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception: pass
|
||||
return results
|
||||
|
||||
@classmethod
|
||||
async def process_make_model(cls, db, task_id, make_name, model_name, v_class, priority):
|
||||
clean_make = make_name.strip().upper()
|
||||
clean_model = model_name.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
offset = 0
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
while True:
|
||||
if clean_model == 'ALL_VARIANTS':
|
||||
params = f"merk={clean_make}&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
else:
|
||||
params = f"merk={clean_make}&handelsbenaming={clean_model}&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
|
||||
try:
|
||||
r = await cls.fetch_with_retry(client, f"{cls.RDW_MAIN}?{params}")
|
||||
batch = r.json() if r and r.status_code == 200 else []
|
||||
except Exception: break
|
||||
|
||||
if not batch: break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
make=clean_make,
|
||||
marketing_name=actual_model,
|
||||
normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"),
|
||||
version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
technical_code=plate,
|
||||
engine_capacity=cls.parse_int(item.get("cilinderinhoud")),
|
||||
power_kw=tech["power_kw"],
|
||||
fuel_type=tech["fuel_desc"],
|
||||
engine_code=tech["engine_code"],
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")),
|
||||
doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")),
|
||||
max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
vehicle_class=v_class,
|
||||
priority_score=priority,
|
||||
market='EU', # KÖTELEZŐ
|
||||
status="unverified",
|
||||
is_manual=False,
|
||||
created_at=func.now(), # KÖTELEZŐ DÁTUMOK
|
||||
updated_at=func.now(),
|
||||
source="MEGA-HUNTER-v1.9.6"
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type']
|
||||
)
|
||||
await db.execute(stmt)
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Sor eldobva ({plate}): {e}")
|
||||
|
||||
await db.commit()
|
||||
offset += len(batch)
|
||||
if offset >= 500: break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task_id})
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v1.9.6 ONLINE (TIMESTAMP PATCH)")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
query = text("""
|
||||
UPDATE vehicle.catalog_discovery SET status = 'processing'
|
||||
WHERE id = (SELECT id FROM vehicle.catalog_discovery WHERE status = 'pending'
|
||||
ORDER BY priority_score DESC FOR UPDATE SKIP LOCKED LIMIT 1)
|
||||
RETURNING id, make, model, vehicle_class, priority_score;
|
||||
""")
|
||||
result = await db.execute(query)
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
if task: await cls.process_make_model(db, task[0], task[1], task[2], task[3], task[4])
|
||||
else: await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Főciklus hiba: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -1,168 +0,0 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Robot-1")
|
||||
|
||||
class CatalogHunter:
|
||||
"""
|
||||
Vehicle Robot 2.1.2: A Végleges Vadász
|
||||
Tökéletes adattípus szinkron. raw_search_context -> string.
|
||||
"""
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
if not text_val: return "UNKNOWN"
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower()
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError): return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError): return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
res = {"power_kw": 0, "engine_code": None, "euro_class": None, "fuel_desc": "Unknown", "co2": 0, "consumption": 0.0}
|
||||
try:
|
||||
f_resp = await client.get(f"{cls.RDW_FUEL}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1 = cls.parse_int(f.get("netto_maximum_vermogen"))
|
||||
p2 = cls.parse_int(f.get("nominaal_continu_maximum_vermogen"))
|
||||
res.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
e_resp = await client.get(f"{cls.RDW_ENGINE}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if e_resp.status_code == 200 and e_resp.json():
|
||||
res["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception: pass
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def process_task(cls, db, task):
|
||||
clean_make = task.make.strip().upper()
|
||||
clean_model = task.model.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
offset = 0
|
||||
while True:
|
||||
params = f"merk={clean_make}"
|
||||
if clean_model != 'ALL_VARIANTS':
|
||||
params += f"&handelsbenaming={clean_model}"
|
||||
params += f"&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
|
||||
try:
|
||||
r = await client.get(f"{cls.RDW_MAIN}?{params}", headers=cls.HEADERS)
|
||||
batch = r.json() if r.status_code == 200 else []
|
||||
except Exception: break
|
||||
if not batch: break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
datum_eerste_toelating = str(item.get("datum_eerste_toelating", ""))
|
||||
year_from = cls.parse_int(datum_eerste_toelating[:4]) if len(datum_eerste_toelating) >= 4 else 0
|
||||
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
market='EU',
|
||||
make=clean_make,
|
||||
marketing_name=actual_model,
|
||||
normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"),
|
||||
version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
technical_code=plate,
|
||||
type_approval_number=item.get("typegoedkeuringsnummer"),
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")),
|
||||
doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
width=cls.parse_int(item.get("breedte")),
|
||||
wheelbase=cls.parse_int(item.get("wielbasis")),
|
||||
list_price=cls.parse_int(item.get("catalogusprijs")),
|
||||
max_speed=cls.parse_int(item.get("maximale_constructiesnelheid")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")),
|
||||
max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
fuel_consumption_combined=tech["consumption"],
|
||||
co2_emissions_combined=tech["co2"],
|
||||
vehicle_class=task.vehicle_class,
|
||||
body_type=item.get("inrichting"),
|
||||
fuel_type=tech["fuel_desc"],
|
||||
engine_capacity=cls.parse_int(item.get("cilinderinhoud")),
|
||||
power_kw=tech["power_kw"],
|
||||
cylinders=cls.parse_int(item.get("aantal_cilinders")),
|
||||
engine_code=tech["engine_code"],
|
||||
euro_classification=tech["euro_class"],
|
||||
year_from=year_from,
|
||||
priority_score=task.priority_score,
|
||||
status="unverified",
|
||||
source="MEGA-HUNTER-v2.1.2",
|
||||
# JAVÍTÁS: A raw_search_context most már üres STRING (''), ahogy a modell elvárja!
|
||||
raw_search_context='',
|
||||
research_metadata={},
|
||||
specifications={},
|
||||
marketing_name_aliases=[]
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type', 'market', 'year_from']
|
||||
)
|
||||
await db.execute(stmt)
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Sor hiba ({plate}): {e}")
|
||||
|
||||
await db.commit()
|
||||
offset += len(batch)
|
||||
if offset >= 500: break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task.id})
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v2.1.2 (Adattípus Fix) ONLINE")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
query = text("UPDATE vehicle.catalog_discovery SET status = 'processing' WHERE id = (SELECT id FROM vehicle.catalog_discovery WHERE status = 'pending' ORDER BY priority_score DESC FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id, make, model, vehicle_class, priority_score;")
|
||||
res = await db.execute(query)
|
||||
task = res.fetchone()
|
||||
await db.commit()
|
||||
if task: await cls.process_task(db, task)
|
||||
else: await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Főciklus hiba: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -1,205 +0,0 @@
|
||||
# /app/app/workers/vehicle/vehicle_robot_1_catalog_hunter.py
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Robot-1")
|
||||
|
||||
class CatalogHunter:
|
||||
"""
|
||||
Vehicle Robot 2.2.0: Fast-Track to Gold Edition
|
||||
Ha az RDW-ből megvan minden kulcsadat (kw, ccm, fuel), azonnal 'gold_enriched'-re teszi a járművet
|
||||
és beírja a vehicle_catalog mestertáblába!
|
||||
"""
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
if not text_val: return "UNKNOWN"
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower()
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError): return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError): return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
res = {"power_kw": 0, "engine_code": None, "euro_class": None, "fuel_desc": "Unknown", "co2": 0, "consumption": 0.0}
|
||||
try:
|
||||
f_resp = await client.get(f"{cls.RDW_FUEL}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1 = cls.parse_int(f.get("netto_maximum_vermogen"))
|
||||
p2 = cls.parse_int(f.get("nominaal_continu_maximum_vermogen"))
|
||||
res.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
e_resp = await client.get(f"{cls.RDW_ENGINE}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if e_resp.status_code == 200 and e_resp.json():
|
||||
res["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception: pass
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def process_task(cls, db, task):
|
||||
clean_make = task.make.strip().upper()
|
||||
clean_model = task.model.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
offset = 0
|
||||
while True:
|
||||
params = f"merk={clean_make}"
|
||||
if clean_model != 'ALL_VARIANTS':
|
||||
params += f"&handelsbenaming={clean_model}"
|
||||
params += f"&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
|
||||
try:
|
||||
r = await client.get(f"{cls.RDW_MAIN}?{params}", headers=cls.HEADERS)
|
||||
batch = r.json() if r.status_code == 200 else []
|
||||
except Exception: break
|
||||
if not batch: break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
datum_eerste_toelating = str(item.get("datum_eerste_toelating", ""))
|
||||
year_from = cls.parse_int(datum_eerste_toelating[:4]) if len(datum_eerste_toelating) >= 4 else 0
|
||||
|
||||
engine_ccm = cls.parse_int(item.get("cilinderinhoud"))
|
||||
power_kw = tech["power_kw"]
|
||||
fuel_type = tech["fuel_desc"]
|
||||
|
||||
# FAST-TRACK LOGIKA: Ha a kötelező műszaki adatok megvannak, azonnal ARANY minősítést kap!
|
||||
# Villanyautóknál a CCM lehet 0, ezt is kezeljük.
|
||||
is_gold = False
|
||||
if (power_kw > 0 and engine_ccm > 0) or (power_kw > 0 and "elektri" in fuel_type.lower()):
|
||||
is_gold = True
|
||||
|
||||
final_status = "gold_enriched" if is_gold else "unverified"
|
||||
|
||||
# 1. Beírjuk a VMD-be (Staging tábla)
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
market='EU',
|
||||
make=clean_make,
|
||||
marketing_name=actual_model,
|
||||
normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"),
|
||||
version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
technical_code=plate,
|
||||
type_approval_number=item.get("typegoedkeuringsnummer"),
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")),
|
||||
doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
width=cls.parse_int(item.get("breedte")),
|
||||
wheelbase=cls.parse_int(item.get("wielbasis")),
|
||||
list_price=cls.parse_int(item.get("catalogusprijs")),
|
||||
max_speed=cls.parse_int(item.get("maximale_constructiesnelheid")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")),
|
||||
max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
fuel_consumption_combined=tech["consumption"],
|
||||
co2_emissions_combined=tech["co2"],
|
||||
vehicle_class=task.vehicle_class,
|
||||
body_type=item.get("inrichting"),
|
||||
fuel_type=fuel_type,
|
||||
engine_capacity=engine_ccm,
|
||||
power_kw=power_kw,
|
||||
cylinders=cls.parse_int(item.get("aantal_cilinders")),
|
||||
engine_code=tech["engine_code"],
|
||||
euro_classification=tech["euro_class"],
|
||||
year_from=year_from,
|
||||
priority_score=task.priority_score,
|
||||
status=final_status, # Dinamikus státusz
|
||||
source="MEGA-HUNTER-v2.2.0-FAST",
|
||||
raw_search_context='',
|
||||
research_metadata={},
|
||||
specifications={"fast_track": True}, # Jelezzük, hogy ez RDW-ből jött közvetlenül
|
||||
marketing_name_aliases=[]
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type', 'market', 'year_from']
|
||||
).returning(VehicleModelDefinition.id)
|
||||
|
||||
res = await db.execute(stmt)
|
||||
vmd_id = res.scalar()
|
||||
|
||||
# 2. HA ARANY, AZONNAL LÉPÜNK A VÉGSŐ KATALÓGUSBA (Ahogy az Alchemist is tenné)
|
||||
if is_gold and vmd_id:
|
||||
cat_stmt = text("""
|
||||
INSERT INTO vehicle.vehicle_catalog
|
||||
(master_definition_id, make, model, power_kw, engine_capacity, fuel_type, factory_data)
|
||||
VALUES (:m_id, :make, :model, :kw, :ccm, :fuel, :factory)
|
||||
ON CONFLICT ON CONSTRAINT uix_vehicle_catalog_full DO NOTHING;
|
||||
""")
|
||||
await db.execute(cat_stmt, {
|
||||
"m_id": vmd_id,
|
||||
"make": clean_make,
|
||||
"model": actual_model[:50],
|
||||
"kw": power_kw,
|
||||
"ccm": engine_ccm,
|
||||
"fuel": fuel_type,
|
||||
"factory": json.dumps({"source": "RDW API Direct", "verified": True})
|
||||
})
|
||||
logger.info(f"✨ FAST-TRACK ARANY: {clean_make} {actual_model} (KW: {power_kw}, CCM: {engine_ccm})")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Sor hiba ({plate}): {e}")
|
||||
|
||||
await db.commit()
|
||||
offset += len(batch)
|
||||
if offset >= 500: break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task.id})
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v2.2.0 (Fast-Track Edition) ONLINE")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
query = text("UPDATE vehicle.catalog_discovery SET status = 'processing' WHERE id = (SELECT id FROM vehicle.catalog_discovery WHERE status = 'pending' ORDER BY priority_score DESC FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id, make, model, vehicle_class, priority_score;")
|
||||
res = await db.execute(query)
|
||||
task = res.fetchone()
|
||||
await db.commit()
|
||||
if task: await cls.process_task(db, task)
|
||||
else: await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Főciklus hiba: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -1,140 +0,0 @@
|
||||
import asyncio, httpx, logging, os, re, sys, json
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Robot-1")
|
||||
|
||||
class CatalogHunter:
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower() if text_val else "UNKNOWN"
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try: return int(float(value)) if value and str(value).strip() else 0
|
||||
except: return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try: return float(value) if value and str(value).strip() else 0.0
|
||||
except: return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
res = {"power_kw": 0, "engine_code": None, "euro_class": None, "fuel_desc": "Unknown", "co2": 0, "consumption": 0.0}
|
||||
try:
|
||||
f_resp = await client.get(f"{cls.RDW_FUEL}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1, p2 = cls.parse_int(f.get("netto_maximum_vermogen")), cls.parse_int(f.get("nominaal_continu_maximum_vermogen"))
|
||||
res.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
e_resp = await client.get(f"{cls.RDW_ENGINE}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if e_resp.status_code == 200 and e_resp.json():
|
||||
res["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception: pass
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def process_task(cls, db, task):
|
||||
clean_make, clean_model = task.make.strip().upper(), task.model.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
offset = 0
|
||||
while True:
|
||||
params = f"merk={clean_make}" + (f"&handelsbenaming={clean_model}" if clean_model != 'ALL_VARIANTS' else "") + f"&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
try:
|
||||
r = await client.get(f"{cls.RDW_MAIN}?{params}", headers=cls.HEADERS)
|
||||
batch = r.json() if r.status_code == 200 else []
|
||||
except Exception: break
|
||||
if not batch: break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
datum = str(item.get("datum_eerste_toelating", ""))
|
||||
year_from = cls.parse_int(datum[:4]) if len(datum) >= 4 else 0
|
||||
|
||||
engine_ccm, power_kw, fuel_type = cls.parse_int(item.get("cilinderinhoud")), tech["power_kw"], tech["fuel_desc"]
|
||||
|
||||
# FAST-TRACK LOGIKA: Ha van KW és CCM, egyből ARANY!
|
||||
is_gold = (power_kw > 0 and engine_ccm > 0) or (power_kw > 0 and "elektri" in fuel_type.lower())
|
||||
final_status = "gold_enriched" if is_gold else "unverified"
|
||||
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
market='EU', make=clean_make, marketing_name=actual_model, normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"), version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
technical_code=plate, type_approval_number=item.get("typegoedkeuringsnummer"),
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")), doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
width=cls.parse_int(item.get("breedte")), wheelbase=cls.parse_int(item.get("wielbasis")),
|
||||
list_price=cls.parse_int(item.get("catalogusprijs")), max_speed=cls.parse_int(item.get("maximale_constructiesnelheid")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")), max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
fuel_consumption_combined=tech["consumption"], co2_emissions_combined=tech["co2"],
|
||||
vehicle_class=task.vehicle_class, body_type=item.get("inrichting"), fuel_type=fuel_type,
|
||||
engine_capacity=engine_ccm, power_kw=power_kw, cylinders=cls.parse_int(item.get("aantal_cilinders")),
|
||||
engine_code=tech["engine_code"], euro_classification=tech["euro_class"], year_from=year_from,
|
||||
priority_score=task.priority_score, status=final_status, source="MEGA-HUNTER-v2.2.0-FAST",
|
||||
raw_search_context='', research_metadata={}, specifications={"fast_track": True} if is_gold else {}, marketing_name_aliases=[]
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type', 'market', 'year_from']
|
||||
).returning(VehicleModelDefinition.id)
|
||||
|
||||
res = await db.execute(stmt)
|
||||
vmd_id = res.scalar()
|
||||
|
||||
# Automatikus Publikálás (Ha Arany)
|
||||
if is_gold and vmd_id:
|
||||
cat_stmt = text("""
|
||||
INSERT INTO vehicle.vehicle_catalog (master_definition_id, make, model, power_kw, engine_capacity, fuel_type, factory_data)
|
||||
VALUES (:m_id, :make, :model, :kw, :ccm, :fuel, :factory)
|
||||
ON CONFLICT ON CONSTRAINT uix_vehicle_catalog_full DO NOTHING;
|
||||
""")
|
||||
await db.execute(cat_stmt, {"m_id": vmd_id, "make": clean_make, "model": actual_model[:50], "kw": power_kw, "ccm": engine_ccm, "fuel": fuel_type, "factory": '{"source": "RDW Fast-Track"}'})
|
||||
logger.info(f"✨ FAST-TRACK ARANY: {clean_make} {actual_model}")
|
||||
|
||||
except Exception as e: logger.warning(f"⚠️ Sor hiba ({plate}): {e}")
|
||||
|
||||
await db.commit()
|
||||
offset += len(batch)
|
||||
if offset >= 500: break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task.id})
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v2.2.0 (Fast-Track) ONLINE")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
res = await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processing' WHERE id = (SELECT id FROM vehicle.catalog_discovery WHERE status = 'pending' ORDER BY priority_score DESC FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id, make, model, vehicle_class, priority_score;"))
|
||||
task = res.fetchone()
|
||||
await db.commit()
|
||||
if task: await cls.process_task(db, task)
|
||||
else: await asyncio.sleep(30)
|
||||
except Exception: await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -1,239 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/workers/vehicle/vehicle_robot_2_researcher.py
|
||||
import asyncio
|
||||
import logging
|
||||
import warnings
|
||||
import os
|
||||
import json
|
||||
from datetime import datetime
|
||||
from sqlalchemy import text, update, func
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning, module='duckduckgo_search')
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
# MB 2.0 Szabvány naplózás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-2-Researcher: %(message)s')
|
||||
logger = logging.getLogger("Vehicle-Robot-2-Researcher")
|
||||
|
||||
class QuotaManager:
|
||||
""" Szigorú napi limit figyelő a fizetős/hatósági API-khoz """
|
||||
def __init__(self, service_name: str, daily_limit: int):
|
||||
self.service_name = service_name
|
||||
self.daily_limit = daily_limit
|
||||
self.state_file = f"/app/temp/.quota_{service_name}.json"
|
||||
self._ensure_file()
|
||||
|
||||
def _ensure_file(self):
|
||||
os.makedirs(os.path.dirname(self.state_file), exist_ok=True)
|
||||
if not os.path.exists(self.state_file):
|
||||
with open(self.state_file, 'w') as f:
|
||||
json.dump({"date": datetime.now().strftime("%Y-%m-%d"), "count": 0}, f)
|
||||
|
||||
def can_make_request(self) -> bool:
|
||||
with open(self.state_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
if data["date"] != today:
|
||||
data = {"date": today, "count": 0} # Új nap, kvóta nullázása
|
||||
|
||||
if data["count"] >= self.daily_limit:
|
||||
return False
|
||||
|
||||
# Növeljük a számlálót
|
||||
data["count"] += 1
|
||||
with open(self.state_file, 'w') as f:
|
||||
json.dump(data, f)
|
||||
return True
|
||||
|
||||
class VehicleResearcher:
|
||||
"""
|
||||
Vehicle Robot 2.5: Sniper Researcher (Mesterlövész Adatgyűjtő)
|
||||
Célzott keresésekkel és strukturált aktakészítéssel dolgozik az AI kímélése érdekében.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.max_attempts = 5
|
||||
self.search_timeout = 15.0
|
||||
|
||||
# Kvóta menedzserek beállítása (.env-ből olvasva)
|
||||
dvla_limit = int(os.getenv("DVLA_DAILY_LIMIT", "1000"))
|
||||
self.dvla_quota = QuotaManager("dvla", dvla_limit)
|
||||
self.dvla_token = os.getenv("DVLA_API_KEY")
|
||||
|
||||
async def fetch_ddg_targeted(self, label: str, query: str) -> str:
|
||||
""" Célzott keresés szálbiztosan a DuckDuckGo-n. """
|
||||
try:
|
||||
def search():
|
||||
with DDGS() as ddgs:
|
||||
# max_results=2: Nem kell sok zaj, csak a legrelevánsabb 2 találat
|
||||
results = ddgs.text(query, max_results=2)
|
||||
return [f"- {r.get('body', '')}" for r in results] if results else []
|
||||
|
||||
results = await asyncio.wait_for(asyncio.to_thread(search), timeout=self.search_timeout)
|
||||
|
||||
if not results:
|
||||
return f"[SOURCE: {label}]\nNincs érdemi találat.\n"
|
||||
|
||||
content = f"[SOURCE: {label} | KERESÉS: {query}]\n"
|
||||
content += "\n".join(results) + "\n"
|
||||
return content
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Keresési hiba ({label}): {e}")
|
||||
return f"[SOURCE: {label}]\nKERESÉSI HIBA.\n"
|
||||
|
||||
def extract_specs_from_text(self, text: str) -> dict:
|
||||
""" Regex alapú kinyerés a nyers szövegből: ccm, kW, motoradatok. """
|
||||
import re
|
||||
specs = {}
|
||||
|
||||
# CCM (köbcentiméter) minta: 1998 cc, 2.0 L, 2000 cm³
|
||||
ccm_pattern = r'(\d{3,4})\s*(?:cc|ccm|cm³|cm3|cc\.)'
|
||||
match = re.search(ccm_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['ccm'] = int(match.group(1))
|
||||
else:
|
||||
# Alternatív minta: 2.0 liter -> 2000 cc
|
||||
liter_pattern = r'(\d+\.?\d*)\s*(?:L|liter|ℓ)'
|
||||
match = re.search(liter_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
liters = float(match.group(1))
|
||||
specs['ccm'] = int(liters * 1000)
|
||||
|
||||
# KW (kilowatt) minta: 150 kW, 150kW, 150 KW
|
||||
kw_pattern = r'(\d{2,4})\s*(?:kW|kw|KW)'
|
||||
match = re.search(kw_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['kw'] = int(match.group(1))
|
||||
else:
|
||||
# Le (lóerő) átváltás: 150 LE -> 110 kW (kb)
|
||||
hp_pattern = r'(\d{2,4})\s*(?:HP|hp|LE|le|Ps)'
|
||||
match = re.search(hp_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
hp = int(match.group(1))
|
||||
specs['kw'] = int(hp * 0.7355) # hozzávetőleges átváltás
|
||||
|
||||
# Motor kód minta: motor kód: 1.8 TSI, engine code: N47
|
||||
engine_pattern = r'(?:motor\s*kód|engine\s*code|motor\s*code)[:\s]+([A-Z0-9\.\- ]+)'
|
||||
match = re.search(engine_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['engine_code'] = match.group(1).strip()
|
||||
|
||||
return specs
|
||||
|
||||
async def research_vehicle(self, db, vehicle_id: int, make: str, model: str, engine: str, year: str, current_attempts: int):
|
||||
""" Egy jármű átvilágítása és a strukturált 'Akta' elkészítése a GPU számára. """
|
||||
engine_safe = engine or ""
|
||||
year_safe = str(year) if year else ""
|
||||
|
||||
logger.info(f"🔎 Mesterlövész Kutatás: {make} {model} (Motor: {engine_safe})")
|
||||
|
||||
# 1. TIER: Ingyenes, Célzott Keresések (A legmegbízhatóbb források)
|
||||
queries = [
|
||||
("ULTIMATE_SPECS", f"{make} {model} {engine_safe} {year_safe} site:ultimatespecs.com"),
|
||||
("AUTO_DATA", f"{make} {model} {engine_safe} {year_safe} site:auto-data.net"),
|
||||
("COMMON_ISSUES", f"{make} {model} {engine_safe} reliability common problems")
|
||||
]
|
||||
|
||||
tasks = [self.fetch_ddg_targeted(label, q) for label, q in queries]
|
||||
search_results = await asyncio.gather(*tasks)
|
||||
|
||||
# 2. TIER: Fizetős / Kvótás API-k (Példa a DVLA helyére)
|
||||
# Ha a jövőben bejön brit rendszám, itt hívjuk meg a DVLA-t:
|
||||
# if has_uk_plate and self.dvla_quota.can_make_request():
|
||||
# uk_data = await self.fetch_dvla_data(plate)
|
||||
# search_results.append(uk_data)
|
||||
|
||||
# 3. ÖSSZESÍTÉS (Az Akta összeállítása)
|
||||
# Maximalizáljuk a szöveg hosszát, hogy az AI GPU ne fulladjon le!
|
||||
full_context = "\n".join(search_results)
|
||||
if len(full_context) > 2500:
|
||||
full_context = full_context[:2500] + "\n...[TRUNCATED TO SAVE GPU TOKENS]"
|
||||
|
||||
# Regex alapú specifikáció kinyerés
|
||||
extracted_specs = self.extract_specs_from_text(full_context)
|
||||
|
||||
try:
|
||||
if len(full_context.strip()) > 150: # Csökkentettük az elvárást, mert a célzott keresés tömörebb
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == vehicle_id)
|
||||
.values(
|
||||
raw_search_context=full_context,
|
||||
research_metadata=extracted_specs,
|
||||
status='awaiting_ai_synthesis', # Kész az Akta, mehet az Alkimistának!
|
||||
last_research_at=func.now(),
|
||||
attempts=current_attempts + 1
|
||||
)
|
||||
)
|
||||
logger.info(f"✅ Akta rögzítve ({len(full_context)} karakter): {make} {model}")
|
||||
else:
|
||||
new_status = 'suspended_research' if current_attempts + 1 >= self.max_attempts else 'unverified'
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == vehicle_id)
|
||||
.values(
|
||||
status=new_status,
|
||||
attempts=current_attempts + 1,
|
||||
last_research_at=func.now()
|
||||
)
|
||||
)
|
||||
if new_status == 'suspended_research':
|
||||
logger.warning(f"🛑 Felfüggesztve (Nincs nyom a weben): {make} {model}")
|
||||
else:
|
||||
logger.warning(f"⚠️ Kevés adat: {make} {model}, visszatéve a sorba.")
|
||||
|
||||
await db.commit()
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(f"🚨 Adatbázis hiba az eredmény mentésénél ({vehicle_id}): {e}")
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
self_instance = cls()
|
||||
logger.info("🚀 Vehicle Researcher 2.5 ONLINE (Sniper & Quota Manager)")
|
||||
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# ATOMI ZÁROLÁS
|
||||
query = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = 'research_in_progress'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.vehicle_model_definitions
|
||||
WHERE status IN ('unverified', 'awaiting_research', 'ACTIVE')
|
||||
AND attempts < :max_attempts
|
||||
AND is_manual = FALSE
|
||||
ORDER BY
|
||||
CASE WHEN make = 'TOYOTA' THEN 1 ELSE 2 END,
|
||||
attempts ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, make, marketing_name, engine_code, year_from, attempts;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {"max_attempts": self_instance.max_attempts})
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
|
||||
if task:
|
||||
v_id, v_make, v_model, v_engine, v_year, v_attempts = task
|
||||
async with AsyncSessionLocal() as process_db:
|
||||
await self_instance.research_vehicle(process_db, v_id, v_make, v_model, v_engine, v_year, v_attempts)
|
||||
|
||||
await asyncio.sleep(2) # Rate limit védelem a DDG felé
|
||||
else:
|
||||
await asyncio.sleep(30)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(VehicleResearcher.run())
|
||||
except KeyboardInterrupt:
|
||||
logger.info("🛑 Kutató robot leállítva.")
|
||||
@@ -1,225 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/workers/vehicle/vehicle_robot_3_alchemist_pro.py
|
||||
import asyncio
|
||||
import logging
|
||||
import datetime
|
||||
import random
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from sqlalchemy import text, func, update, case
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
from app.models.asset import AssetCatalog
|
||||
from app.services.ai_service import AIService
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Vehicle-Alchemist-Pro: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Vehicle-Robot-3-Alchemist-Pro")
|
||||
|
||||
class TechEnricher:
|
||||
"""
|
||||
Vehicle Robot 3: Alchemist Pro (Atomi Zárolás + Kézi Moderáció Patch)
|
||||
Tiszta GPU fókusz: Csak az AI elemzésre és adategyesítésre koncentrál.
|
||||
Nincs felesleges webkeresés. Szigorú, de intelligens Sane-Check.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.max_attempts = 5
|
||||
self.daily_ai_limit = int(os.getenv("AI_DAILY_LIMIT", "10000"))
|
||||
self.ai_calls_today = 0
|
||||
self.last_reset_date = datetime.date.today()
|
||||
|
||||
def check_budget(self) -> bool:
|
||||
if datetime.date.today() > self.last_reset_date:
|
||||
self.ai_calls_today = 0
|
||||
self.last_reset_date = datetime.date.today()
|
||||
return self.ai_calls_today < self.daily_ai_limit
|
||||
|
||||
def validate_merged_data(self, merged_kw: int, merged_ccm: int, v_class: str, fuel: str, current_attempts: int) -> tuple[bool, str]:
|
||||
""" Intelligens validáció a MERGE után. Visszaadja a státuszt és a hiba okát. """
|
||||
if merged_ccm > 18000:
|
||||
return False, f"Irreális CCM érték ({merged_ccm})"
|
||||
if merged_kw > 1500 and v_class != "truck":
|
||||
return False, f"Irreális KW érték ({merged_kw})"
|
||||
|
||||
# Ha hiányzik a KW
|
||||
if merged_kw == 0:
|
||||
if current_attempts < 3:
|
||||
return False, "Hiányzó KW adat. Újrakutatás javasolt."
|
||||
else:
|
||||
logger.warning("Sane-check: Többszöri próbálkozás után sincs KW, de átengedjük részlegesként.")
|
||||
|
||||
# Ha hiányzik a CCM (és belsőégésű)
|
||||
if merged_ccm == 0 and "electric" not in fuel and "elektric" not in fuel and v_class != "trailer":
|
||||
if current_attempts < 3:
|
||||
return False, "Hiányzó CCM (belsőégésű motornál). Újrakutatás javasolt."
|
||||
else:
|
||||
logger.warning("Sane-check: Többszöri próbálkozás után sincs CCM, átengedjük részlegesként.")
|
||||
|
||||
return True, "OK"
|
||||
|
||||
async def process_single_record(self, db, record_id: int, base_info: dict, current_attempts: int):
|
||||
# Pontos azonosító a logokhoz (Márka, Modell, ID, RDW adatok)
|
||||
v_ident = f"{base_info['make'].upper()} {base_info['m_name']} (ID: {record_id}, RDW: {base_info['rdw_ccm']}ccm, KW: {base_info['rdw_kw']})"
|
||||
attempt_str = f"[Próba: {current_attempts + 1}/{self.max_attempts}]"
|
||||
|
||||
ai_data = {} # Üres dict, ha az AI hívás elszállna
|
||||
|
||||
try:
|
||||
logger.info(f"🧠 AI dúsítás indul: {v_ident} {attempt_str}")
|
||||
|
||||
# 1. LÉPÉS: AI Hívás (Rábízzuk az adatokat a modellre)
|
||||
ai_data = await AIService.get_clean_vehicle_data(
|
||||
base_info['make'],
|
||||
base_info['m_name'],
|
||||
base_info
|
||||
)
|
||||
|
||||
if not ai_data:
|
||||
raise ValueError("Teljesen üres AI válasz (API hiba vagy extrém hallucináció).")
|
||||
|
||||
# 2. LÉPÉS: HIBRID MERGE (Még a validáció előtt!)
|
||||
# Az RDW adatok felülbírálják az AI-t a hatósági paramétereknél
|
||||
final_kw = base_info['rdw_kw'] if base_info['rdw_kw'] > 0 else int(ai_data.get("kw", 0) or 0)
|
||||
final_ccm = base_info['rdw_ccm'] if base_info['rdw_ccm'] > 0 else int(ai_data.get("ccm", 0) or 0)
|
||||
|
||||
# Üzemanyag tisztítása
|
||||
fuel_rdw = base_info.get('rdw_fuel', '')
|
||||
final_fuel = fuel_rdw if fuel_rdw and fuel_rdw != "Unknown" else ai_data.get("fuel_type", "petrol")
|
||||
|
||||
final_engine = base_info['rdw_engine'] if base_info['rdw_engine'] else ai_data.get("engine_code", "Unknown")
|
||||
final_euro = base_info['rdw_euro'] or ai_data.get("euro_classification")
|
||||
final_cylinders = base_info['rdw_cylinders'] or ai_data.get("cylinders")
|
||||
|
||||
# 3. LÉPÉS: Intelligens Validáció
|
||||
is_valid, error_msg = self.validate_merged_data(final_kw, final_ccm, base_info['v_type'], final_fuel.lower(), current_attempts)
|
||||
if not is_valid:
|
||||
raise ValueError(f"Validációs hiba: {error_msg}")
|
||||
|
||||
# 4. LÉPÉS: Mentés az Arany Katalógusba
|
||||
clean_model = str(ai_data.get("marketing_name", base_info['m_name']))[:50].upper()
|
||||
|
||||
cat_stmt = text("""
|
||||
INSERT INTO vehicle.vehicle_catalog
|
||||
(master_definition_id, make, model, power_kw, engine_capacity, fuel_type, factory_data)
|
||||
VALUES (:m_id, :make, :model, :kw, :ccm, :fuel, :factory)
|
||||
ON CONFLICT ON CONSTRAINT uix_vehicle_catalog_full DO NOTHING
|
||||
RETURNING id;
|
||||
""")
|
||||
|
||||
await db.execute(cat_stmt, {
|
||||
"m_id": record_id,
|
||||
"make": base_info['make'].upper(),
|
||||
"model": clean_model,
|
||||
"kw": final_kw,
|
||||
"ccm": final_ccm,
|
||||
"fuel": final_fuel,
|
||||
"factory": json.dumps(ai_data)
|
||||
})
|
||||
|
||||
# 5. LÉPÉS: Staging tábla (VMD) lezárása
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == record_id)
|
||||
.values(
|
||||
status="gold_enriched",
|
||||
engine_capacity=final_ccm,
|
||||
power_kw=final_kw,
|
||||
fuel_type=final_fuel,
|
||||
engine_code=final_engine,
|
||||
euro_classification=final_euro,
|
||||
cylinders=final_cylinders,
|
||||
specifications=ai_data, # Elmentjük az AI teljes outputját a mestertáblába is
|
||||
updated_at=func.now()
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
logger.info(f"✨ ARANY REKORD KÉSZ: {v_ident}")
|
||||
self.ai_calls_today += 1
|
||||
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.warning(f"⚠️ Alkimista hiba - {v_ident}: {e}")
|
||||
|
||||
# Ha elértük a limitet, KÉZI MODERÁCIÓRA küldjük, egyébként vissza a Kutatónak
|
||||
new_status = 'manual_review_needed' if current_attempts + 1 >= self.max_attempts else 'unverified'
|
||||
|
||||
# Elmentjük az AI részleges válaszát (vagy a hibát), hogy az admin lássa, mit rontott el a gép
|
||||
review_data = ai_data if ai_data else {"error": "Nincs értékelhető JSON adat az AI-tól", "raw_context": base_info['web_context']}
|
||||
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == record_id)
|
||||
.values(
|
||||
attempts=current_attempts + 1,
|
||||
last_error=str(e)[:200],
|
||||
status=new_status,
|
||||
specifications=review_data, # Kézi ellenőrzéshez beírjuk a törött adatot!
|
||||
updated_at=func.now()
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
if new_status == 'unverified':
|
||||
logger.info(f"♻️ Akta visszaküldve a Robot-2-nek (Kutató). {attempt_str}")
|
||||
else:
|
||||
logger.error(f"🛑 Max próbálkozás elérve! Kézi moderációra küldve: {v_ident}")
|
||||
|
||||
async def run(self):
|
||||
logger.info(f"🚀 Alchemist Pro HIBRID ONLINE (Atomi Zárolás + Moderáció Patch)")
|
||||
while True:
|
||||
if not self.check_budget():
|
||||
logger.warning("💸 Napi AI limit kimerítve! Pihenés...")
|
||||
await asyncio.sleep(3600); continue
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# ATOMI ZÁROLÁS (A "Szent Grál" a race condition ellen)
|
||||
query = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = 'ai_synthesis_in_progress'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.vehicle_model_definitions
|
||||
WHERE status IN ('awaiting_ai_synthesis', 'ACTIVE')
|
||||
AND attempts < :max_attempts
|
||||
AND is_manual = FALSE
|
||||
ORDER BY
|
||||
CASE WHEN status = 'awaiting_ai_synthesis' THEN 1 ELSE 2 END,
|
||||
priority_score DESC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, make, marketing_name, vehicle_class, power_kw, engine_capacity,
|
||||
fuel_type, engine_code, euro_classification, cylinders, raw_search_context, attempts;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {"max_attempts": self.max_attempts})
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
|
||||
if task:
|
||||
# Szétbontjuk a lekérdezett rekordot a base_info dict-be
|
||||
r_id = task[0]
|
||||
base_info = {
|
||||
"make": task[1], "m_name": task[2], "v_type": task[3] or "car",
|
||||
"rdw_kw": task[4] or 0, "rdw_ccm": task[5] or 0,
|
||||
"rdw_fuel": task[6] or "petrol", "rdw_engine": task[7] or "",
|
||||
"rdw_euro": task[8], "rdw_cylinders": task[9],
|
||||
"web_context": task[10] or ""
|
||||
}
|
||||
attempts = task[11]
|
||||
|
||||
# Külön adatbázis kapcsolat a feldolgozáshoz (hosszú AI hívás miatt)
|
||||
async with AsyncSessionLocal() as process_db:
|
||||
await self.process_single_record(process_db, r_id, base_info, attempts)
|
||||
|
||||
# GPU hűtés / Ollama rate limit
|
||||
await asyncio.sleep(random.uniform(1.5, 3.5))
|
||||
else:
|
||||
logger.info("😴 Nincs feldolgozandó akta, az Alkimista pihen...")
|
||||
await asyncio.sleep(15)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(TechEnricher().run())
|
||||
@@ -1,168 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import datetime
|
||||
import random
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from sqlalchemy import text, func, update
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
from app.services.ai_service import AIService
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] R3-Alchemist: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Robot-3-Alchemist")
|
||||
|
||||
class TechEnricher:
|
||||
"""
|
||||
Vehicle Robot 3: Alchemist Pro (Sentinel Gateway Edition)
|
||||
Az AIService 2.2-t használja (Ollama -> Groq Fallback).
|
||||
Kinyeri a felszereltségi szintet (trim_level) és pótolja a hiányzó adatokat.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.max_attempts = 5
|
||||
self.daily_ai_limit = int(os.getenv("AI_DAILY_LIMIT", "10000"))
|
||||
self.ai_calls_today = 0
|
||||
self.last_reset_date = datetime.date.today()
|
||||
|
||||
def check_budget(self) -> bool:
|
||||
if datetime.date.today() > self.last_reset_date:
|
||||
self.ai_calls_today = 0
|
||||
self.last_reset_date = datetime.date.today()
|
||||
return self.ai_calls_today < self.daily_ai_limit
|
||||
|
||||
def validate_merged_data(self, merged_kw: int, merged_ccm: int, v_class: str, fuel: str, current_attempts: int) -> tuple[bool, str]:
|
||||
if merged_ccm > 18000:
|
||||
return False, f"Irreális CCM érték ({merged_ccm})"
|
||||
if merged_kw > 1500 and v_class not in ["truck", "other"]:
|
||||
return False, f"Irreális KW érték ({merged_kw})"
|
||||
|
||||
if merged_kw == 0 and current_attempts < 3:
|
||||
return False, "Hiányzó KW adat. Újrakutatás javasolt."
|
||||
|
||||
if merged_ccm == 0 and "elektr" not in fuel.lower() and v_class != "trailer" and current_attempts < 3:
|
||||
return False, "Hiányzó CCM (belsőégésű motornál)."
|
||||
|
||||
return True, "OK"
|
||||
|
||||
async def process_single_record(self, db, record_id: int, base_info: dict, current_attempts: int):
|
||||
v_ident = f"{base_info['make'].upper()} {base_info['m_name']} (ID: {record_id})"
|
||||
attempt_str = f"[Próba: {current_attempts + 1}/{self.max_attempts}]"
|
||||
|
||||
try:
|
||||
logger.info(f"🧠 AI dúsítás indul: {v_ident} {attempt_str}")
|
||||
|
||||
# Szigorú Prompt a Master AI Service-nek
|
||||
prompt = f"""
|
||||
Elemezd az alábbi járműadatokat és a webes kutatást! Készíts belőle egy JSON objektumot.
|
||||
Jármű: {base_info['make']} {base_info['m_name']}
|
||||
Hatósági adatok: {base_info['rdw_ccm']} ccm, {base_info['rdw_kw']} kW, Üzemanyag: {base_info['rdw_fuel']}
|
||||
Webes szöveg: {base_info['web_context'][:2000]}
|
||||
|
||||
FELADATOK:
|
||||
1. Keresd meg a felszereltségi szintet (trim_level) a modell nevéből vagy a szövegből (pl. AMG, Highline, Titanium, M-Sport, Elegance, ST-Line). Ha nincs, legyen üres string.
|
||||
2. Ha az RDW adatokban a kW vagy a ccm 0, pótold a szövegből a helyes értéket!
|
||||
|
||||
KIZÁRÓLAG EGY ÉRVÉNYES JSON-T ADJ VISSZA! (A Groq/Gemini miatt kötelező a JSON szó használata).
|
||||
Várt kulcsok: "kw" (int), "ccm" (int), "trim_level" (string), "transmission" (string), "drive_type" (string).
|
||||
"""
|
||||
|
||||
# Hívjuk a te profi Gateway-edet! (_execute_ai_call átveszi a db session-t is a beállításokhoz)
|
||||
ai_data = await AIService._execute_ai_call(db, prompt, model_key="text")
|
||||
|
||||
if not ai_data:
|
||||
raise ValueError("Üres AI válasz (Minden fallback elbukott).")
|
||||
|
||||
# HIBRID MERGE
|
||||
final_kw = base_info['rdw_kw'] if base_info['rdw_kw'] > 0 else int(ai_data.get("kw", 0) or 0)
|
||||
final_ccm = base_info['rdw_ccm'] if base_info['rdw_ccm'] > 0 else int(ai_data.get("ccm", 0) or 0)
|
||||
trim_level = str(ai_data.get("trim_level", ""))[:100]
|
||||
|
||||
# Sane-Check
|
||||
is_valid, error_msg = self.validate_merged_data(final_kw, final_ccm, base_info['v_type'], base_info['rdw_fuel'], current_attempts)
|
||||
if not is_valid:
|
||||
raise ValueError(f"Validációs hiba: {error_msg}")
|
||||
|
||||
# Staging tábla frissítése (Arany minősítés)
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == record_id)
|
||||
.values(
|
||||
status="gold_enriched",
|
||||
engine_capacity=final_ccm,
|
||||
power_kw=final_kw,
|
||||
trim_level=trim_level if trim_level.lower() not in ["null", "none"] else "",
|
||||
specifications=ai_data,
|
||||
updated_at=func.now()
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
logger.info(f"✨ ARANY REKORD KÉSZ: {v_ident} | Trim: {trim_level}")
|
||||
self.ai_calls_today += 1
|
||||
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.warning(f"⚠️ Alkimista hiba - {v_ident}: {e}")
|
||||
|
||||
new_status = 'manual_review_needed' if current_attempts + 1 >= self.max_attempts else 'unverified'
|
||||
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == record_id)
|
||||
.values(
|
||||
attempts=current_attempts + 1,
|
||||
last_error=str(e)[:200],
|
||||
status=new_status,
|
||||
updated_at=func.now()
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
if new_status == 'unverified':
|
||||
logger.info(f"♻️ Akta visszaküldve a Kutatónak (R2). {attempt_str}")
|
||||
|
||||
async def run(self):
|
||||
logger.info(f"🚀 R3 Alchemist Pro ONLINE (Sentinel Gateway Integráció)")
|
||||
while True:
|
||||
if not self.check_budget():
|
||||
logger.warning("💸 Napi AI limit kimerítve! Pihenés...")
|
||||
await asyncio.sleep(3600); continue
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
query = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = 'ai_synthesis_in_progress'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.vehicle_model_definitions
|
||||
WHERE status = 'awaiting_ai_synthesis'
|
||||
AND attempts < :max_attempts
|
||||
AND is_manual = FALSE
|
||||
ORDER BY priority_score DESC
|
||||
FOR UPDATE SKIP LOCKED LIMIT 1
|
||||
)
|
||||
RETURNING id, make, marketing_name, vehicle_class, power_kw, engine_capacity, fuel_type, raw_search_context, attempts;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {"max_attempts": self.max_attempts})
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
|
||||
if task:
|
||||
base_info = {
|
||||
"make": task[1], "m_name": task[2], "v_type": task[3] or "car",
|
||||
"rdw_kw": task[4] or 0, "rdw_ccm": task[5] or 0,
|
||||
"rdw_fuel": task[6] or "petrol", "web_context": task[7] or ""
|
||||
}
|
||||
async with AsyncSessionLocal() as process_db:
|
||||
await self.process_single_record(process_db, task[0], base_info, task[8])
|
||||
|
||||
else:
|
||||
await asyncio.sleep(10)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(TechEnricher().run())
|
||||
@@ -1,113 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def test_scraper():
|
||||
# Két probléma-fókuszú URL: a modern Aprilia és a régi, hibás HTML-ű BMW
|
||||
test_urls = [
|
||||
"https://www.autoevolution.com/moto/aprilia-rs-660-factory-2025.html",
|
||||
"https://www.autoevolution.com/moto/bmw-f-650-gs-2011.html"
|
||||
]
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
context = await browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
for url in test_urls:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"🌍 MEGNYITÁS: {url}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# A DOM betöltése megvárása
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
||||
await asyncio.sleep(2) # Várunk picit a JS futásra
|
||||
|
||||
# A TÖKÉLETESÍTETT AUTOEVOLUTION PARSZOLÓ
|
||||
script = """
|
||||
() => {
|
||||
let results = {};
|
||||
|
||||
// 1. MÓDSZER: Régi motorok (pl. BMW F650GS) -> td.left és td.right
|
||||
let leftCells = document.querySelectorAll('td.left');
|
||||
leftCells.forEach(cell => {
|
||||
let key = cell.innerText.replace(/:$/, '').trim();
|
||||
let rightCell = cell.nextElementSibling;
|
||||
if(rightCell && rightCell.classList.contains('right')) {
|
||||
results[key] = rightCell.innerText.trim();
|
||||
}
|
||||
});
|
||||
|
||||
// 2. MÓDSZER: Modern motorok (pl. Aprilia) -> dt és dd
|
||||
let dts = document.querySelectorAll('dt');
|
||||
dts.forEach(dt => {
|
||||
let key = dt.innerText.replace(/:$/, '').trim();
|
||||
let dd = dt.nextElementSibling;
|
||||
if(dd && dd.tagName.toLowerCase() === 'dd') {
|
||||
results[key] = dd.innerText.trim();
|
||||
}
|
||||
});
|
||||
|
||||
// 3. MÓDSZER: Alternatív modern layout -> span.label és span.value
|
||||
let specRows = document.querySelectorAll('.spec-row');
|
||||
specRows.forEach(row => {
|
||||
let label = row.querySelector('.label');
|
||||
let value = row.querySelector('.value');
|
||||
if(label && value) {
|
||||
let key = label.innerText.replace(/:$/, '').trim();
|
||||
if (!results[key]) {
|
||||
results[key] = value.innerText.trim();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 4. MÓDSZER: "Adler" típusú elavult leírások fallbackje -> Vastagított szöveg
|
||||
if (Object.keys(results).length === 0) {
|
||||
document.querySelectorAll('b, strong').forEach(b => {
|
||||
let key = b.innerText.replace(/:$/, '').trim();
|
||||
if(key.length > 2 && key.length < 30) {
|
||||
let val = "";
|
||||
// Ha a szöveg közvetlenül a tag után van (Text Node)
|
||||
if(b.nextSibling && b.nextSibling.nodeType === 3) {
|
||||
val = b.nextSibling.textContent.trim();
|
||||
}
|
||||
// Ha egy másik elemben van
|
||||
else if (b.nextElementSibling && b.nextElementSibling.tagName !== 'B') {
|
||||
val = b.nextElementSibling.innerText.trim();
|
||||
}
|
||||
if(val && !results[key]) {
|
||||
results[key] = val;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
"""
|
||||
|
||||
data = await page.evaluate(script)
|
||||
|
||||
if data and len(data) > 0:
|
||||
# Kiszűrjük a zajt, csak a releváns műszaki adatokat hagyjuk meg
|
||||
relevant_keys = ["Type", "Displacement", "Bore X Stroke", "Compression Ratio",
|
||||
"Horsepower", "Torque", "Fuel System", "Gearbox", "Clutch",
|
||||
"Final Drive", "Frame", "Front Suspension", "Rear Suspension",
|
||||
"Front Brake", "Rear Brake", "Overall Length", "Overall Width",
|
||||
"Seat Height", "Wheelbase", "Fuel Capacity", "Weight", "Dry Weight",
|
||||
"Wet Weight", "Front", "Rear"]
|
||||
|
||||
filtered_data = {k: v for k, v in data.items() if any(rk.lower() in k.lower() for rk in relevant_keys)}
|
||||
|
||||
print("\n🟢 KINYERT ADATOK (DOM PARSZOLÓ):")
|
||||
print(json.dumps(filtered_data if filtered_data else data, indent=2, ensure_ascii=False))
|
||||
print(f"\n✅ Összesen {len(filtered_data if filtered_data else data)} műszaki paramétert találtam.")
|
||||
else:
|
||||
print("\n🔴 NULLA ADAT - A DOM parszoló nem talált egyezést.")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_scraper())
|
||||
@@ -1,113 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import requests
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
# --- TECHNIKAI SZÓTÁR ÉS MAPPING ---
|
||||
# Ez a szótár fordítja le az UltimateSpecs kulcsokat az adatbázis oszlopneveire
|
||||
MAPPING = {
|
||||
"Maximum power": "power_kw",
|
||||
"Engine capacity": "engine_capacity",
|
||||
"Maximum torque": "torque_nm",
|
||||
"Top Speed": "max_speed",
|
||||
"Acceleration 0 to 100 km/h": "acceleration_0_100",
|
||||
"Curb Weight": "curb_weight",
|
||||
"Wheelbase": "wheelbase",
|
||||
"Num. of Seats": "seats",
|
||||
"Drive wheels - Traction - Layout": "drive_type",
|
||||
"Body": "body_type"
|
||||
}
|
||||
|
||||
async def r5_test_run():
|
||||
print("🚀 R5 Hibrid Robot indítása (Teszt üzemmód)...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 1. KIVÁLASZTÁS: Kiveszünk egy olyan autót, ami még nincs dúsítva (R1 bázisból)
|
||||
query = text("""
|
||||
SELECT id, make, marketing_name, year_from, technical_code, fuel_type
|
||||
FROM vehicle.vehicle_model_definitions
|
||||
WHERE (power_kw IS NULL OR power_kw = 0 OR engine_capacity IS NULL OR engine_capacity = 0)
|
||||
AND status IN ('manual_review_needed', 'research_failed_empty', 'pending', 'enrich_ready')
|
||||
ORDER BY priority_score DESC
|
||||
LIMIT 1
|
||||
""")
|
||||
target = (await db.execute(query)).fetchone()
|
||||
|
||||
if not target:
|
||||
print("✨ Nincs feldolgozatlan autó az adatbázisban.")
|
||||
return
|
||||
|
||||
t_id, make, model, year, tech_code, fuel = target
|
||||
print(f"🎯 Célpont: {make} {model} ({year})")
|
||||
print(f"📌 Technical Code: {tech_code or 'Nincs megadva'}")
|
||||
|
||||
# 2. RDW ADATOK (Holland hatósági bázis)
|
||||
# Ha van technical_code (pl. Fiatnál a típusazonosító), az RDW-ből pontos adatot kapunk
|
||||
rdw_data = {}
|
||||
if tech_code:
|
||||
print("🇳🇱 RDW adatok lekérése...")
|
||||
# Az RDW API m9d7-ebf2 táblája tartalmazza a típus specifikációkat
|
||||
rdw_url = f"https://opendata.rdw.nl/resource/m9d7-ebf2.json?handelsbenaming={tech_code.upper()}"
|
||||
try:
|
||||
res = requests.get(rdw_url, timeout=5).json()
|
||||
if res:
|
||||
rdw_data = {
|
||||
"power_kw": int(float(res[0].get('nettomaximumvermogen', 0))),
|
||||
"engine_capacity": int(res[0].get('cilinderinhoud', 0)),
|
||||
"curb_weight": int(res[0].get('massa_ledig_voertuig', 0))
|
||||
}
|
||||
print("✅ RDW adatok sikeresen betöltve.")
|
||||
except:
|
||||
print("⚠️ RDW nem elérhető vagy nincs találat.")
|
||||
|
||||
# 3. ULTIMATESPECS ADATOK (Szimulált kaparás a kért logika alapján)
|
||||
print("🏁 UltimateSpecs adatok gyűjtése...")
|
||||
# Itt futna a Playwright scraper, ami kinyeri a táblázatot
|
||||
# Példa nyers adatokra, amit az oldalról szedünk le:
|
||||
raw_web_data = {
|
||||
"Maximum power": "103 PS / 76 kW @ 5750 rpm",
|
||||
"Engine capacity": "1581 cm3",
|
||||
"Maximum torque": "144 Nm @ 4000 rpm",
|
||||
"Top Speed": "180 km/h",
|
||||
"Acceleration 0 to 100 km/h": "11.5 s",
|
||||
"Curb Weight": "1090 kg",
|
||||
"Wheelbase": "254 cm",
|
||||
"Body": "Hatchback"
|
||||
}
|
||||
|
||||
# 4. ÖSSZEFŰZÉS ÉS FORDÍTÁS
|
||||
final_mdm_record = {
|
||||
"id": t_id,
|
||||
"make": make,
|
||||
"marketing_name": model,
|
||||
"year_from": year,
|
||||
"fuel_type": fuel
|
||||
}
|
||||
|
||||
# Alkalmazzuk a mappinget és a regex tisztítást
|
||||
for web_key, db_key in MAPPING.items():
|
||||
val = raw_web_data.get(web_key)
|
||||
if val:
|
||||
# Számértékek kinyerése (pl. "76 kW" -> 76, "1581 cm3" -> 1581)
|
||||
numbers = re.findall(r'\d+', str(val))
|
||||
if numbers:
|
||||
# Ha több szám van (pl. kW és LE), a relevánsat választjuk
|
||||
final_mdm_record[db_key] = numbers[1] if "kW" in str(val) and len(numbers)>1 else numbers[0]
|
||||
else:
|
||||
final_mdm_record[db_key] = val
|
||||
|
||||
# RDW adatok prioritása (ezek a legpontosabbak, felülírják a webet)
|
||||
final_mdm_record.update({k: v for k, v in rdw_data.items() if v})
|
||||
|
||||
# --- TERMINÁL KIMENET ---
|
||||
print("\n" + "="*50)
|
||||
print("📊 VÉGLEGES MDM REKORD (ELŐNÉZET)")
|
||||
print("="*50)
|
||||
print(json.dumps(final_mdm_record, indent=2, ensure_ascii=False))
|
||||
print("="*50)
|
||||
print("\n[R5] Ha az adatok rendben vannak, mehet az élesítés?")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(r5_test_run())
|
||||
@@ -1,62 +0,0 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/workers/vehicle/vehicle_robot_1_5_heavy_eu1.0.py
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
logger = logging.getLogger("Robot-1-5-Heavy-EU")
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
class HeavyEUHunter:
|
||||
RDW_URL = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
|
||||
@classmethod
|
||||
async def fetch_rdw_heavy(cls, vehicle_type: str):
|
||||
query_url = f"{cls.RDW_URL}?voertuigsoort={vehicle_type}&$select=merk,handelsbenaming&$limit=10000"
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
try:
|
||||
resp = await client.get(query_url)
|
||||
return resp.json() if resp.status_code == 200 else []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ RDW Error: {e}")
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🚛 Robot 1.5 (EU Heavy Duty) indítása - Kötegelt mód...")
|
||||
job_list = {
|
||||
"Vrachtwagen": "truck",
|
||||
"Bus": "bus",
|
||||
"Kampeerauto": "rv"
|
||||
}
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
for rdw_name, internal_class in job_list.items():
|
||||
logger.info(f"📥 {rdw_name} adatok letöltése...")
|
||||
data = await cls.fetch_rdw_heavy(rdw_name)
|
||||
|
||||
if not data: continue
|
||||
|
||||
# A 10.000 adatot egyetlen listába gyűjtjük
|
||||
insert_data = []
|
||||
for item in data:
|
||||
make = item.get('merk', '').upper().strip()
|
||||
model = item.get('handelsbenaming', '').upper().strip()
|
||||
if make and model:
|
||||
insert_data.append({"make": make, "model": model, "v_class": internal_class})
|
||||
|
||||
if insert_data:
|
||||
query = text("""
|
||||
INSERT INTO vehicle.catalog_discovery
|
||||
(make, model, vehicle_class, status, market, priority_score, source)
|
||||
VALUES (:make, :model, :v_class, 'pending', 'EU', 20, 'RDW-HEAVY')
|
||||
ON CONFLICT ON CONSTRAINT _make_model_market_year_uc DO NOTHING
|
||||
""")
|
||||
# Egyetlen SQL hívással beszúrjuk akár a 10.000 sort is!
|
||||
await db.execute(query, insert_data)
|
||||
await db.commit()
|
||||
logger.info(f"✅ {rdw_name}: {len(insert_data)} EU-s nagygép beküldve kötegelve.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(HeavyEUHunter.run())
|
||||
@@ -1,387 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import urllib.parse
|
||||
import sys
|
||||
import signal
|
||||
import re
|
||||
from playwright.async_api import async_playwright
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
# R2.3 - SENTINEL (Hardened & Obedient Edition)
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [R2.3-SENTINEL] %(message)s')
|
||||
logger = logging.getLogger("R2.3")
|
||||
|
||||
# --- 1. SZŰRÉSEK ÉS TILTÓLISTÁK ---
|
||||
# Csak olyan típusokat keresünk, amik nem utánfutók vagy munkagépek
|
||||
JUNK_LIST = [
|
||||
'SARIS', 'ANSSEMS', 'HAPERT', 'HUMBAUR', 'EDUARD', 'IFOR WILLIAMS', 'FENDT',
|
||||
'HOBBY', 'ADRIA', 'PEECON', 'JAKO', 'KAWECO', 'POTTINGER', 'BOCKMANN',
|
||||
'JOHN DEERE', 'CLAAS', 'IVECO', 'SCANIA', 'MAN', 'DAF', 'KNAUS', 'PÖSSL', 'HYMER', 'WESTFALIA'
|
||||
]
|
||||
|
||||
# --- 2. FORDÍTÁSOK (DE/NL -> EN) ---
|
||||
TRANSLATIONS = {
|
||||
"3ER REIHE": "3 Series", "5ER REIHE": "5 Series", "1ER REIHE": "1 Series", "7ER REIHE": "7 Series",
|
||||
"E-KLASSE": "E Class", "C-KLASSE": "C Class", "S-KLASSE": "S Class", "A-KLASSE": "A Class",
|
||||
"REIHE": "Series", "KLASSE": "Class", "BESTELWAGEN": "Van"
|
||||
}
|
||||
|
||||
class RobotScout:
|
||||
def __init__(self):
|
||||
self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
self.running = True
|
||||
|
||||
def clean_name(self, make, model):
|
||||
"""Standardizált angol név előállítása."""
|
||||
m = model.upper()
|
||||
for de, en in TRANSLATIONS.items():
|
||||
m = m.replace(de, en)
|
||||
# Márkanév duplázódás törlése (pl. VOLVO VOLVO V60 -> VOLVO V60)
|
||||
m = m.replace(make.upper(), "").strip()
|
||||
return f"{make} {m}"
|
||||
|
||||
# --- COLUMN MAPPING for scraping ---
|
||||
COLUMN_MAPPING = {
|
||||
"horsepower": "power_kw",
|
||||
"engine displacement": "engine_capacity",
|
||||
"maximum torque": "torque_nm",
|
||||
"top speed": "max_speed",
|
||||
"curb weight": "curb_weight",
|
||||
"wheelbase": "wheelbase",
|
||||
"num. of seats": "seats"
|
||||
}
|
||||
|
||||
def clean_number(self, val: str, key: str = "") -> int:
|
||||
if not val or val == "-": return 0
|
||||
try:
|
||||
if "hp" in val.lower() or "kw" in val.lower():
|
||||
kw_match = re.search(r'(\d+)\s*kw', val.lower())
|
||||
if kw_match: return int(kw_match.group(1))
|
||||
nums = re.findall(r'\d+', val.replace(' ', '').replace(',', '').replace('.', ''))
|
||||
return int(nums[0]) if nums else 0
|
||||
except: return 0
|
||||
|
||||
async def get_car_links(self, page, make, model, year, use_year=True):
|
||||
"""Minden autós link kigyűjtése fallback mechanizmussal retry logikával."""
|
||||
clean_model = self.clean_name(make, model)
|
||||
search_query = f"{clean_model} {year}" if use_year else clean_model
|
||||
url = f"https://www.ultimatespecs.com/index.php?q={urllib.parse.quote(search_query)}"
|
||||
|
||||
logger.info(f"🔎 KERESÉS: {search_query}")
|
||||
|
||||
async def _fetch_links():
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=25000)
|
||||
|
||||
# 1. Ha direkt az adatlapon vagyunk
|
||||
if any(x in page.url for x in ['/car-specs/', '/motorcycles-specs/']):
|
||||
logger.info("🎯 Direkt találat!")
|
||||
return [{"name": await page.title(), "url": page.url}]
|
||||
|
||||
# 2. Várakozás és linkek kigyűjtése
|
||||
await asyncio.sleep(2)
|
||||
variants = await page.evaluate("""
|
||||
() => {
|
||||
let results = [];
|
||||
document.querySelectorAll('a').forEach(a => {
|
||||
let href = a.getAttribute('href') || '';
|
||||
let text = a.innerText.trim();
|
||||
// Csak technikai adatlapokat gyűjtünk, reklámokat/kategóriákat nem
|
||||
if ((href.includes('/car-specs/') || href.includes('/motorcycles-specs/'))
|
||||
&& href.includes('.html') && text.length > 3) {
|
||||
results.push({ name: text, url: href });
|
||||
}
|
||||
});
|
||||
return results;
|
||||
}
|
||||
""")
|
||||
|
||||
# 3. Fallback: Ha nincs találat évvel, próbálja év nélkül
|
||||
if not variants and use_year:
|
||||
logger.info(" ↳ Nincs találat évszámmal, próbálkozom évszám nélkül...")
|
||||
return await self.get_car_links(page, make, model, year, use_year=False)
|
||||
|
||||
return variants
|
||||
|
||||
try:
|
||||
variants = await self._retry_with_backoff(
|
||||
_fetch_links,
|
||||
max_attempts=3,
|
||||
base_delay=2,
|
||||
exception_message=f"❌ Hálózati hiba a(z) {url} oldalon"
|
||||
)
|
||||
return variants if variants is not None else []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Hálózati hiba (végleges): {str(e)[:50]}")
|
||||
return []
|
||||
|
||||
async def _retry_with_backoff(self, func, max_attempts=3, base_delay=2,
|
||||
exception_message="Retry failed", retry_exceptions=True):
|
||||
"""Helper function for retry logic with exponential backoff."""
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
return await func()
|
||||
except Exception as e:
|
||||
if attempt == max_attempts - 1:
|
||||
logger.error(f"{exception_message} after {max_attempts} attempts: {str(e)[:100]}")
|
||||
raise
|
||||
else:
|
||||
delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
|
||||
logger.warning(f"⚠️ Attempt {attempt + 1} failed: {str(e)[:50]}. Retrying in {delay:.1f}s...")
|
||||
await asyncio.sleep(delay)
|
||||
return None
|
||||
|
||||
async def scrape_car_details(self, page, url):
|
||||
"""Scrape car specifications from a given Ultimate Specs URL with comprehensive data extraction and retry logic."""
|
||||
async def _scrape():
|
||||
await page.goto(url, wait_until="networkidle", timeout=30000)
|
||||
|
||||
# Parsing all specification tables and sections
|
||||
full_specs = await page.evaluate("""
|
||||
() => {
|
||||
let results = {};
|
||||
|
||||
// 1. Collect all specification tables (existing logic)
|
||||
document.querySelectorAll('table.table_specs, table.responsive').forEach(table => {
|
||||
table.querySelectorAll('tr').forEach(row => {
|
||||
let t = row.querySelector('.table_specs_title, .td_title, td:first-child');
|
||||
let v = row.querySelector('.table_specs_value, .td_value, td:last-child');
|
||||
if(t && v) {
|
||||
let k = t.innerText.replace(':','').trim().toLowerCase();
|
||||
let val = v.innerText.trim();
|
||||
if(k && val && val !== "-") results[k] = val;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// 2. Collect section headers and their content for additional technical data
|
||||
// Look for h2, h3, h4 elements that might contain section titles
|
||||
const sections = {};
|
||||
const headers = document.querySelectorAll('h2, h3, h4, .section-title, .specs-header');
|
||||
|
||||
headers.forEach(header => {
|
||||
const title = header.innerText.trim();
|
||||
if (title && title.length > 0) {
|
||||
// Find the next table or div with specs after this header
|
||||
let nextElement = header.nextElementSibling;
|
||||
let sectionData = {};
|
||||
|
||||
// Look for tables or lists in the next few siblings
|
||||
for (let i = 0; i < 5 && nextElement; i++) {
|
||||
if (nextElement.tagName === 'TABLE') {
|
||||
nextElement.querySelectorAll('tr').forEach(row => {
|
||||
let t = row.querySelector('td:first-child');
|
||||
let v = row.querySelector('td:last-child');
|
||||
if(t && v) {
|
||||
let k = t.innerText.replace(':','').trim().toLowerCase();
|
||||
let val = v.innerText.trim();
|
||||
if(k && val && val !== "-") {
|
||||
sectionData[k] = val;
|
||||
// Also add to main results with section prefix
|
||||
results[`${title.toLowerCase().replace(/ /g, '_')}_${k}`] = val;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
nextElement = nextElement.nextElementSibling;
|
||||
}
|
||||
|
||||
sections[title.toLowerCase().replace(/ /g, '_')] = sectionData;
|
||||
}
|
||||
});
|
||||
|
||||
// 3. Extract specific known sections by looking for text patterns
|
||||
const pageText = document.body.innerText.toLowerCase();
|
||||
|
||||
// Check for electric/hybrid sections
|
||||
if (pageText.includes('electric engine') || pageText.includes('battery')) {
|
||||
// Try to find battery voltage, capacity, etc.
|
||||
const batteryRegex = /battery\s*voltage[:\s]*([\d\.]+)\s*v/gi;
|
||||
const match = batteryRegex.exec(document.body.innerText);
|
||||
if (match) results['battery_voltage_v'] = match[1];
|
||||
}
|
||||
|
||||
// 4. Extract dimensions data
|
||||
const dimensionPatterns = {
|
||||
'wheelbase': /wheelbase[:\s]*([\d\.]+)\s*cm/gi,
|
||||
'length': /length[:\s]*([\d\.]+)\s*cm/gi,
|
||||
'width': /width[:\s]*([\d\.]+)\s*cm/gi,
|
||||
'height': /height[:\s]*([\d\.]+)\s*cm/gi,
|
||||
'curb_weight': /curb\s*weight[:\s]*([\d\.]+)\s*kg/gi,
|
||||
'towing_capacity': /towing\s*capacity[:\s]*([\d\.]+)\s*kg/gi
|
||||
};
|
||||
|
||||
for (const [key, regex] of Object.entries(dimensionPatterns)) {
|
||||
const match = regex.exec(document.body.innerText);
|
||||
if (match) results[key] = match[1];
|
||||
}
|
||||
|
||||
// 5. Add sections data as a nested object
|
||||
results['_sections'] = sections;
|
||||
|
||||
return results;
|
||||
}
|
||||
""")
|
||||
return full_specs
|
||||
|
||||
try:
|
||||
logger.info(f"🌐 Scraping: {url}")
|
||||
full_specs = await self._retry_with_backoff(
|
||||
_scrape,
|
||||
max_attempts=3,
|
||||
base_delay=2,
|
||||
exception_message=f"❌ Scrape hiba a(z) {url} oldalon"
|
||||
)
|
||||
return full_specs
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Scrape hiba (végleges): {str(e)[:100]}...")
|
||||
return None
|
||||
|
||||
async def run(self):
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
context = await browser.new_context(user_agent=self.user_agent)
|
||||
page = await context.new_page()
|
||||
|
||||
while self.running:
|
||||
# --- A FÉK: 3-6 mp szigorú pihenő minden kör elején ---
|
||||
wait = random.uniform(3, 6)
|
||||
logger.info(f"💤 Várakozás {wait:.1f} mp...")
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Következő feldolgozatlan autó (John Deere, Iveco, stb. kizárva)
|
||||
target = (await db.execute(text("""
|
||||
SELECT id, make, marketing_name, year_from FROM vehicle.vehicle_model_definitions
|
||||
WHERE status IN ('pending', 'manual_review_needed')
|
||||
AND NOT (make = ANY(:junks))
|
||||
ORDER BY priority_score DESC LIMIT 1
|
||||
"""), {"junks": JUNK_LIST})).fetchone()
|
||||
|
||||
if not target:
|
||||
logger.info("✨ Minden tétel feldolgozva.")
|
||||
break
|
||||
|
||||
t_id, make, model, year = target
|
||||
logger.info(f"🚀 CÉLPONT: {make} {model} ({year}) [ID: {t_id}]")
|
||||
|
||||
try:
|
||||
links = await self.get_car_links(page, make, model, year)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Hálózati hiba linkek lekérésekor: {str(e)[:100]}")
|
||||
await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_network' WHERE id=:id"), {"id": t_id})
|
||||
await db.commit()
|
||||
continue
|
||||
|
||||
if not links:
|
||||
logger.warning(f"❌ Nem található adatlap. research_failed_empty rögzítése.")
|
||||
await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_empty' WHERE id=:id"), {"id": t_id})
|
||||
await db.commit()
|
||||
continue
|
||||
|
||||
# --- 1. SCRAPE THE FIRST LINK FOR IMMEDIATE ENRICHMENT ---
|
||||
first_link = None
|
||||
if links:
|
||||
first_link = links[0]
|
||||
full_url = first_link['url'] if first_link['url'].startswith('http') else f"https://www.ultimatespecs.com{first_link['url']}"
|
||||
logger.info(f"⚡ Azonnali adatgyűjtés: {full_url}")
|
||||
web_data = await self.scrape_car_details(page, full_url)
|
||||
|
||||
if web_data is None:
|
||||
# Scraping failed after all retries
|
||||
logger.error(f"❌ Scraping sikertelen minden próbálkozás után. research_failed_parsing rögzítése.")
|
||||
await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_parsing' WHERE id=:id"), {"id": t_id})
|
||||
await db.commit()
|
||||
# Continue to save links as variants anyway
|
||||
web_data = {}
|
||||
elif len(web_data) >= 5:
|
||||
# Map scraped data to columns
|
||||
updates = {col: self.clean_number(web_data.get(k)) for k, col in self.COLUMN_MAPPING.items()}
|
||||
# Also extract fuel_type, transmission, etc. if possible
|
||||
fuel_type = web_data.get('fuel type', 'Unknown')
|
||||
transmission_type = web_data.get('transmission', 'Unknown')
|
||||
drive_type = web_data.get('drive type', 'Unknown')
|
||||
body_type = web_data.get('body type', 'Unknown')
|
||||
engine_capacity = updates.get('engine_capacity', 0)
|
||||
power_kw = updates.get('power_kw', 0)
|
||||
|
||||
# Update the original record with scraped data
|
||||
await db.execute(text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET power_kw = :power_kw, engine_capacity = :engine_capacity,
|
||||
torque_nm = :torque_nm, max_speed = :max_speed,
|
||||
curb_weight = :curb_weight,
|
||||
wheelbase = :wheelbase, seats = :seats,
|
||||
fuel_type = :fuel_type, transmission_type = :transmission_type,
|
||||
drive_type = :drive_type, body_type = :body_type,
|
||||
specifications = specifications || :full_json,
|
||||
status = 'awaiting_ai_synthesis', updated_at = NOW()
|
||||
WHERE id = :id
|
||||
"""), {
|
||||
**updates,
|
||||
"id": t_id,
|
||||
"fuel_type": fuel_type,
|
||||
"transmission_type": transmission_type,
|
||||
"drive_type": drive_type,
|
||||
"body_type": body_type,
|
||||
"full_json": json.dumps(web_data)
|
||||
})
|
||||
logger.info(f"✅ AZONNALI PUBLIKÁLÁS: {make} {model} ({power_kw} kW)")
|
||||
else:
|
||||
logger.warning("⚠️ Scraping kevés adatot talált, csak linkek mentve.")
|
||||
|
||||
# --- 2. SAVE ALL LINKS AS NEW VARIANT RECORDS (including first if not enriched) ---
|
||||
added = 0
|
||||
for l in links:
|
||||
full_url = l['url'] if l['url'].startswith('http') else f"https://www.ultimatespecs.com{l['url']}"
|
||||
|
||||
# JAVÍTÁS: column "source_url" hiba ellen raw_api_data-t nézünk
|
||||
check_query = text("SELECT id FROM vehicle.vehicle_model_definitions WHERE raw_api_data->>'url' = :u")
|
||||
exists = (await db.execute(check_query, {"u": full_url})).fetchone()
|
||||
|
||||
if not exists:
|
||||
# Create normalized name from marketing name
|
||||
normalized = l['name'].lower().replace(' ', '_').replace('-', '_').replace('.', '').replace(',', '')[:200]
|
||||
|
||||
await db.execute(text("""
|
||||
INSERT INTO vehicle.vehicle_model_definitions
|
||||
(make, marketing_name, normalized_name, year_from, status,
|
||||
raw_api_data, priority_score, source, market,
|
||||
technical_code, variant_code, version_code,
|
||||
specifications, marketing_name_aliases, raw_search_context)
|
||||
VALUES (:make, :name, :normalized, :year, 'awaiting_ai_synthesis',
|
||||
:raw, 30, 'ultimatespecs', 'EU',
|
||||
'UNKNOWN', 'UNKNOWN', 'UNKNOWN',
|
||||
'{}'::jsonb, '[]'::jsonb, '')
|
||||
"""), {
|
||||
"make": make, "name": l['name'], "normalized": normalized,
|
||||
"year": year, "raw": json.dumps({"url": full_url}), "priority": 30
|
||||
})
|
||||
added += 1
|
||||
|
||||
# Eredeti rekord archiválása (ha még nem publikáltuk)
|
||||
if not web_data:
|
||||
await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='expanded_to_variants', updated_at=NOW() WHERE id=:id"), {"id": t_id})
|
||||
|
||||
await db.commit()
|
||||
logger.info(f"✅ SIKER: {added} új variáció mentve. R4-R5 robotok értesítve.")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
scout = RobotScout()
|
||||
# Handle CTRL+C
|
||||
def stop_signal(sig, frame):
|
||||
logger.info("🛑 LEÁLLÍTÁS (Kérés érzékelve)...")
|
||||
scout.running = False
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, stop_signal)
|
||||
|
||||
try:
|
||||
asyncio.run(scout.run())
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
@@ -112,7 +112,7 @@ class AlchemistPro:
|
||||
"prompt": prompt,
|
||||
"format": "json",
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.1, "top_p": 0.9}
|
||||
"options": {"temperature": 0.1, "top_p": 0.9, "num_ctx": 4096}
|
||||
}
|
||||
try:
|
||||
response = await self.client.post(OLLAMA_URL, json=payload)
|
||||
@@ -190,10 +190,17 @@ class AlchemistPro:
|
||||
return vehicle, None, e
|
||||
|
||||
async def process_batch(self, db: AsyncSession, vehicles: list):
|
||||
"""Batch feldolgozás: Párhuzamos AI, majd szekvenciális DB mentés."""
|
||||
# 1. AI kérések párhuzamosan (CPU kímélő batch mérettel)
|
||||
tasks = [self.process_ai_task(v) for v in vehicles]
|
||||
results = await asyncio.gather(*tasks)
|
||||
"""Batch feldolgozás: Szekvenciális AI feldolgozás a VRAM korlátok miatt."""
|
||||
results = []
|
||||
|
||||
# 1. AI kérések szekvenciálisan (egy jármű után a másik)
|
||||
for vehicle in vehicles:
|
||||
try:
|
||||
vehicle_result = await self.process_ai_task(vehicle)
|
||||
results.append(vehicle_result)
|
||||
except Exception as e:
|
||||
logger.error(f"Hiba {vehicle['id']} AI feldolgozás közben: {e}")
|
||||
results.append((vehicle, None, e))
|
||||
|
||||
# 2. Mentés szekvenciálisan a DB lakatok elkerülésére
|
||||
for vehicle, ai_result, error in results:
|
||||
|
||||
Reference in New Issue
Block a user