2026.03.29 20:00 Gitea_manager javítás előtt
This commit is contained in:
114
archive/old_files/.env_old
Executable file
114
archive/old_files/.env_old
Executable file
@@ -0,0 +1,114 @@
|
||||
COMPOSE_PROJECT_NAME=service_finder
|
||||
|
||||
# --- ADATBÁZIS KAPCSOLAT (Központi) ---
|
||||
# Itt a 'shared-postgres' nevet használjuk, ami a központi konténer neve
|
||||
APP_DB_HOST=shared-postgres
|
||||
APP_DB_PORT=5432
|
||||
APP_DB_NAME=service_finder
|
||||
APP_DB_USER=service_finder_app
|
||||
# IDE ÍRD AZT A JELSZÓT, amit a pgAdminban/ALTER USER parancsnál adtál a 'service_finder_app'-nak!
|
||||
APP_DB_PASSWORD='MiskociA74'
|
||||
|
||||
# Ez a legfontosabb sor, ezt használja a Python program:
|
||||
DATABASE_URL=postgresql+asyncpg://service_finder_app:MiskociA74@shared-postgres:5432/service_finder
|
||||
# Migrációhoz (néha szinkron driver kell):
|
||||
MIGRATION_DATABASE_URL=postgresql+asyncpg://service_finder_app:MiskociA74@shared-postgres:5432/service_finder
|
||||
|
||||
# --- ALKALMAZÁS BEÁLLÍTÁSOK ---
|
||||
ALLOWED_ORIGINS="https://app.profibot.hu,https://dev.profibot.hu,http://localhost:3000"
|
||||
PYTHONPATH=/app
|
||||
|
||||
# --- MINIO (Fájltárolás) ---
|
||||
# Ez maradhat helyi konténer, vagy köthetjük a központihoz is, de most hagyjuk a projektben
|
||||
MINIO_ENDPOINT=minio:9000
|
||||
MINIO_ROOT_USER=kincses
|
||||
MINIO_ROOT_PASSWORD='MiskociA74'
|
||||
MINIO_ACCESS_KEY=kincses
|
||||
MINIO_SECRET_KEY='MiskociA74'
|
||||
|
||||
# --- EGYÉB API KULCSOK ---
|
||||
SENDGRID_API_KEY=SG.SENDGRID_API_KEY=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
|
||||
FROM_EMAIL=info@profibot.hu
|
||||
|
||||
# Biztonsági kulcs a tokenekhez (KÖTELEZŐ!)
|
||||
SECRET_KEY=2dca2ff3bf9b8184e14038d5d08e646b31bd4a5f5ffc7e19d28e294f3bb3760b
|
||||
_______________________________________________________________
|
||||
# ==============================================================================
|
||||
# 🛠️ INFRASTRUKTÚRA (Docker & Database)
|
||||
# ==============================================================================
|
||||
# Adatbázis alapok
|
||||
POSTGRES_USER=kincses
|
||||
POSTGRES_PASSWORD='MiskociA74'
|
||||
POSTGRES_DB=service_finder
|
||||
|
||||
# Kapcsolati URL a Python számára (Központi shared-postgres)
|
||||
DATABASE_URL=postgresql+asyncpg://service_finder_app:MiskociA74@shared-postgres:5432/service_finder
|
||||
|
||||
# Migrációhoz használt URL (Alembic számára)
|
||||
MIGRATION_DATABASE_URL=postgresql+asyncpg://service_finder_app:MiskociA74@shared-postgres:5432/service_finder
|
||||
|
||||
# Redis elérés
|
||||
REDIS_URL=redis://service_finder_redis:6379/0
|
||||
|
||||
# ==============================================================================
|
||||
# 🚀 ALKALMAZÁS BEÁLLÍTÁSOK (FastAPI)
|
||||
# ==============================================================================
|
||||
ENV=development
|
||||
DEBUG=True
|
||||
PYTHONPATH=/app
|
||||
|
||||
# Biztonsági kulcs a JWT tokenekhez (Generálj egy hosszú véletlen sort!)
|
||||
# Példa generáláshoz: openssl rand -hex 32
|
||||
SECRET_KEY='2dca2ff3bf9b8184e14038d5d08e646b31bd4a5f5ffc7e19d28e294f3bb3760b'
|
||||
ALGORITHM=HS256
|
||||
|
||||
# CORS: Milyen címekről érhető el az API? (Vesszővel elválasztva)
|
||||
CORS_ORIGINS=https://app.profibot.hu,https://dev.profibot.hu,http://localhost:3000,http://192.168.100.10:3000
|
||||
|
||||
# Frontend címe a kiküldött linkekhez (Visszaigazolás, jelszó-visszaállítás)
|
||||
FRONTEND_BASE_URL=http://192.168.100.10:3000
|
||||
|
||||
# ==============================================================================
|
||||
# 📧 EMAIL RENDSZER (SMTP / SendGrid)
|
||||
# ==============================================================================
|
||||
# EMAIL_PROVIDER lehet: 'smtp' vagy 'sendgrid' vagy 'disabled'
|
||||
EMAIL_PROVIDER=sendgrid
|
||||
EMAILS_FROM_EMAIL=info@profibot.hu
|
||||
EMAILS_FROM_NAME='Service Finder'
|
||||
|
||||
# SendGrid beállítások
|
||||
SENDGRID_API_KEY=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
|
||||
|
||||
# SMTP Fallback (Csak ha az EMAIL_PROVIDER=smtp)
|
||||
SMTP_HOST=smtp.gmail.com
|
||||
SMTP_PORT=587
|
||||
SMTP_USER=info@profibot.hu
|
||||
SMTP_PASSWORD='SAJÁT_APP_PASSWORD'
|
||||
|
||||
# ==============================================================================
|
||||
# 📦 MINIO (Fájltárolás - NAS-ra kivezetve)
|
||||
# ==============================================================================
|
||||
MINIO_ENDPOINT=minio:9000
|
||||
MINIO_ROOT_USER=kincses
|
||||
MINIO_ROOT_PASSWORD='MiskociA74'
|
||||
MINIO_ACCESS_KEY=kincses
|
||||
MINIO_SECRET_KEY='MiskociA74'
|
||||
|
||||
# --- n8n CONFIG ---
|
||||
N8N_DB_PASSWORD=MiskociA74
|
||||
# Az n8n-en belül a központi DB elérése:
|
||||
# Host: shared-postgres
|
||||
# User: service_finder_app
|
||||
|
||||
# --- Frontend ---
|
||||
FRONTEND_BASE_URL=https://dev.profibot.hu/docs
|
||||
|
||||
|
||||
|
||||
# Holland autó adatbázis free token
|
||||
RDW_APP_TOKEN=kSMUn0tvnmoM6TMSegLpFvKI8
|
||||
|
||||
# gemini service_finder_robot
|
||||
GEMINI_API_KEY=AIzaSyAaCVNPwf8PCphu_pt6spjAa2OVu8Exug8
|
||||
|
||||
DeepSeek API key = sk-1871b668aac44b50859ee6c54fe95e21
|
||||
175
archive/old_files/backend/app/models/marketplace/service.py.old
Executable file
175
archive/old_files/backend/app/models/marketplace/service.py.old
Executable file
@@ -0,0 +1,175 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/models/marketplace/service.py
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Optional
|
||||
from sqlalchemy import Integer, String, Boolean, DateTime, ForeignKey, text, Text, Float, Index, Numeric, BigInteger
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from sqlalchemy.dialects.postgresql import UUID as PG_UUID, JSONB
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
# MB 2.0: Központi aszinkron adatbázis motorból húzzuk be a Base-t
|
||||
from app.database import Base
|
||||
|
||||
class ServiceProfile(Base):
|
||||
""" Szerviz szolgáltató adatai (v1.3.1). """
|
||||
__tablename__ = "service_profiles"
|
||||
__table_args__ = (
|
||||
Index('idx_service_fingerprint', 'fingerprint', unique=True),
|
||||
{"schema": "marketplace"}
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
organization_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("fleet.organizations.id"), unique=True)
|
||||
parent_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("marketplace.service_profiles.id"))
|
||||
|
||||
fingerprint: Mapped[str] = mapped_column(String(255), index=True, nullable=False)
|
||||
location: Mapped[Any] = mapped_column(Geometry(geometry_type='POINT', srid=4326, spatial_index=False), index=True)
|
||||
|
||||
status: Mapped[str] = mapped_column(String(20), server_default=text("'ghost'"), index=True)
|
||||
last_audit_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
google_place_id: Mapped[Optional[str]] = mapped_column(String(100), unique=True)
|
||||
rating: Mapped[Optional[float]] = mapped_column(Float)
|
||||
user_ratings_total: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
|
||||
# Aggregated verified review ratings (Social 3)
|
||||
rating_verified_count: Mapped[Optional[int]] = mapped_column(Integer, server_default=text("0"))
|
||||
rating_price_avg: Mapped[Optional[float]] = mapped_column(Float)
|
||||
rating_quality_avg: Mapped[Optional[float]] = mapped_column(Float)
|
||||
rating_time_avg: Mapped[Optional[float]] = mapped_column(Float)
|
||||
rating_communication_avg: Mapped[Optional[float]] = mapped_column(Float)
|
||||
rating_overall: Mapped[Optional[float]] = mapped_column(Float)
|
||||
last_review_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
vibe_analysis: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
social_links: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
specialization_tags: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
trust_score: Mapped[int] = mapped_column(Integer, default=30)
|
||||
is_verified: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
verification_log: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
opening_hours: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
contact_phone: Mapped[Optional[str]] = mapped_column(String)
|
||||
contact_email: Mapped[Optional[str]] = mapped_column(String)
|
||||
website: Mapped[Optional[str]] = mapped_column(String)
|
||||
bio: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
# Kapcsolatok
|
||||
organization: Mapped["Organization"] = relationship("Organization", back_populates="service_profile")
|
||||
expertises: Mapped[List["ServiceExpertise"]] = relationship("ServiceExpertise", back_populates="service")
|
||||
reviews: Mapped[List["ServiceReview"]] = relationship("ServiceReview", back_populates="service")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), onupdate=func.now())
|
||||
|
||||
class ExpertiseTag(Base):
|
||||
"""
|
||||
Szakmai címkék mesterlistája (MB 2.0).
|
||||
Ez a tábla vezérli a robotok keresését és a Gamification pontozást is.
|
||||
"""
|
||||
__tablename__ = "expertise_tags"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
|
||||
# Egyedi azonosító kulcs (pl. 'ENGINE_REBUILD')
|
||||
key: Mapped[str] = mapped_column(String(50), unique=True, index=True)
|
||||
|
||||
# Megjelenítendő nevek
|
||||
name_hu: Mapped[Optional[str]] = mapped_column(String(100))
|
||||
name_en: Mapped[Optional[str]] = mapped_column(String(100))
|
||||
|
||||
# Főcsoport (pl. 'MECHANICS', 'ELECTRICAL', 'EMERGENCY')
|
||||
category: Mapped[Optional[str]] = mapped_column(String(30), index=True)
|
||||
|
||||
# --- 🎮 GAMIFICATION ÉS DISCOVERY ---
|
||||
|
||||
# Hivatalos címke (True) vagy júzer/robot által javasolt (False)
|
||||
is_official: Mapped[bool] = mapped_column(Boolean, default=True, server_default=text("true"))
|
||||
|
||||
# Ha júzer javasolta, itt tároljuk, ki volt az (XP jóváíráshoz)
|
||||
suggested_by_id: Mapped[Optional[int]] = mapped_column(BigInteger, ForeignKey("identity.persons.id"))
|
||||
|
||||
# ÁLLÍTHATÓ PONTÉRTÉK: Az adatbázisból jön, így bármikor módosítható.
|
||||
# Ritka szakmáknál magasabb, gyakoriaknál alacsonyabb érték állítható be.
|
||||
discovery_points: Mapped[int] = mapped_column(Integer, default=10, server_default=text("10"))
|
||||
|
||||
# Robot kulcsszavak (JSONB): ["fék", "betét", "tárcsa", "fékfolyadék"]
|
||||
# A Scout robot ez alapján azonosítja be a szervizt a weboldala alapján.
|
||||
search_keywords: Mapped[Any] = mapped_column(JSONB, server_default=text("'[]'::jsonb"))
|
||||
|
||||
# Népszerűségi mutató (hányszor lett felhasználva a rendszerben)
|
||||
usage_count: Mapped[int] = mapped_column(Integer, default=0, server_default=text("0"))
|
||||
|
||||
# UI ikon azonosító (pl. 'wrench', 'tire-flat', 'car-electric')
|
||||
icon: Mapped[Optional[str]] = mapped_column(String(50))
|
||||
|
||||
# Leírás a szakmáról (Adminisztratív célokra)
|
||||
description: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
# Időbélyegek
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), onupdate=func.now())
|
||||
|
||||
# --- KAPCSOLATOK ---
|
||||
services: Mapped[List["ServiceExpertise"]] = relationship("ServiceExpertise", back_populates="tag")
|
||||
# Visszamutatás a beküldőre (ha van)
|
||||
suggested_by: Mapped[Optional["Person"]] = relationship("Person")
|
||||
|
||||
class ServiceExpertise(Base):
|
||||
"""
|
||||
KAPCSOLÓTÁBLA: Ez köti össze a szervizt a szakmáival.
|
||||
Itt tároljuk, hogy az adott szerviznél mennyire validált egy szakma.
|
||||
"""
|
||||
__tablename__ = "service_expertises"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
service_id: Mapped[int] = mapped_column(Integer, ForeignKey("marketplace.service_profiles.id", ondelete="CASCADE"))
|
||||
expertise_id: Mapped[int] = mapped_column(Integer, ForeignKey("marketplace.expertise_tags.id", ondelete="CASCADE"))
|
||||
|
||||
# Mennyire biztos ez a tudás? (0: robot találta, 1: júzer mondta, 2: igazolt szakma)
|
||||
confidence_level: Mapped[int] = mapped_column(Integer, default=0, server_default=text("0"))
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=text("now()"))
|
||||
|
||||
# Kapcsolatok visszafelé
|
||||
service = relationship("ServiceProfile", back_populates="expertises")
|
||||
tag = relationship("ExpertiseTag", back_populates="services")
|
||||
|
||||
class ServiceStaging(Base):
|
||||
""" Hunter (robot) adatok tárolója. """
|
||||
__tablename__ = "service_staging"
|
||||
__table_args__ = (
|
||||
Index('idx_staging_fingerprint', 'fingerprint', unique=True),
|
||||
{"schema": "marketplace"}
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String, index=True, nullable=False)
|
||||
postal_code: Mapped[Optional[str]] = mapped_column(String(10), index=True)
|
||||
city: Mapped[Optional[str]] = mapped_column(String(100), index=True)
|
||||
full_address: Mapped[Optional[str]] = mapped_column(String)
|
||||
fingerprint: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
raw_data: Mapped[Any] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
# Additional contact and identification fields
|
||||
contact_phone: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
website: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
|
||||
external_id: Mapped[Optional[str]] = mapped_column(String(100), nullable=True, index=True)
|
||||
|
||||
status: Mapped[str] = mapped_column(String(20), server_default=text("'pending'"), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
class DiscoveryParameter(Base):
|
||||
""" Robot vezérlési paraméterek adminból. """
|
||||
__tablename__ = "discovery_parameters"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
city: Mapped[str] = mapped_column(String(100))
|
||||
keyword: Mapped[str] = mapped_column(String(100))
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
73
archive/old_files/backend/app/models/marketplace/staged_data1.2_.py.old
Executable file
73
archive/old_files/backend/app/models/marketplace/staged_data1.2_.py.old
Executable file
@@ -0,0 +1,73 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/models/marketplace/staged_data.py
|
||||
from datetime import datetime
|
||||
from typing import Optional, Any
|
||||
from sqlalchemy import String, Integer, DateTime, text, Boolean, Float
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.sql import func
|
||||
from app.db.base_class import Base
|
||||
|
||||
class StagedVehicleData(Base):
|
||||
""" Robot 2.1 (Researcher) nyers adatgyűjtője. """
|
||||
__tablename__ = "staged_vehicle_data"
|
||||
__table_args__ = {"schema": "system"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_url: Mapped[Optional[str]] = mapped_column(String)
|
||||
raw_data: Mapped[dict] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
|
||||
status: Mapped[str] = mapped_column(String(20), default="PENDING", index=True)
|
||||
error_log: Mapped[Optional[str]] = mapped_column(String)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
class ServiceStaging(Base):
|
||||
""" Robot 1.3 (Scout) által talált nyers szerviz adatok és a Robot 5 (Auditor) naplója. """
|
||||
__tablename__ = "service_staging"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), index=True)
|
||||
source: Mapped[Optional[str]] = mapped_column(String(50))
|
||||
external_id: Mapped[Optional[str]] = mapped_column(String(100), index=True)
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
|
||||
# Elérhetőségek
|
||||
city: Mapped[str] = mapped_column(String(100), index=True)
|
||||
postal_code: Mapped[Optional[str]] = mapped_column(String(10))
|
||||
full_address: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
contact_phone: Mapped[Optional[str]] = mapped_column(String(50))
|
||||
website: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
contact_email: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
|
||||
|
||||
# Beküldés és Bizalom
|
||||
description: Mapped[Optional[str]] = mapped_column(Text)
|
||||
submitted_by: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("identity.users.id"))
|
||||
trust_score: Mapped[int] = mapped_column(Integer, default=0, server_default=text("0"))
|
||||
|
||||
# Nyers adatok és Státusz
|
||||
raw_data: Mapped[dict] = mapped_column(JSONB, server_default=text("'{}'::jsonb"))
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending", index=True)
|
||||
|
||||
# --- Robot 5 (Auditor) technikai mezők ---
|
||||
# Ezek kellenek a munka naplózásához
|
||||
rejection_reason: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
published_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
service_profile_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
organization_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
audit_trail: Mapped[Optional[dict]] = mapped_column(JSONB)
|
||||
|
||||
# Időbélyegek
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), onupdate=func.now())
|
||||
|
||||
class DiscoveryParameter(Base):
|
||||
""" Felderítési paraméterek (Városok, ahol a Scout keres). """
|
||||
__tablename__ = "discovery_parameters"
|
||||
__table_args__ = {"schema": "marketplace"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
city: Mapped[str] = mapped_column(String(100), unique=True, index=True)
|
||||
keyword: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
58
archive/old_files/backend/app/scripts/move_tables.py.old
Normal file
58
archive/old_files/backend/app/scripts/move_tables.py.old
Normal file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Move tables from system schema to gamification schema.
|
||||
"""
|
||||
import asyncio
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import text
|
||||
|
||||
async def move_tables():
|
||||
# Use the same DATABASE_URL as sync_engine
|
||||
from app.core.config import settings
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Check if tables exist in system schema
|
||||
result = await conn.execute(text("""
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_name IN ('competitions', 'user_scores')
|
||||
ORDER BY table_schema;
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
print("Current tables:")
|
||||
for row in rows:
|
||||
print(f" {row.table_schema}.{row.table_name}")
|
||||
|
||||
# Move competitions
|
||||
print("\nMoving system.competitions to gamification.competitions...")
|
||||
try:
|
||||
await conn.execute(text('ALTER TABLE system.competitions SET SCHEMA gamification;'))
|
||||
print(" OK")
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
# Move user_scores
|
||||
print("Moving system.user_scores to gamification.user_scores...")
|
||||
try:
|
||||
await conn.execute(text('ALTER TABLE system.user_scores SET SCHEMA gamification;'))
|
||||
print(" OK")
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
# Verify
|
||||
result = await conn.execute(text("""
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_name IN ('competitions', 'user_scores')
|
||||
ORDER BY table_schema;
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
print("\nAfter moving:")
|
||||
for row in rows:
|
||||
print(f" {row.table_schema}.{row.table_name}")
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(move_tables())
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Rename tables in system schema to deprecated to avoid extra detection.
|
||||
"""
|
||||
import asyncio
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import text
|
||||
|
||||
async def rename():
|
||||
from app.core.config import settings
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Check if tables exist
|
||||
result = await conn.execute(text("""
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'system' AND table_name IN ('competitions', 'user_scores');
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
print("Tables to rename:")
|
||||
for row in rows:
|
||||
print(f" {row.table_schema}.{row.table_name}")
|
||||
|
||||
# Rename competitions
|
||||
try:
|
||||
await conn.execute(text('ALTER TABLE system.competitions RENAME TO competitions_deprecated;'))
|
||||
print("Renamed system.competitions -> system.competitions_deprecated")
|
||||
except Exception as e:
|
||||
print(f"Error renaming competitions: {e}")
|
||||
|
||||
# Rename user_scores
|
||||
try:
|
||||
await conn.execute(text('ALTER TABLE system.user_scores RENAME TO user_scores_deprecated;'))
|
||||
print("Renamed system.user_scores -> system.user_scores_deprecated")
|
||||
except Exception as e:
|
||||
print(f"Error renaming user_scores: {e}")
|
||||
|
||||
# Verify
|
||||
result = await conn.execute(text("""
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'system' AND table_name LIKE '%deprecated';
|
||||
"""))
|
||||
rows = result.fetchall()
|
||||
print("\nAfter rename:")
|
||||
for row in rows:
|
||||
print(f" {row.table_schema}.{row.table_name}")
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(rename())
|
||||
170
archive/old_files/backend/app/scripts/sync_engine1.0.py.old
Normal file
170
archive/old_files/backend/app/scripts/sync_engine1.0.py.old
Normal file
@@ -0,0 +1,170 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/scripts/sync_engine.py
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Universal Schema Synchronizer
|
||||
|
||||
Dynamically imports all SQLAlchemy models from app.models, compares them with the live database,
|
||||
and creates missing tables/columns without dropping anything.
|
||||
|
||||
Safety First:
|
||||
- NEVER drops tables or columns.
|
||||
- Prints planned SQL before execution.
|
||||
- Requires confirmation for destructive operations (none in this script).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import inspect, text
|
||||
from sqlalchemy.schema import CreateTable, AddConstraint
|
||||
from sqlalchemy.sql.ddl import CreateColumn
|
||||
|
||||
# Add backend to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from app.database import Base
|
||||
from app.core.config import settings
|
||||
|
||||
def dynamic_import_models():
|
||||
"""
|
||||
Dynamically import all .py files in app.models directory to ensure Base.metadata is populated.
|
||||
"""
|
||||
models_dir = Path(__file__).parent.parent / "models"
|
||||
imported = []
|
||||
|
||||
for py_file in models_dir.glob("*.py"):
|
||||
if py_file.name == "__init__.py":
|
||||
continue
|
||||
module_name = f"app.models.{py_file.stem}"
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
imported.append(module_name)
|
||||
print(f"✅ Imported {module_name}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not import {module_name}: {e}")
|
||||
|
||||
# Also ensure the __init__ is loaded (it imports many models manually)
|
||||
import app.models
|
||||
print(f"📦 Total tables in Base.metadata: {len(Base.metadata.tables)}")
|
||||
return imported
|
||||
|
||||
async def compare_and_repair():
|
||||
"""
|
||||
Compare SQLAlchemy metadata with live database and create missing tables/columns.
|
||||
"""
|
||||
print("🔗 Connecting to database...")
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
def get_diff_and_repair(connection):
|
||||
inspector = inspect(connection)
|
||||
|
||||
# Get all schemas from models
|
||||
expected_schemas = sorted({t.schema for t in Base.metadata.sorted_tables if t.schema})
|
||||
print(f"📋 Expected schemas: {expected_schemas}")
|
||||
|
||||
# Ensure enum types exist in marketplace schema
|
||||
if 'marketplace' in expected_schemas:
|
||||
print("\n🔧 Ensuring enum types in marketplace schema...")
|
||||
# moderation_status enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'moderation_status' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.moderation_status AS ENUM ('pending', 'approved', 'rejected');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
# source_type enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'source_type' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.source_type AS ENUM ('manual', 'ocr', 'import');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
print("✅ Enum types ensured.")
|
||||
|
||||
for schema in expected_schemas:
|
||||
print(f"\n--- 🔍 Checking schema '{schema}' ---")
|
||||
|
||||
# Check if schema exists
|
||||
db_schemas = inspector.get_schema_names()
|
||||
if schema not in db_schemas:
|
||||
print(f"❌ Schema '{schema}' missing. Creating...")
|
||||
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
|
||||
print(f"✅ Schema '{schema}' created.")
|
||||
|
||||
# Get tables in this schema from models
|
||||
model_tables = [t for t in Base.metadata.sorted_tables if t.schema == schema]
|
||||
db_tables = inspector.get_table_names(schema=schema)
|
||||
|
||||
for table in model_tables:
|
||||
if table.name not in db_tables:
|
||||
print(f"❌ Missing table: {schema}.{table.name}")
|
||||
# Generate CREATE TABLE statement
|
||||
create_stmt = CreateTable(table)
|
||||
# Print SQL for debugging
|
||||
sql_str = str(create_stmt.compile(bind=engine))
|
||||
print(f" SQL: {sql_str}")
|
||||
connection.execute(create_stmt)
|
||||
print(f"✅ Table {schema}.{table.name} created.")
|
||||
else:
|
||||
# Check columns
|
||||
db_columns = {c['name']: c for c in inspector.get_columns(table.name, schema=schema)}
|
||||
model_columns = table.columns
|
||||
|
||||
missing_cols = []
|
||||
for col in model_columns:
|
||||
if col.name not in db_columns:
|
||||
missing_cols.append(col)
|
||||
|
||||
if missing_cols:
|
||||
print(f"⚠️ Table {schema}.{table.name} missing columns: {[c.name for c in missing_cols]}")
|
||||
for col in missing_cols:
|
||||
# Generate ADD COLUMN statement
|
||||
col_type = col.type.compile(dialect=engine.dialect)
|
||||
sql = f'ALTER TABLE "{schema}"."{table.name}" ADD COLUMN "{col.name}" {col_type}'
|
||||
if col.nullable is False:
|
||||
sql += " NOT NULL"
|
||||
if col.default is not None:
|
||||
# Handle default values (simplistic)
|
||||
sql += f" DEFAULT {col.default.arg}"
|
||||
print(f" SQL: {sql}")
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Column {col.name} added.")
|
||||
else:
|
||||
print(f"✅ Table {schema}.{table.name} is up‑to‑date.")
|
||||
|
||||
print("\n--- ✅ Schema synchronization complete. ---")
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(get_diff_and_repair)
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
async def main():
|
||||
print("🚀 Universal Schema Synchronizer")
|
||||
print("=" * 50)
|
||||
|
||||
# Step 1: Dynamic import
|
||||
print("\n📥 Step 1: Dynamically importing all models...")
|
||||
dynamic_import_models()
|
||||
|
||||
# Step 2: Compare and repair
|
||||
print("\n🔧 Step 2: Comparing with database and repairing...")
|
||||
await compare_and_repair()
|
||||
|
||||
# Step 3: Final verification
|
||||
print("\n📊 Step 3: Final verification...")
|
||||
# Run compare_schema.py logic to confirm everything is green
|
||||
from app.tests_internal.diagnostics.compare_schema import compare
|
||||
await compare()
|
||||
|
||||
print("\n✨ Synchronization finished successfully!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
232
archive/old_files/backend/app/scripts/unified_db_sync_1.0.py.old
Normal file
232
archive/old_files/backend/app/scripts/unified_db_sync_1.0.py.old
Normal file
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Unified Database Synchronizer with Deep Constraint & Index Support
|
||||
|
||||
Dynamically imports all SQLAlchemy models, compares metadata with live database,
|
||||
and creates missing tables, columns, unique constraints, and indexes.
|
||||
|
||||
Safety First:
|
||||
- NEVER drops tables, columns, constraints, or indexes.
|
||||
- Prints planned SQL before execution.
|
||||
- Requires confirmation for destructive operations (none in this script).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import inspect, text, UniqueConstraint, Index
|
||||
from sqlalchemy.schema import CreateTable, AddConstraint, CreateIndex
|
||||
from sqlalchemy.sql.ddl import CreateColumn
|
||||
|
||||
# Add backend to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from app.database import Base
|
||||
from app.core.config import settings
|
||||
|
||||
def dynamic_import_models():
|
||||
"""
|
||||
Dynamically import all .py files in app.models directory to ensure Base.metadata is populated.
|
||||
"""
|
||||
models_dir = Path(__file__).parent.parent / "models"
|
||||
imported = []
|
||||
|
||||
for py_file in models_dir.glob("*.py"):
|
||||
if py_file.name == "__init__.py":
|
||||
continue
|
||||
module_name = f"app.models.{py_file.stem}"
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
imported.append(module_name)
|
||||
print(f"✅ Imported {module_name}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not import {module_name}: {e}")
|
||||
|
||||
# Also ensure the __init__ is loaded (it imports many models manually)
|
||||
import app.models
|
||||
print(f"📦 Total tables in Base.metadata: {len(Base.metadata.tables)}")
|
||||
return imported
|
||||
|
||||
async def compare_and_repair(apply: bool = False):
|
||||
"""
|
||||
Compare SQLAlchemy metadata with live database and create missing
|
||||
tables, columns, unique constraints, and indexes.
|
||||
|
||||
If apply is False, only prints SQL statements without executing.
|
||||
"""
|
||||
print("🔗 Connecting to database...")
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
def get_diff_and_repair(connection):
|
||||
inspector = inspect(connection)
|
||||
|
||||
# Get all schemas from models
|
||||
expected_schemas = sorted({t.schema for t in Base.metadata.sorted_tables if t.schema})
|
||||
print(f"📋 Expected schemas: {expected_schemas}")
|
||||
|
||||
# Ensure enum types exist in marketplace schema
|
||||
if 'marketplace' in expected_schemas:
|
||||
print("\n🔧 Ensuring enum types in marketplace schema...")
|
||||
# moderation_status enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'moderation_status' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.moderation_status AS ENUM ('pending', 'approved', 'rejected');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
# source_type enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'source_type' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.source_type AS ENUM ('manual', 'ocr', 'import');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
print("✅ Enum types ensured.")
|
||||
|
||||
for schema in expected_schemas:
|
||||
print(f"\n--- 🔍 Checking schema '{schema}' ---")
|
||||
|
||||
# Check if schema exists
|
||||
db_schemas = inspector.get_schema_names()
|
||||
if schema not in db_schemas:
|
||||
print(f"❌ Schema '{schema}' missing. Creating...")
|
||||
if apply:
|
||||
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
|
||||
print(f"✅ Schema '{schema}' created.")
|
||||
else:
|
||||
print(f" SQL: CREATE SCHEMA IF NOT EXISTS \"{schema}\"")
|
||||
|
||||
# Get tables in this schema from models
|
||||
model_tables = [t for t in Base.metadata.sorted_tables if t.schema == schema]
|
||||
db_tables = inspector.get_table_names(schema=schema)
|
||||
|
||||
for table in model_tables:
|
||||
if table.name not in db_tables:
|
||||
print(f"❌ Missing table: {schema}.{table.name}")
|
||||
# Generate CREATE TABLE statement
|
||||
create_stmt = CreateTable(table)
|
||||
sql_str = str(create_stmt.compile(bind=engine))
|
||||
print(f" SQL: {sql_str}")
|
||||
if apply:
|
||||
connection.execute(create_stmt)
|
||||
print(f"✅ Table {schema}.{table.name} created.")
|
||||
continue
|
||||
|
||||
# Check columns
|
||||
db_columns = {c['name']: c for c in inspector.get_columns(table.name, schema=schema)}
|
||||
model_columns = table.columns
|
||||
|
||||
missing_cols = []
|
||||
for col in model_columns:
|
||||
if col.name not in db_columns:
|
||||
missing_cols.append(col)
|
||||
|
||||
if missing_cols:
|
||||
print(f"⚠️ Table {schema}.{table.name} missing columns: {[c.name for c in missing_cols]}")
|
||||
for col in missing_cols:
|
||||
col_type = col.type.compile(dialect=engine.dialect)
|
||||
sql = f'ALTER TABLE "{schema}"."{table.name}" ADD COLUMN "{col.name}" {col_type}'
|
||||
if col.nullable is False:
|
||||
sql += " NOT NULL"
|
||||
if col.default is not None:
|
||||
sql += f" DEFAULT {col.default.arg}"
|
||||
print(f" SQL: {sql}")
|
||||
if apply:
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Column {col.name} added.")
|
||||
else:
|
||||
print(f"✅ Table {schema}.{table.name} columns are up‑to‑date.")
|
||||
|
||||
# Check Unique Constraints
|
||||
db_unique_constraints = inspector.get_unique_constraints(table.name, schema=schema)
|
||||
# Map by column names (since constraint names may differ)
|
||||
db_unique_map = {}
|
||||
for uc in db_unique_constraints:
|
||||
key = tuple(sorted(uc['column_names']))
|
||||
db_unique_map[key] = uc['name']
|
||||
|
||||
# Find unique constraints defined in model
|
||||
model_unique_constraints = [c for c in table.constraints if isinstance(c, UniqueConstraint)]
|
||||
for uc in model_unique_constraints:
|
||||
uc_columns = tuple(sorted([col.name for col in uc.columns]))
|
||||
if uc_columns not in db_unique_map:
|
||||
# Constraint missing
|
||||
constraint_name = uc.name or f"uq_{table.name}_{'_'.join(uc_columns)}"
|
||||
columns_sql = ', '.join([f'"{col}"' for col in uc_columns])
|
||||
sql = f'ALTER TABLE "{schema}"."{table.name}" ADD CONSTRAINT "{constraint_name}" UNIQUE ({columns_sql})'
|
||||
print(f"⚠️ Missing unique constraint on {schema}.{table.name} columns {uc_columns}")
|
||||
print(f" SQL: {sql}")
|
||||
if apply:
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Unique constraint {constraint_name} added.")
|
||||
else:
|
||||
print(f"✅ Unique constraint on {uc_columns} exists.")
|
||||
|
||||
# Check Indexes
|
||||
db_indexes = inspector.get_indexes(table.name, schema=schema)
|
||||
db_index_map = {}
|
||||
for idx in db_indexes:
|
||||
key = tuple(sorted(idx['column_names']))
|
||||
db_index_map[key] = idx['name']
|
||||
|
||||
# Find indexes defined in model (Index objects)
|
||||
model_indexes = [idx for idx in table.indexes]
|
||||
for idx in model_indexes:
|
||||
idx_columns = tuple(sorted([col.name for col in idx.columns]))
|
||||
if idx_columns not in db_index_map:
|
||||
# Index missing
|
||||
index_name = idx.name or f"idx_{table.name}_{'_'.join(idx_columns)}"
|
||||
columns_sql = ', '.join([f'"{col}"' for col in idx_columns])
|
||||
unique_sql = "UNIQUE " if idx.unique else ""
|
||||
sql = f'CREATE {unique_sql}INDEX "{index_name}" ON "{schema}"."{table.name}" ({columns_sql})'
|
||||
print(f"⚠️ Missing index on {schema}.{table.name} columns {idx_columns}")
|
||||
print(f" SQL: {sql}")
|
||||
if apply:
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Index {index_name} added.")
|
||||
else:
|
||||
print(f"✅ Index on {idx_columns} exists.")
|
||||
|
||||
print("\n--- ✅ Schema synchronization complete. ---")
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(get_diff_and_repair)
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
async def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Unified Database Synchronizer")
|
||||
parser.add_argument('--apply', action='store_true', help='Apply changes to database (otherwise dry‑run)')
|
||||
args = parser.parse_args()
|
||||
|
||||
print("🚀 Unified Database Synchronizer")
|
||||
print("=" * 50)
|
||||
|
||||
# Step 1: Dynamic import
|
||||
print("\n📥 Step 1: Dynamically importing all models...")
|
||||
dynamic_import_models()
|
||||
|
||||
# Step 2: Compare and repair
|
||||
print("\n🔧 Step 2: Comparing with database and repairing...")
|
||||
await compare_and_repair(apply=args.apply)
|
||||
|
||||
# Step 3: Final verification
|
||||
print("\n📊 Step 3: Final verification...")
|
||||
try:
|
||||
from app.tests_internal.diagnostics.compare_schema import compare
|
||||
await compare()
|
||||
except ImportError:
|
||||
print("⚠️ compare_schema module not found, skipping verification.")
|
||||
|
||||
print("\n✨ Synchronization finished successfully!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
111
archive/old_files/backend/app/services/ai_service_googleApi_old.py.old
Executable file
111
archive/old_files/backend/app/services/ai_service_googleApi_old.py.old
Executable file
@@ -0,0 +1,111 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import asyncio
|
||||
import re
|
||||
from typing import Dict, Any, Optional
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
from sqlalchemy import select
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import SystemParameter
|
||||
|
||||
logger = logging.getLogger("AI-Service")
|
||||
|
||||
class AIService:
|
||||
"""
|
||||
AI Service v1.2.5 - Final Integrated Edition
|
||||
- Robot 2: Technikai dúsítás (Search + Regex JSON parsing)
|
||||
- Robot 3: OCR (Controlled JSON generation)
|
||||
"""
|
||||
api_key = os.getenv("GEMINI_API_KEY")
|
||||
client = genai.Client(api_key=api_key) if api_key else None
|
||||
PRIMARY_MODEL = "gemini-2.0-flash"
|
||||
|
||||
@classmethod
|
||||
async def get_config_delay(cls) -> float:
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
stmt = select(SystemParameter).where(SystemParameter.key == "AI_REQUEST_DELAY")
|
||||
res = await db.execute(stmt)
|
||||
param = res.scalar_one_or_none()
|
||||
return float(param.value) if param else 1.0
|
||||
except Exception: return 1.0
|
||||
|
||||
@classmethod
|
||||
async def get_clean_vehicle_data(cls, make: str, raw_model: str, v_type: str) -> Optional[Dict[str, Any]]:
|
||||
"""Robot 2: Adatbányászat Google Search segítségével."""
|
||||
if not cls.client: return None
|
||||
await asyncio.sleep(await cls.get_config_delay())
|
||||
|
||||
search_tool = types.Tool(google_search=types.GoogleSearch())
|
||||
|
||||
prompt = f"""
|
||||
KERESS RÁ az interneten: {make} {raw_model} ({v_type}) pontos gyári modellkódja és technikai adatai.
|
||||
Adj választ szigorúan csak egy JSON blokkban:
|
||||
{{
|
||||
"marketing_name": "tiszta név",
|
||||
"synonyms": ["név1", "név2"],
|
||||
"technical_code": "gyári kód",
|
||||
"year_from": int,
|
||||
"year_to": int_vagy_null,
|
||||
"ccm": int,
|
||||
"kw": int,
|
||||
"maintenance": {{ "oil_type": "string", "oil_qty": float, "spark_plug": "string", "coolant": "string" }}
|
||||
}}
|
||||
FONTOS: A 'technical_code' NEM lehet üres. Ha nem találod, adj 'N/A' értéket!
|
||||
"""
|
||||
|
||||
# Search tool használata esetén a response_mime_type tilos!
|
||||
config = types.GenerateContentConfig(
|
||||
system_instruction="Profi járműtechnikai adatbányász vagy. Csak tiszta JSON-t válaszolsz markdown kódblokk nélkül.",
|
||||
tools=[search_tool],
|
||||
temperature=0.1
|
||||
)
|
||||
|
||||
try:
|
||||
response = cls.client.models.generate_content(model=cls.PRIMARY_MODEL, contents=prompt, config=config)
|
||||
text = response.text
|
||||
# Tisztítás: ha az AI mégis tenne bele markdown jeleket
|
||||
clean_json = re.sub(r'```json\s*|```', '', text).strip()
|
||||
res_json = json.loads(clean_json)
|
||||
if isinstance(res_json, list) and len(res_json) > 0: res_json = res_json[0]
|
||||
return res_json if isinstance(res_json, dict) else None
|
||||
except Exception as e:
|
||||
logger.error(f"❌ AI hiba ({make} {raw_model}): {e}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def analyze_document_image(cls, image_data: bytes, doc_type: str) -> Optional[Dict[str, Any]]:
|
||||
"""Robot 3: OCR funkció - Forgalmi, Személyi, Számla, Odometer."""
|
||||
if not cls.client: return None
|
||||
await asyncio.sleep(await cls.get_config_delay())
|
||||
|
||||
prompts = {
|
||||
"identity": "Személyes okmány adatok (név, szám, lejárat).",
|
||||
"vehicle_reg": "Forgalmi adatok (rendszám, alvázszám, kW, ccm).",
|
||||
"invoice": "Számla adatok (partner, végösszeg, dátum).",
|
||||
"odometer": "Csak a kilométeróra állása számként."
|
||||
}
|
||||
|
||||
# Itt maradhat a response_mime_type, mert nem használunk Search-öt
|
||||
config = types.GenerateContentConfig(
|
||||
system_instruction="Profi OCR dokumentum-elemző vagy. Csak tiszta JSON-t válaszolsz.",
|
||||
response_mime_type="application/json"
|
||||
)
|
||||
|
||||
try:
|
||||
response = cls.client.models.generate_content(
|
||||
model=cls.PRIMARY_MODEL,
|
||||
contents=[
|
||||
f"Elemezd ezt a képet ({doc_type}): {prompts.get(doc_type, 'OCR')}",
|
||||
types.Part.from_bytes(data=image_data, mime_type="image/jpeg")
|
||||
],
|
||||
config=config
|
||||
)
|
||||
res_json = json.loads(response.text)
|
||||
if isinstance(res_json, list) and len(res_json) > 0: res_json = res_json[0]
|
||||
return res_json if isinstance(res_json, dict) else None
|
||||
except Exception as e:
|
||||
logger.error(f"❌ OCR hiba: {e}")
|
||||
return None
|
||||
@@ -0,0 +1,208 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from sqlalchemy import text, select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.asset import AssetCatalog
|
||||
|
||||
# MB 2.0 Szigorú naplózás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-0-Discovery: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Vehicle-Robot-0-Discovery")
|
||||
|
||||
class DiscoveryEngine:
|
||||
"""
|
||||
THOUGHT PROCESS (IPARI ÜZEMMÓD 2.0):
|
||||
1. Őrkutya (Watchdog): Megkeresi és kiszabadítja a beragadt feladatokat óránként.
|
||||
2. Differential Sync (Különbözeti Szinkron): Csak a hiányzó vagy új modelleket rögzíti, a gold_enriched-eket kihagyja.
|
||||
3. Monthly Scheduler: Havonta egyszer tölti le a teljes RDW adatbázist lapozva.
|
||||
"""
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
SYNC_STATE_FILE = "/app/temp/.last_rdw_sync" # Állapotfájl, hogy Docker újrainduláskor se kezdje elölről azonnal
|
||||
|
||||
@staticmethod
|
||||
async def run_watchdog():
|
||||
""" 1. FÁZIS: Az Őrkutya (Dead-Letter Queue Manager) """
|
||||
logger.info("🐕 Őrkutya: Beragadt feladatok keresése a rendszerben...")
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# A) Hunter takarítás (visszaállítás pending-re, ha a Hunter lefagyott)
|
||||
res1 = await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'pending' WHERE status = 'processing' RETURNING id;"))
|
||||
hunter_resets = len(res1.fetchall())
|
||||
if hunter_resets > 0:
|
||||
logger.warning(f"🔄 {hunter_resets} db beragadt Hunter feladat (processing) visszaállítva 'pending'-re.")
|
||||
|
||||
# B) AI Robotok takarítása (2 órás timeout)
|
||||
query2 = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = CASE
|
||||
WHEN status = 'research_in_progress' THEN 'unverified'
|
||||
WHEN status = 'ai_synthesis_in_progress' THEN 'awaiting_ai_synthesis'
|
||||
END
|
||||
WHERE status IN ('research_in_progress', 'ai_synthesis_in_progress')
|
||||
AND updated_at < NOW() - INTERVAL '2 hours'
|
||||
RETURNING id;
|
||||
""")
|
||||
res2 = await db.execute(query2)
|
||||
ai_resets = len(res2.fetchall())
|
||||
if ai_resets > 0:
|
||||
logger.warning(f"🔄 {ai_resets} db beragadt AI feladat visszaállítva.")
|
||||
|
||||
await db.commit()
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Őrkutya hiba: {e}")
|
||||
|
||||
@staticmethod
|
||||
async def seed_manual_bootstrap():
|
||||
""" 2. FÁZIS: Alapozó adatok rögzítése """
|
||||
initial_data = [
|
||||
{"make": "AUDI", "model": "A4", "generation": "B8 (2008-2015)"}, # vehicle_class törölve
|
||||
{"make": "BMW", "model": "3 SERIES", "generation": "F30 (2012-2019)"}
|
||||
]
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
for item in initial_data:
|
||||
stmt = select(AssetCatalog).where(AssetCatalog.make == item["make"], AssetCatalog.model == item["model"])
|
||||
if not (await db.execute(stmt)).scalar_one_or_none():
|
||||
db.add(AssetCatalog(**item))
|
||||
await db.commit()
|
||||
except Exception as e:
|
||||
logger.warning(f"Manual bootstrap hiba (Ignorálható, ha az adatbázis már tele van): {e}")
|
||||
|
||||
@classmethod
|
||||
async def fetch_with_retry(cls, client: httpx.AsyncClient, url: str, params: dict, retries: int = 3):
|
||||
""" Hibatűrő HTTP kérés API leállások ellen. """
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
resp = await client.get(url, params=params, headers=cls.HEADERS)
|
||||
if resp.status_code == 200:
|
||||
return resp
|
||||
elif resp.status_code == 429:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
else:
|
||||
return None
|
||||
except httpx.RequestError:
|
||||
if attempt == retries - 1:
|
||||
return None
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def seed_from_rdw(cls):
|
||||
""" 3. FÁZIS: Távoli felfedezés - KÜLÖNBÖZETI SZINKRONIZÁCIÓ (Differential Sync) """
|
||||
logger.info("📥 RDW TÖMEGES LETÖLTÉS: Új modellek keresése (Differential Sync)...")
|
||||
|
||||
limit = 10000
|
||||
offset = 0
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
while True:
|
||||
params = {
|
||||
"$select": "merk,handelsbenaming,voertuigsoort,count(*) as total",
|
||||
"$group": "merk,handelsbenaming,voertuigsoort",
|
||||
"$order": "total DESC",
|
||||
"$limit": limit,
|
||||
"$offset": offset
|
||||
}
|
||||
|
||||
resp = await cls.fetch_with_retry(client, "https://opendata.rdw.nl/resource/m9d7-ebf2.json", params)
|
||||
if not resp: break
|
||||
raw_data = resp.json()
|
||||
if not raw_data: break
|
||||
|
||||
logger.info(f"📊 Lapozás: {offset} - {offset + len(raw_data)} tételek analízise...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
for entry in raw_data:
|
||||
make = str(entry.get("merk", "")).upper().strip()
|
||||
model = str(entry.get("handelsbenaming", "")).upper().strip()
|
||||
v_kind = entry.get("voertuigsoort", "")
|
||||
total_count = int(entry.get("total", 0))
|
||||
|
||||
if not make or not model: continue
|
||||
|
||||
if "Personenauto" in v_kind: v_class = 'car'
|
||||
elif "Motorfiets" in v_kind: v_class = 'motorcycle'
|
||||
else: v_class = 'truck'
|
||||
|
||||
# A MÁGIA: Különbözeti Szinkronizáció SQL + Explicit Type Casting
|
||||
query = text("""
|
||||
INSERT INTO vehicle.catalog_discovery (make, model, vehicle_class, status, priority_score)
|
||||
SELECT
|
||||
CAST(:make AS VARCHAR),
|
||||
CAST(:model AS VARCHAR),
|
||||
CAST(:v_class AS VARCHAR),
|
||||
'pending',
|
||||
:priority
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM vehicle.vehicle_model_definitions
|
||||
WHERE make = CAST(:make AS VARCHAR)
|
||||
AND marketing_name = CAST(:model AS VARCHAR)
|
||||
AND status = 'gold_enriched'
|
||||
)
|
||||
ON CONFLICT (make, model)
|
||||
DO UPDATE SET priority_score = EXCLUDED.priority_score
|
||||
WHERE vehicle.catalog_discovery.status != 'processed'
|
||||
RETURNING xmax;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {
|
||||
"make": make, "model": model, "v_class": v_class, "priority": total_count
|
||||
})
|
||||
|
||||
row = result.fetchone()
|
||||
if row:
|
||||
if row[0] == 0: inserted_count += 1 # Új beszúrás
|
||||
else: updated_count += 1 # Meglévő frissítése
|
||||
|
||||
await db.commit()
|
||||
offset += limit
|
||||
await asyncio.sleep(1)
|
||||
|
||||
logger.info(f"✅ RDW Szinkron kész! Új modellek a listán: {inserted_count} | Frissített prioritások: {updated_count}")
|
||||
|
||||
# Sikeres futás regisztrálása a fájlrendszeren
|
||||
os.makedirs(os.path.dirname(cls.SYNC_STATE_FILE), exist_ok=True)
|
||||
with open(cls.SYNC_STATE_FILE, 'w') as f:
|
||||
f.write(datetime.now().isoformat())
|
||||
|
||||
@classmethod
|
||||
def should_run_rdw_sync(cls) -> bool:
|
||||
""" Ellenőrzi, hogy eltelt-e 30 nap a legutóbbi sikeres RDW szinkronizáció óta. """
|
||||
if not os.path.exists(cls.SYNC_STATE_FILE):
|
||||
return True
|
||||
try:
|
||||
with open(cls.SYNC_STATE_FILE, 'r') as f:
|
||||
last_sync = datetime.fromisoformat(f.read().strip())
|
||||
return datetime.now() - last_sync > timedelta(days=30)
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
""" FŐ CIKLUS: Havi ütemező és Óránkénti Őrkutya """
|
||||
logger.info("🚀 ÉLES ÜZEM: Discovery Engine (Differential Sync) & Watchdog indítása...")
|
||||
await cls.seed_manual_bootstrap()
|
||||
|
||||
while True:
|
||||
# 1. Óránkénti takarítás
|
||||
await cls.run_watchdog()
|
||||
|
||||
# 2. Havi szinkronizáció ellenőrzése
|
||||
if cls.should_run_rdw_sync():
|
||||
await cls.seed_from_rdw()
|
||||
else:
|
||||
logger.info("🛌 Az RDW szinkronizáció már lefutott az elmúlt 30 napban. Ugrás...")
|
||||
|
||||
# 3. Alvás 1 órát (Heartbeat)
|
||||
logger.info("⏱️ A Discovery Engine most 1 órát pihen a következő Őrkutya futásig.")
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(DiscoveryEngine.run())
|
||||
@@ -0,0 +1,108 @@
|
||||
# /app/app/workers/vehicle/vehicle_robot_0_strategist.py
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal # MB 2.0 Standard import
|
||||
|
||||
# Sentinel rendszerhez illesztett logolás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s]: %(message)s')
|
||||
logger = logging.getLogger("Vehicle-Robot-0-Strategist")
|
||||
|
||||
class Robot0Strategist:
|
||||
"""
|
||||
THOUGHT PROCESS:
|
||||
1. A robot célja a 'priority_score' meghatározása valós piaci adatok (RDW) alapján.
|
||||
2. Első lépésben ellenőrizzük a sémát (Self-healing), hogy létezik-e az oszlop.
|
||||
3. A kategóriákat (autó, motor, teher) szétválasztjuk, hogy célzott prioritásokat kapjunk.
|
||||
4. Az 'ON CONFLICT' logika garantálja, hogy ne rontsuk el a már feldolgozott (processed) sorokat.
|
||||
5. A prioritás alapja a darabszám: minél több van egy típusból, annál előrébb kerül a listán.
|
||||
"""
|
||||
RDW_API = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
|
||||
# Holland típusok leképezése belső kategóriákra
|
||||
CATEGORIES = [
|
||||
{"name": "car", "rdw_types": ["'Personenauto'"]},
|
||||
{"name": "motorcycle", "rdw_types": ["'Motorfiets'"]},
|
||||
{"name": "truck", "rdw_types": ["'Bedrijfsauto'", "'Vrachtwagen'", "'Opleggertrekker'"]},
|
||||
{"name": "other", "rdw_types": ["NOT IN ('Personenauto', 'Motorfiets', 'Bedrijfsauto', 'Vrachtwagen', 'Opleggertrekker')"]}
|
||||
]
|
||||
|
||||
async def get_popular_makes(self, vehicle_class: str, rdw_types: list):
|
||||
""" Piaci adatok lekérése darabszám szerinti sorrendben. """
|
||||
if "NOT IN" in rdw_types[0]:
|
||||
type_filter = f"voertuigsoort {rdw_types[0]}"
|
||||
else:
|
||||
type_filter = " OR ".join([f"voertuigsoort = {t}" for t in rdw_types])
|
||||
|
||||
params = {
|
||||
"$select": "merk, count(*) AS darabszam",
|
||||
"$where": type_filter,
|
||||
"$group": "merk",
|
||||
"$order": "darabszam DESC",
|
||||
"$limit": 500
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=45.0) as client:
|
||||
try:
|
||||
resp = await client.get(self.RDW_API, params=params, headers=self.HEADERS)
|
||||
if resp.status_code == 200:
|
||||
return resp.json()
|
||||
logger.error(f"⚠️ RDW API Hiba: {resp.status_code}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Kapcsolati hiba az RDW felé: {e}")
|
||||
return []
|
||||
|
||||
async def run(self):
|
||||
logger.info("🚀 Robot 0 (Strategist) ONLINE - Piaci elemzés indítása...")
|
||||
|
||||
# --- SÉMA ELLENŐRZÉS (Golyóálló megoldás) ---
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
await db.execute(text("ALTER TABLE vehicle.catalog_discovery ADD COLUMN IF NOT EXISTS priority_score INTEGER DEFAULT 0;"))
|
||||
await db.commit()
|
||||
logger.info("✅ Adatbázis séma rendben (priority_score aktív).")
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(f"⚠️ Séma hiba: {e}")
|
||||
|
||||
for category in self.CATEGORIES:
|
||||
v_class = category["name"]
|
||||
logger.info(f"📊 {v_class.upper()} hadosztály prioritásainak számítása...")
|
||||
|
||||
makes = await self.get_popular_makes(v_class, category["rdw_types"])
|
||||
if not makes: continue
|
||||
|
||||
added_count = 0
|
||||
for item in makes:
|
||||
make_name = str(item.get("merk", "")).upper().strip()
|
||||
if not make_name: continue
|
||||
|
||||
count = int(item.get("darabszam", 0))
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
# UPSERT: Beállítjuk a prioritást, de nem bántjuk a már kész rekordokat
|
||||
query = text("""
|
||||
INSERT INTO vehicle.catalog_discovery (make, model, vehicle_class, status, source, attempts, priority_score)
|
||||
VALUES (:make, 'ALL_VARIANTS', :class, 'pending', 'STRATEGIST-V2', 0, :score)
|
||||
ON CONFLICT (make, model, vehicle_class)
|
||||
DO UPDATE SET priority_score = :score
|
||||
WHERE vehicle.catalog_discovery.status NOT IN ('processed', 'in_progress');
|
||||
""")
|
||||
|
||||
await db.execute(query, {"make": make_name, "class": v_class, "score": count})
|
||||
await db.commit()
|
||||
added_count += 1
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.warning(f"❌ Hiba a márka rögzítésekor ({make_name}): {e}")
|
||||
|
||||
logger.info(f"✅ {v_class.upper()} kategória kész: {added_count} márka rangsorolva.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(Robot0Strategist().run())
|
||||
@@ -0,0 +1,224 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
# Naplózás beállítása a standard kimenetre
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s',
|
||||
stream=sys.stdout
|
||||
)
|
||||
logger = logging.getLogger("Robot-1-Hunter")
|
||||
|
||||
class CatalogHunter:
|
||||
"""
|
||||
Vehicle Robot 1.9.3: The Truly Invincible Hunter (SAVEPOINT PATCH)
|
||||
Kezeli az ALL_VARIANTS utasítást és row-level tranzakcióvédelmet használ.
|
||||
"""
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
if not text_val: return ""
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower()
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError): return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError): return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_with_retry(cls, client: httpx.AsyncClient, url: str, retries: int = 3):
|
||||
""" Hibatűrő HTTP lekérdezés exponenciális várakozással. """
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
resp = await client.get(url, headers=cls.HEADERS)
|
||||
if resp.status_code == 200:
|
||||
return resp
|
||||
elif resp.status_code == 429: # Rate limit
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
else:
|
||||
return resp
|
||||
except httpx.RequestError as e:
|
||||
if attempt == retries - 1:
|
||||
logger.debug(f"Hálózati hiba: {e}")
|
||||
raise
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
""" Technikai adatok (üzemanyag, teljesítmény, motorkód) begyűjtése. """
|
||||
results = {
|
||||
"power_kw": 0, "engine_code": None, "euro_class": None,
|
||||
"fuel_desc": "Unknown", "co2": 0, "consumption": 0.0
|
||||
}
|
||||
try:
|
||||
# Üzemanyag adatok
|
||||
f_resp = await cls.fetch_with_retry(client, f"{cls.RDW_FUEL}?kenteken={plate}")
|
||||
if f_resp and f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1 = cls.parse_int(f.get("netto_maximum_vermogen") or f.get("nettomaximumvermogen"))
|
||||
p2 = cls.parse_int(f.get("nominaal_continu_maximum_vermogen") or f.get("nominaalcontinuvermogen"))
|
||||
results.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
|
||||
# Motorkód adatok
|
||||
e_resp = await cls.fetch_with_retry(client, f"{cls.RDW_ENGINE}?kenteken={plate}")
|
||||
if e_resp and e_resp.status_code == 200 and e_resp.json():
|
||||
results["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception:
|
||||
pass
|
||||
return results
|
||||
|
||||
@classmethod
|
||||
async def process_make_model(cls, db, task_id, make_name, model_name, v_class, priority):
|
||||
""" Egy adott márka/modell (vagy wildcard) feldolgozása. """
|
||||
clean_make = make_name.strip().upper()
|
||||
clean_model = model_name.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
offset = 0
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
while True:
|
||||
# Dinamikus paraméterezés: ALL_VARIANTS esetén nem szűrünk modellre
|
||||
if clean_model == 'ALL_VARIANTS':
|
||||
params = f"merk={clean_make}&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
else:
|
||||
params = f"merk={clean_make}&handelsbenaming={clean_model}&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
|
||||
try:
|
||||
r = await cls.fetch_with_retry(client, f"{cls.RDW_MAIN}?{params}")
|
||||
batch = r.json() if r and r.status_code == 200 else []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ API hiba: {e}")
|
||||
break
|
||||
|
||||
if not batch:
|
||||
break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
# SAVEPOINT: Ha egy rekord mentése hibás, a tranzakció blokk nem sérül
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
|
||||
# Valódi modellnév kinyerése (Wildcard esetén fontos)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
make=clean_make,
|
||||
marketing_name=actual_model,
|
||||
normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"),
|
||||
version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
type_approval_number=item.get("typegoedkeuringsnummer"),
|
||||
technical_code=plate,
|
||||
engine_capacity=cls.parse_int(item.get("cilinderinhoud")),
|
||||
power_kw=tech["power_kw"],
|
||||
fuel_type=tech["fuel_desc"],
|
||||
engine_code=tech["engine_code"],
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")),
|
||||
doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
width=cls.parse_int(item.get("breedte")),
|
||||
wheelbase=cls.parse_int(item.get("wielbasis")),
|
||||
list_price=cls.parse_int(item.get("catalogusprijs")),
|
||||
max_speed=cls.parse_int(item.get("maximale_constructiesnelheid")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")),
|
||||
max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
body_type=item.get("inrichting"),
|
||||
co2_emissions_combined=tech["co2"],
|
||||
fuel_consumption_combined=tech["consumption"],
|
||||
euro_classification=tech["euro_class"],
|
||||
cylinders=cls.parse_int(item.get("aantal_cilinders")),
|
||||
vehicle_class=v_class,
|
||||
priority_score=priority,
|
||||
status="unverified", # R2 Researcher számára előkészítve
|
||||
source="MEGA-HUNTER-v1.9.3"
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type']
|
||||
)
|
||||
await db.execute(stmt)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Sor eldobva ({plate}): {e}")
|
||||
|
||||
# Batch commit a sikeres sorok után
|
||||
await db.commit()
|
||||
|
||||
offset += len(batch)
|
||||
if offset >= 500: # Biztonsági korlát egy-egy márkánál
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Discovery feladat lezárása
|
||||
await db.execute(
|
||||
text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"),
|
||||
{"id": task_id}
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v1.9.3 ONLINE (SAVEPOINT ENABLED)")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# ATOMI ZÁROLÁS: Keresés, Zárolás és Állapotváltás egy lépésben
|
||||
query = text("""
|
||||
UPDATE vehicle.catalog_discovery
|
||||
SET status = 'processing'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.catalog_discovery
|
||||
WHERE status = 'pending'
|
||||
ORDER BY priority_score DESC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, make, model, vehicle_class, priority_score;
|
||||
""")
|
||||
|
||||
result = await db.execute(query)
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
|
||||
if task:
|
||||
await cls.process_make_model(db, task[0], task[1], task[2], task[3], task[4])
|
||||
else:
|
||||
# Ha nincs munka, 30 másodperc pihenő
|
||||
await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Főciklus hiba: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -0,0 +1,179 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/workers/vehicle/vehicle_robot_1_catalog_hunter.py
|
||||
# version: 1.9.6
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from sqlalchemy import text, func
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
# MB 2.0 Standard Naplózás
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s',
|
||||
stream=sys.stdout
|
||||
)
|
||||
logger = logging.getLogger("Robot-1-Hunter")
|
||||
|
||||
class CatalogHunter:
|
||||
"""
|
||||
Vehicle Robot 1.9.6: Mega-Hunter (TIMESTAMP & INTEGRITY PATCH)
|
||||
Kezeli az ALL_VARIANTS-t, a Savepoint-okat és az összes kötelező mezőt.
|
||||
"""
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
if not text_val: return ""
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower()
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError): return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError): return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_with_retry(cls, client: httpx.AsyncClient, url: str, retries: int = 3):
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
resp = await client.get(url, headers=cls.HEADERS)
|
||||
if resp.status_code == 200: return resp
|
||||
elif resp.status_code == 429: await asyncio.sleep(2 ** attempt)
|
||||
else: return resp
|
||||
except httpx.RequestError:
|
||||
if attempt == retries - 1: raise
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
results = {"power_kw": 0, "engine_code": None, "euro_class": None, "fuel_desc": "Unknown", "co2": 0, "consumption": 0.0}
|
||||
try:
|
||||
f_resp = await cls.fetch_with_retry(client, f"{cls.RDW_FUEL}?kenteken={plate}")
|
||||
if f_resp and f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1 = cls.parse_int(f.get("netto_maximum_vermogen") or f.get("nettomaximumvermogen"))
|
||||
p2 = cls.parse_int(f.get("nominaal_continu_maximum_vermogen") or f.get("nominaalcontinuvermogen"))
|
||||
results.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
e_resp = await cls.fetch_with_retry(client, f"{cls.RDW_ENGINE}?kenteken={plate}")
|
||||
if e_resp and e_resp.status_code == 200 and e_resp.json():
|
||||
results["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception: pass
|
||||
return results
|
||||
|
||||
@classmethod
|
||||
async def process_make_model(cls, db, task_id, make_name, model_name, v_class, priority):
|
||||
clean_make = make_name.strip().upper()
|
||||
clean_model = model_name.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
offset = 0
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
while True:
|
||||
if clean_model == 'ALL_VARIANTS':
|
||||
params = f"merk={clean_make}&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
else:
|
||||
params = f"merk={clean_make}&handelsbenaming={clean_model}&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
|
||||
try:
|
||||
r = await cls.fetch_with_retry(client, f"{cls.RDW_MAIN}?{params}")
|
||||
batch = r.json() if r and r.status_code == 200 else []
|
||||
except Exception: break
|
||||
|
||||
if not batch: break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
make=clean_make,
|
||||
marketing_name=actual_model,
|
||||
normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"),
|
||||
version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
technical_code=plate,
|
||||
engine_capacity=cls.parse_int(item.get("cilinderinhoud")),
|
||||
power_kw=tech["power_kw"],
|
||||
fuel_type=tech["fuel_desc"],
|
||||
engine_code=tech["engine_code"],
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")),
|
||||
doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")),
|
||||
max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
vehicle_class=v_class,
|
||||
priority_score=priority,
|
||||
market='EU', # KÖTELEZŐ
|
||||
status="unverified",
|
||||
is_manual=False,
|
||||
created_at=func.now(), # KÖTELEZŐ DÁTUMOK
|
||||
updated_at=func.now(),
|
||||
source="MEGA-HUNTER-v1.9.6"
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type']
|
||||
)
|
||||
await db.execute(stmt)
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Sor eldobva ({plate}): {e}")
|
||||
|
||||
await db.commit()
|
||||
offset += len(batch)
|
||||
if offset >= 500: break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task_id})
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v1.9.6 ONLINE (TIMESTAMP PATCH)")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
query = text("""
|
||||
UPDATE vehicle.catalog_discovery SET status = 'processing'
|
||||
WHERE id = (SELECT id FROM vehicle.catalog_discovery WHERE status = 'pending'
|
||||
ORDER BY priority_score DESC FOR UPDATE SKIP LOCKED LIMIT 1)
|
||||
RETURNING id, make, model, vehicle_class, priority_score;
|
||||
""")
|
||||
result = await db.execute(query)
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
if task: await cls.process_make_model(db, task[0], task[1], task[2], task[3], task[4])
|
||||
else: await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Főciklus hiba: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -0,0 +1,168 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Robot-1")
|
||||
|
||||
class CatalogHunter:
|
||||
"""
|
||||
Vehicle Robot 2.1.2: A Végleges Vadász
|
||||
Tökéletes adattípus szinkron. raw_search_context -> string.
|
||||
"""
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
if not text_val: return "UNKNOWN"
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower()
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError): return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError): return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
res = {"power_kw": 0, "engine_code": None, "euro_class": None, "fuel_desc": "Unknown", "co2": 0, "consumption": 0.0}
|
||||
try:
|
||||
f_resp = await client.get(f"{cls.RDW_FUEL}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1 = cls.parse_int(f.get("netto_maximum_vermogen"))
|
||||
p2 = cls.parse_int(f.get("nominaal_continu_maximum_vermogen"))
|
||||
res.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
e_resp = await client.get(f"{cls.RDW_ENGINE}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if e_resp.status_code == 200 and e_resp.json():
|
||||
res["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception: pass
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def process_task(cls, db, task):
|
||||
clean_make = task.make.strip().upper()
|
||||
clean_model = task.model.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
offset = 0
|
||||
while True:
|
||||
params = f"merk={clean_make}"
|
||||
if clean_model != 'ALL_VARIANTS':
|
||||
params += f"&handelsbenaming={clean_model}"
|
||||
params += f"&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
|
||||
try:
|
||||
r = await client.get(f"{cls.RDW_MAIN}?{params}", headers=cls.HEADERS)
|
||||
batch = r.json() if r.status_code == 200 else []
|
||||
except Exception: break
|
||||
if not batch: break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
datum_eerste_toelating = str(item.get("datum_eerste_toelating", ""))
|
||||
year_from = cls.parse_int(datum_eerste_toelating[:4]) if len(datum_eerste_toelating) >= 4 else 0
|
||||
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
market='EU',
|
||||
make=clean_make,
|
||||
marketing_name=actual_model,
|
||||
normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"),
|
||||
version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
technical_code=plate,
|
||||
type_approval_number=item.get("typegoedkeuringsnummer"),
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")),
|
||||
doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
width=cls.parse_int(item.get("breedte")),
|
||||
wheelbase=cls.parse_int(item.get("wielbasis")),
|
||||
list_price=cls.parse_int(item.get("catalogusprijs")),
|
||||
max_speed=cls.parse_int(item.get("maximale_constructiesnelheid")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")),
|
||||
max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
fuel_consumption_combined=tech["consumption"],
|
||||
co2_emissions_combined=tech["co2"],
|
||||
vehicle_class=task.vehicle_class,
|
||||
body_type=item.get("inrichting"),
|
||||
fuel_type=tech["fuel_desc"],
|
||||
engine_capacity=cls.parse_int(item.get("cilinderinhoud")),
|
||||
power_kw=tech["power_kw"],
|
||||
cylinders=cls.parse_int(item.get("aantal_cilinders")),
|
||||
engine_code=tech["engine_code"],
|
||||
euro_classification=tech["euro_class"],
|
||||
year_from=year_from,
|
||||
priority_score=task.priority_score,
|
||||
status="unverified",
|
||||
source="MEGA-HUNTER-v2.1.2",
|
||||
# JAVÍTÁS: A raw_search_context most már üres STRING (''), ahogy a modell elvárja!
|
||||
raw_search_context='',
|
||||
research_metadata={},
|
||||
specifications={},
|
||||
marketing_name_aliases=[]
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type', 'market', 'year_from']
|
||||
)
|
||||
await db.execute(stmt)
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Sor hiba ({plate}): {e}")
|
||||
|
||||
await db.commit()
|
||||
offset += len(batch)
|
||||
if offset >= 500: break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task.id})
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v2.1.2 (Adattípus Fix) ONLINE")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
query = text("UPDATE vehicle.catalog_discovery SET status = 'processing' WHERE id = (SELECT id FROM vehicle.catalog_discovery WHERE status = 'pending' ORDER BY priority_score DESC FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id, make, model, vehicle_class, priority_score;")
|
||||
res = await db.execute(query)
|
||||
task = res.fetchone()
|
||||
await db.commit()
|
||||
if task: await cls.process_task(db, task)
|
||||
else: await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Főciklus hiba: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -0,0 +1,205 @@
|
||||
# /app/app/workers/vehicle/vehicle_robot_1_catalog_hunter.py
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Robot-1")
|
||||
|
||||
class CatalogHunter:
|
||||
"""
|
||||
Vehicle Robot 2.2.0: Fast-Track to Gold Edition
|
||||
Ha az RDW-ből megvan minden kulcsadat (kw, ccm, fuel), azonnal 'gold_enriched'-re teszi a járművet
|
||||
és beírja a vehicle_catalog mestertáblába!
|
||||
"""
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
if not text_val: return "UNKNOWN"
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower()
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError): return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try:
|
||||
if value is None or str(value).strip() == "": return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError): return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
res = {"power_kw": 0, "engine_code": None, "euro_class": None, "fuel_desc": "Unknown", "co2": 0, "consumption": 0.0}
|
||||
try:
|
||||
f_resp = await client.get(f"{cls.RDW_FUEL}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1 = cls.parse_int(f.get("netto_maximum_vermogen"))
|
||||
p2 = cls.parse_int(f.get("nominaal_continu_maximum_vermogen"))
|
||||
res.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
e_resp = await client.get(f"{cls.RDW_ENGINE}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if e_resp.status_code == 200 and e_resp.json():
|
||||
res["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception: pass
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def process_task(cls, db, task):
|
||||
clean_make = task.make.strip().upper()
|
||||
clean_model = task.model.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
offset = 0
|
||||
while True:
|
||||
params = f"merk={clean_make}"
|
||||
if clean_model != 'ALL_VARIANTS':
|
||||
params += f"&handelsbenaming={clean_model}"
|
||||
params += f"&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
|
||||
try:
|
||||
r = await client.get(f"{cls.RDW_MAIN}?{params}", headers=cls.HEADERS)
|
||||
batch = r.json() if r.status_code == 200 else []
|
||||
except Exception: break
|
||||
if not batch: break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
datum_eerste_toelating = str(item.get("datum_eerste_toelating", ""))
|
||||
year_from = cls.parse_int(datum_eerste_toelating[:4]) if len(datum_eerste_toelating) >= 4 else 0
|
||||
|
||||
engine_ccm = cls.parse_int(item.get("cilinderinhoud"))
|
||||
power_kw = tech["power_kw"]
|
||||
fuel_type = tech["fuel_desc"]
|
||||
|
||||
# FAST-TRACK LOGIKA: Ha a kötelező műszaki adatok megvannak, azonnal ARANY minősítést kap!
|
||||
# Villanyautóknál a CCM lehet 0, ezt is kezeljük.
|
||||
is_gold = False
|
||||
if (power_kw > 0 and engine_ccm > 0) or (power_kw > 0 and "elektri" in fuel_type.lower()):
|
||||
is_gold = True
|
||||
|
||||
final_status = "gold_enriched" if is_gold else "unverified"
|
||||
|
||||
# 1. Beírjuk a VMD-be (Staging tábla)
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
market='EU',
|
||||
make=clean_make,
|
||||
marketing_name=actual_model,
|
||||
normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"),
|
||||
version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
technical_code=plate,
|
||||
type_approval_number=item.get("typegoedkeuringsnummer"),
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")),
|
||||
doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
width=cls.parse_int(item.get("breedte")),
|
||||
wheelbase=cls.parse_int(item.get("wielbasis")),
|
||||
list_price=cls.parse_int(item.get("catalogusprijs")),
|
||||
max_speed=cls.parse_int(item.get("maximale_constructiesnelheid")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")),
|
||||
max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
fuel_consumption_combined=tech["consumption"],
|
||||
co2_emissions_combined=tech["co2"],
|
||||
vehicle_class=task.vehicle_class,
|
||||
body_type=item.get("inrichting"),
|
||||
fuel_type=fuel_type,
|
||||
engine_capacity=engine_ccm,
|
||||
power_kw=power_kw,
|
||||
cylinders=cls.parse_int(item.get("aantal_cilinders")),
|
||||
engine_code=tech["engine_code"],
|
||||
euro_classification=tech["euro_class"],
|
||||
year_from=year_from,
|
||||
priority_score=task.priority_score,
|
||||
status=final_status, # Dinamikus státusz
|
||||
source="MEGA-HUNTER-v2.2.0-FAST",
|
||||
raw_search_context='',
|
||||
research_metadata={},
|
||||
specifications={"fast_track": True}, # Jelezzük, hogy ez RDW-ből jött közvetlenül
|
||||
marketing_name_aliases=[]
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type', 'market', 'year_from']
|
||||
).returning(VehicleModelDefinition.id)
|
||||
|
||||
res = await db.execute(stmt)
|
||||
vmd_id = res.scalar()
|
||||
|
||||
# 2. HA ARANY, AZONNAL LÉPÜNK A VÉGSŐ KATALÓGUSBA (Ahogy az Alchemist is tenné)
|
||||
if is_gold and vmd_id:
|
||||
cat_stmt = text("""
|
||||
INSERT INTO vehicle.vehicle_catalog
|
||||
(master_definition_id, make, model, power_kw, engine_capacity, fuel_type, factory_data)
|
||||
VALUES (:m_id, :make, :model, :kw, :ccm, :fuel, :factory)
|
||||
ON CONFLICT ON CONSTRAINT uix_vehicle_catalog_full DO NOTHING;
|
||||
""")
|
||||
await db.execute(cat_stmt, {
|
||||
"m_id": vmd_id,
|
||||
"make": clean_make,
|
||||
"model": actual_model[:50],
|
||||
"kw": power_kw,
|
||||
"ccm": engine_ccm,
|
||||
"fuel": fuel_type,
|
||||
"factory": json.dumps({"source": "RDW API Direct", "verified": True})
|
||||
})
|
||||
logger.info(f"✨ FAST-TRACK ARANY: {clean_make} {actual_model} (KW: {power_kw}, CCM: {engine_ccm})")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Sor hiba ({plate}): {e}")
|
||||
|
||||
await db.commit()
|
||||
offset += len(batch)
|
||||
if offset >= 500: break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task.id})
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v2.2.0 (Fast-Track Edition) ONLINE")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
query = text("UPDATE vehicle.catalog_discovery SET status = 'processing' WHERE id = (SELECT id FROM vehicle.catalog_discovery WHERE status = 'pending' ORDER BY priority_score DESC FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id, make, model, vehicle_class, priority_score;")
|
||||
res = await db.execute(query)
|
||||
task = res.fetchone()
|
||||
await db.commit()
|
||||
if task: await cls.process_task(db, task)
|
||||
else: await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Főciklus hiba: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -0,0 +1,140 @@
|
||||
import asyncio, httpx, logging, os, re, sys, json
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-1-Hunter: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Robot-1")
|
||||
|
||||
class CatalogHunter:
|
||||
RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json"
|
||||
RDW_ENGINE = "https://opendata.rdw.nl/resource/jh96-v4pq.json"
|
||||
RDW_TOKEN = os.getenv("RDW_APP_TOKEN")
|
||||
HEADERS = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {}
|
||||
BATCH_SIZE = 50
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, text_val: str) -> str:
|
||||
return re.sub(r'[^a-zA-Z0-9]', '', text_val).lower() if text_val else "UNKNOWN"
|
||||
|
||||
@classmethod
|
||||
def parse_int(cls, value) -> int:
|
||||
try: return int(float(value)) if value and str(value).strip() else 0
|
||||
except: return 0
|
||||
|
||||
@classmethod
|
||||
def parse_float(cls, value) -> float:
|
||||
try: return float(value) if value and str(value).strip() else 0.0
|
||||
except: return 0.0
|
||||
|
||||
@classmethod
|
||||
async def fetch_tech_details(cls, client, plate):
|
||||
res = {"power_kw": 0, "engine_code": None, "euro_class": None, "fuel_desc": "Unknown", "co2": 0, "consumption": 0.0}
|
||||
try:
|
||||
f_resp = await client.get(f"{cls.RDW_FUEL}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if f_resp.status_code == 200 and f_resp.json():
|
||||
f = f_resp.json()[0]
|
||||
p1, p2 = cls.parse_int(f.get("netto_maximum_vermogen")), cls.parse_int(f.get("nominaal_continu_maximum_vermogen"))
|
||||
res.update({
|
||||
"power_kw": max(p1, p2),
|
||||
"fuel_desc": f.get("brandstof_omschrijving") or "Unknown",
|
||||
"euro_class": f.get("euro_klasse") or f.get("uitlaatemissieniveau"),
|
||||
"co2": cls.parse_int(f.get("co2_uitstoot_gecombineerd")),
|
||||
"consumption": cls.parse_float(f.get("brandstofverbruik_gecombineerd"))
|
||||
})
|
||||
e_resp = await client.get(f"{cls.RDW_ENGINE}?kenteken={plate}", headers=cls.HEADERS)
|
||||
if e_resp.status_code == 200 and e_resp.json():
|
||||
res["engine_code"] = e_resp.json()[0].get("motorcode")
|
||||
except Exception: pass
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def process_task(cls, db, task):
|
||||
clean_make, clean_model = task.make.strip().upper(), task.model.strip().upper()
|
||||
logger.info(f"🎯 ADATGYŰJTÉS INDUL: {clean_make} {clean_model}")
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
offset = 0
|
||||
while True:
|
||||
params = f"merk={clean_make}" + (f"&handelsbenaming={clean_model}" if clean_model != 'ALL_VARIANTS' else "") + f"&$limit={cls.BATCH_SIZE}&$offset={offset}&$order=kenteken DESC"
|
||||
try:
|
||||
r = await client.get(f"{cls.RDW_MAIN}?{params}", headers=cls.HEADERS)
|
||||
batch = r.json() if r.status_code == 200 else []
|
||||
except Exception: break
|
||||
if not batch: break
|
||||
|
||||
for item in batch:
|
||||
plate = item.get("kenteken", "UNKNOWN")
|
||||
try:
|
||||
async with db.begin_nested():
|
||||
tech = await cls.fetch_tech_details(client, plate)
|
||||
actual_model = (item.get("handelsbenaming") or clean_model).upper()
|
||||
norm_name = cls.normalize(actual_model.replace(clean_make, "").strip() or actual_model)
|
||||
|
||||
datum = str(item.get("datum_eerste_toelating", ""))
|
||||
year_from = cls.parse_int(datum[:4]) if len(datum) >= 4 else 0
|
||||
|
||||
engine_ccm, power_kw, fuel_type = cls.parse_int(item.get("cilinderinhoud")), tech["power_kw"], tech["fuel_desc"]
|
||||
|
||||
# FAST-TRACK LOGIKA: Ha van KW és CCM, egyből ARANY!
|
||||
is_gold = (power_kw > 0 and engine_ccm > 0) or (power_kw > 0 and "elektri" in fuel_type.lower())
|
||||
final_status = "gold_enriched" if is_gold else "unverified"
|
||||
|
||||
stmt = insert(VehicleModelDefinition).values(
|
||||
market='EU', make=clean_make, marketing_name=actual_model, normalized_name=norm_name,
|
||||
variant_code=item.get("variant", "UNKNOWN"), version_code=item.get("uitvoering", "UNKNOWN"),
|
||||
technical_code=plate, type_approval_number=item.get("typegoedkeuringsnummer"),
|
||||
seats=cls.parse_int(item.get("aantal_zitplaatsen")), doors=cls.parse_int(item.get("aantal_deuren")),
|
||||
width=cls.parse_int(item.get("breedte")), wheelbase=cls.parse_int(item.get("wielbasis")),
|
||||
list_price=cls.parse_int(item.get("catalogusprijs")), max_speed=cls.parse_int(item.get("maximale_constructiesnelheid")),
|
||||
curb_weight=cls.parse_int(item.get("massa_ledig_voertuig")), max_weight=cls.parse_int(item.get("technische_max_massa_voertuig")),
|
||||
fuel_consumption_combined=tech["consumption"], co2_emissions_combined=tech["co2"],
|
||||
vehicle_class=task.vehicle_class, body_type=item.get("inrichting"), fuel_type=fuel_type,
|
||||
engine_capacity=engine_ccm, power_kw=power_kw, cylinders=cls.parse_int(item.get("aantal_cilinders")),
|
||||
engine_code=tech["engine_code"], euro_classification=tech["euro_class"], year_from=year_from,
|
||||
priority_score=task.priority_score, status=final_status, source="MEGA-HUNTER-v2.2.0-FAST",
|
||||
raw_search_context='', research_metadata={}, specifications={"fast_track": True} if is_gold else {}, marketing_name_aliases=[]
|
||||
).on_conflict_do_nothing(
|
||||
index_elements=['make', 'normalized_name', 'variant_code', 'version_code', 'fuel_type', 'market', 'year_from']
|
||||
).returning(VehicleModelDefinition.id)
|
||||
|
||||
res = await db.execute(stmt)
|
||||
vmd_id = res.scalar()
|
||||
|
||||
# Automatikus Publikálás (Ha Arany)
|
||||
if is_gold and vmd_id:
|
||||
cat_stmt = text("""
|
||||
INSERT INTO vehicle.vehicle_catalog (master_definition_id, make, model, power_kw, engine_capacity, fuel_type, factory_data)
|
||||
VALUES (:m_id, :make, :model, :kw, :ccm, :fuel, :factory)
|
||||
ON CONFLICT ON CONSTRAINT uix_vehicle_catalog_full DO NOTHING;
|
||||
""")
|
||||
await db.execute(cat_stmt, {"m_id": vmd_id, "make": clean_make, "model": actual_model[:50], "kw": power_kw, "ccm": engine_ccm, "fuel": fuel_type, "factory": '{"source": "RDW Fast-Track"}'})
|
||||
logger.info(f"✨ FAST-TRACK ARANY: {clean_make} {actual_model}")
|
||||
|
||||
except Exception as e: logger.warning(f"⚠️ Sor hiba ({plate}): {e}")
|
||||
|
||||
await db.commit()
|
||||
offset += len(batch)
|
||||
if offset >= 500: break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task.id})
|
||||
await db.commit()
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🤖 Mega-Hunter v2.2.0 (Fast-Track) ONLINE")
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
res = await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processing' WHERE id = (SELECT id FROM vehicle.catalog_discovery WHERE status = 'pending' ORDER BY priority_score DESC FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id, make, model, vehicle_class, priority_score;"))
|
||||
task = res.fetchone()
|
||||
await db.commit()
|
||||
if task: await cls.process_task(db, task)
|
||||
else: await asyncio.sleep(30)
|
||||
except Exception: await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CatalogHunter.run())
|
||||
@@ -0,0 +1,239 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/workers/vehicle/vehicle_robot_2_researcher.py
|
||||
import asyncio
|
||||
import logging
|
||||
import warnings
|
||||
import os
|
||||
import json
|
||||
from datetime import datetime
|
||||
from sqlalchemy import text, update, func
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning, module='duckduckgo_search')
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
# MB 2.0 Szabvány naplózás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Robot-2-Researcher: %(message)s')
|
||||
logger = logging.getLogger("Vehicle-Robot-2-Researcher")
|
||||
|
||||
class QuotaManager:
|
||||
""" Szigorú napi limit figyelő a fizetős/hatósági API-khoz """
|
||||
def __init__(self, service_name: str, daily_limit: int):
|
||||
self.service_name = service_name
|
||||
self.daily_limit = daily_limit
|
||||
self.state_file = f"/app/temp/.quota_{service_name}.json"
|
||||
self._ensure_file()
|
||||
|
||||
def _ensure_file(self):
|
||||
os.makedirs(os.path.dirname(self.state_file), exist_ok=True)
|
||||
if not os.path.exists(self.state_file):
|
||||
with open(self.state_file, 'w') as f:
|
||||
json.dump({"date": datetime.now().strftime("%Y-%m-%d"), "count": 0}, f)
|
||||
|
||||
def can_make_request(self) -> bool:
|
||||
with open(self.state_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
if data["date"] != today:
|
||||
data = {"date": today, "count": 0} # Új nap, kvóta nullázása
|
||||
|
||||
if data["count"] >= self.daily_limit:
|
||||
return False
|
||||
|
||||
# Növeljük a számlálót
|
||||
data["count"] += 1
|
||||
with open(self.state_file, 'w') as f:
|
||||
json.dump(data, f)
|
||||
return True
|
||||
|
||||
class VehicleResearcher:
|
||||
"""
|
||||
Vehicle Robot 2.5: Sniper Researcher (Mesterlövész Adatgyűjtő)
|
||||
Célzott keresésekkel és strukturált aktakészítéssel dolgozik az AI kímélése érdekében.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.max_attempts = 5
|
||||
self.search_timeout = 15.0
|
||||
|
||||
# Kvóta menedzserek beállítása (.env-ből olvasva)
|
||||
dvla_limit = int(os.getenv("DVLA_DAILY_LIMIT", "1000"))
|
||||
self.dvla_quota = QuotaManager("dvla", dvla_limit)
|
||||
self.dvla_token = os.getenv("DVLA_API_KEY")
|
||||
|
||||
async def fetch_ddg_targeted(self, label: str, query: str) -> str:
|
||||
""" Célzott keresés szálbiztosan a DuckDuckGo-n. """
|
||||
try:
|
||||
def search():
|
||||
with DDGS() as ddgs:
|
||||
# max_results=2: Nem kell sok zaj, csak a legrelevánsabb 2 találat
|
||||
results = ddgs.text(query, max_results=2)
|
||||
return [f"- {r.get('body', '')}" for r in results] if results else []
|
||||
|
||||
results = await asyncio.wait_for(asyncio.to_thread(search), timeout=self.search_timeout)
|
||||
|
||||
if not results:
|
||||
return f"[SOURCE: {label}]\nNincs érdemi találat.\n"
|
||||
|
||||
content = f"[SOURCE: {label} | KERESÉS: {query}]\n"
|
||||
content += "\n".join(results) + "\n"
|
||||
return content
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Keresési hiba ({label}): {e}")
|
||||
return f"[SOURCE: {label}]\nKERESÉSI HIBA.\n"
|
||||
|
||||
def extract_specs_from_text(self, text: str) -> dict:
|
||||
""" Regex alapú kinyerés a nyers szövegből: ccm, kW, motoradatok. """
|
||||
import re
|
||||
specs = {}
|
||||
|
||||
# CCM (köbcentiméter) minta: 1998 cc, 2.0 L, 2000 cm³
|
||||
ccm_pattern = r'(\d{3,4})\s*(?:cc|ccm|cm³|cm3|cc\.)'
|
||||
match = re.search(ccm_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['ccm'] = int(match.group(1))
|
||||
else:
|
||||
# Alternatív minta: 2.0 liter -> 2000 cc
|
||||
liter_pattern = r'(\d+\.?\d*)\s*(?:L|liter|ℓ)'
|
||||
match = re.search(liter_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
liters = float(match.group(1))
|
||||
specs['ccm'] = int(liters * 1000)
|
||||
|
||||
# KW (kilowatt) minta: 150 kW, 150kW, 150 KW
|
||||
kw_pattern = r'(\d{2,4})\s*(?:kW|kw|KW)'
|
||||
match = re.search(kw_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['kw'] = int(match.group(1))
|
||||
else:
|
||||
# Le (lóerő) átváltás: 150 LE -> 110 kW (kb)
|
||||
hp_pattern = r'(\d{2,4})\s*(?:HP|hp|LE|le|Ps)'
|
||||
match = re.search(hp_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
hp = int(match.group(1))
|
||||
specs['kw'] = int(hp * 0.7355) # hozzávetőleges átváltás
|
||||
|
||||
# Motor kód minta: motor kód: 1.8 TSI, engine code: N47
|
||||
engine_pattern = r'(?:motor\s*kód|engine\s*code|motor\s*code)[:\s]+([A-Z0-9\.\- ]+)'
|
||||
match = re.search(engine_pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
specs['engine_code'] = match.group(1).strip()
|
||||
|
||||
return specs
|
||||
|
||||
async def research_vehicle(self, db, vehicle_id: int, make: str, model: str, engine: str, year: str, current_attempts: int):
|
||||
""" Egy jármű átvilágítása és a strukturált 'Akta' elkészítése a GPU számára. """
|
||||
engine_safe = engine or ""
|
||||
year_safe = str(year) if year else ""
|
||||
|
||||
logger.info(f"🔎 Mesterlövész Kutatás: {make} {model} (Motor: {engine_safe})")
|
||||
|
||||
# 1. TIER: Ingyenes, Célzott Keresések (A legmegbízhatóbb források)
|
||||
queries = [
|
||||
("ULTIMATE_SPECS", f"{make} {model} {engine_safe} {year_safe} site:ultimatespecs.com"),
|
||||
("AUTO_DATA", f"{make} {model} {engine_safe} {year_safe} site:auto-data.net"),
|
||||
("COMMON_ISSUES", f"{make} {model} {engine_safe} reliability common problems")
|
||||
]
|
||||
|
||||
tasks = [self.fetch_ddg_targeted(label, q) for label, q in queries]
|
||||
search_results = await asyncio.gather(*tasks)
|
||||
|
||||
# 2. TIER: Fizetős / Kvótás API-k (Példa a DVLA helyére)
|
||||
# Ha a jövőben bejön brit rendszám, itt hívjuk meg a DVLA-t:
|
||||
# if has_uk_plate and self.dvla_quota.can_make_request():
|
||||
# uk_data = await self.fetch_dvla_data(plate)
|
||||
# search_results.append(uk_data)
|
||||
|
||||
# 3. ÖSSZESÍTÉS (Az Akta összeállítása)
|
||||
# Maximalizáljuk a szöveg hosszát, hogy az AI GPU ne fulladjon le!
|
||||
full_context = "\n".join(search_results)
|
||||
if len(full_context) > 2500:
|
||||
full_context = full_context[:2500] + "\n...[TRUNCATED TO SAVE GPU TOKENS]"
|
||||
|
||||
# Regex alapú specifikáció kinyerés
|
||||
extracted_specs = self.extract_specs_from_text(full_context)
|
||||
|
||||
try:
|
||||
if len(full_context.strip()) > 150: # Csökkentettük az elvárást, mert a célzott keresés tömörebb
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == vehicle_id)
|
||||
.values(
|
||||
raw_search_context=full_context,
|
||||
research_metadata=extracted_specs,
|
||||
status='awaiting_ai_synthesis', # Kész az Akta, mehet az Alkimistának!
|
||||
last_research_at=func.now(),
|
||||
attempts=current_attempts + 1
|
||||
)
|
||||
)
|
||||
logger.info(f"✅ Akta rögzítve ({len(full_context)} karakter): {make} {model}")
|
||||
else:
|
||||
new_status = 'suspended_research' if current_attempts + 1 >= self.max_attempts else 'unverified'
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == vehicle_id)
|
||||
.values(
|
||||
status=new_status,
|
||||
attempts=current_attempts + 1,
|
||||
last_research_at=func.now()
|
||||
)
|
||||
)
|
||||
if new_status == 'suspended_research':
|
||||
logger.warning(f"🛑 Felfüggesztve (Nincs nyom a weben): {make} {model}")
|
||||
else:
|
||||
logger.warning(f"⚠️ Kevés adat: {make} {model}, visszatéve a sorba.")
|
||||
|
||||
await db.commit()
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(f"🚨 Adatbázis hiba az eredmény mentésénél ({vehicle_id}): {e}")
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
self_instance = cls()
|
||||
logger.info("🚀 Vehicle Researcher 2.5 ONLINE (Sniper & Quota Manager)")
|
||||
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# ATOMI ZÁROLÁS
|
||||
query = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = 'research_in_progress'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.vehicle_model_definitions
|
||||
WHERE status IN ('unverified', 'awaiting_research', 'ACTIVE')
|
||||
AND attempts < :max_attempts
|
||||
AND is_manual = FALSE
|
||||
ORDER BY
|
||||
CASE WHEN make = 'TOYOTA' THEN 1 ELSE 2 END,
|
||||
attempts ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, make, marketing_name, engine_code, year_from, attempts;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {"max_attempts": self_instance.max_attempts})
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
|
||||
if task:
|
||||
v_id, v_make, v_model, v_engine, v_year, v_attempts = task
|
||||
async with AsyncSessionLocal() as process_db:
|
||||
await self_instance.research_vehicle(process_db, v_id, v_make, v_model, v_engine, v_year, v_attempts)
|
||||
|
||||
await asyncio.sleep(2) # Rate limit védelem a DDG felé
|
||||
else:
|
||||
await asyncio.sleep(30)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(VehicleResearcher.run())
|
||||
except KeyboardInterrupt:
|
||||
logger.info("🛑 Kutató robot leállítva.")
|
||||
@@ -0,0 +1,225 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/workers/vehicle/vehicle_robot_3_alchemist_pro.py
|
||||
import asyncio
|
||||
import logging
|
||||
import datetime
|
||||
import random
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from sqlalchemy import text, func, update, case
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
from app.models.asset import AssetCatalog
|
||||
from app.services.ai_service import AIService
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Vehicle-Alchemist-Pro: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Vehicle-Robot-3-Alchemist-Pro")
|
||||
|
||||
class TechEnricher:
|
||||
"""
|
||||
Vehicle Robot 3: Alchemist Pro (Atomi Zárolás + Kézi Moderáció Patch)
|
||||
Tiszta GPU fókusz: Csak az AI elemzésre és adategyesítésre koncentrál.
|
||||
Nincs felesleges webkeresés. Szigorú, de intelligens Sane-Check.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.max_attempts = 5
|
||||
self.daily_ai_limit = int(os.getenv("AI_DAILY_LIMIT", "10000"))
|
||||
self.ai_calls_today = 0
|
||||
self.last_reset_date = datetime.date.today()
|
||||
|
||||
def check_budget(self) -> bool:
|
||||
if datetime.date.today() > self.last_reset_date:
|
||||
self.ai_calls_today = 0
|
||||
self.last_reset_date = datetime.date.today()
|
||||
return self.ai_calls_today < self.daily_ai_limit
|
||||
|
||||
def validate_merged_data(self, merged_kw: int, merged_ccm: int, v_class: str, fuel: str, current_attempts: int) -> tuple[bool, str]:
|
||||
""" Intelligens validáció a MERGE után. Visszaadja a státuszt és a hiba okát. """
|
||||
if merged_ccm > 18000:
|
||||
return False, f"Irreális CCM érték ({merged_ccm})"
|
||||
if merged_kw > 1500 and v_class != "truck":
|
||||
return False, f"Irreális KW érték ({merged_kw})"
|
||||
|
||||
# Ha hiányzik a KW
|
||||
if merged_kw == 0:
|
||||
if current_attempts < 3:
|
||||
return False, "Hiányzó KW adat. Újrakutatás javasolt."
|
||||
else:
|
||||
logger.warning("Sane-check: Többszöri próbálkozás után sincs KW, de átengedjük részlegesként.")
|
||||
|
||||
# Ha hiányzik a CCM (és belsőégésű)
|
||||
if merged_ccm == 0 and "electric" not in fuel and "elektric" not in fuel and v_class != "trailer":
|
||||
if current_attempts < 3:
|
||||
return False, "Hiányzó CCM (belsőégésű motornál). Újrakutatás javasolt."
|
||||
else:
|
||||
logger.warning("Sane-check: Többszöri próbálkozás után sincs CCM, átengedjük részlegesként.")
|
||||
|
||||
return True, "OK"
|
||||
|
||||
async def process_single_record(self, db, record_id: int, base_info: dict, current_attempts: int):
|
||||
# Pontos azonosító a logokhoz (Márka, Modell, ID, RDW adatok)
|
||||
v_ident = f"{base_info['make'].upper()} {base_info['m_name']} (ID: {record_id}, RDW: {base_info['rdw_ccm']}ccm, KW: {base_info['rdw_kw']})"
|
||||
attempt_str = f"[Próba: {current_attempts + 1}/{self.max_attempts}]"
|
||||
|
||||
ai_data = {} # Üres dict, ha az AI hívás elszállna
|
||||
|
||||
try:
|
||||
logger.info(f"🧠 AI dúsítás indul: {v_ident} {attempt_str}")
|
||||
|
||||
# 1. LÉPÉS: AI Hívás (Rábízzuk az adatokat a modellre)
|
||||
ai_data = await AIService.get_clean_vehicle_data(
|
||||
base_info['make'],
|
||||
base_info['m_name'],
|
||||
base_info
|
||||
)
|
||||
|
||||
if not ai_data:
|
||||
raise ValueError("Teljesen üres AI válasz (API hiba vagy extrém hallucináció).")
|
||||
|
||||
# 2. LÉPÉS: HIBRID MERGE (Még a validáció előtt!)
|
||||
# Az RDW adatok felülbírálják az AI-t a hatósági paramétereknél
|
||||
final_kw = base_info['rdw_kw'] if base_info['rdw_kw'] > 0 else int(ai_data.get("kw", 0) or 0)
|
||||
final_ccm = base_info['rdw_ccm'] if base_info['rdw_ccm'] > 0 else int(ai_data.get("ccm", 0) or 0)
|
||||
|
||||
# Üzemanyag tisztítása
|
||||
fuel_rdw = base_info.get('rdw_fuel', '')
|
||||
final_fuel = fuel_rdw if fuel_rdw and fuel_rdw != "Unknown" else ai_data.get("fuel_type", "petrol")
|
||||
|
||||
final_engine = base_info['rdw_engine'] if base_info['rdw_engine'] else ai_data.get("engine_code", "Unknown")
|
||||
final_euro = base_info['rdw_euro'] or ai_data.get("euro_classification")
|
||||
final_cylinders = base_info['rdw_cylinders'] or ai_data.get("cylinders")
|
||||
|
||||
# 3. LÉPÉS: Intelligens Validáció
|
||||
is_valid, error_msg = self.validate_merged_data(final_kw, final_ccm, base_info['v_type'], final_fuel.lower(), current_attempts)
|
||||
if not is_valid:
|
||||
raise ValueError(f"Validációs hiba: {error_msg}")
|
||||
|
||||
# 4. LÉPÉS: Mentés az Arany Katalógusba
|
||||
clean_model = str(ai_data.get("marketing_name", base_info['m_name']))[:50].upper()
|
||||
|
||||
cat_stmt = text("""
|
||||
INSERT INTO vehicle.vehicle_catalog
|
||||
(master_definition_id, make, model, power_kw, engine_capacity, fuel_type, factory_data)
|
||||
VALUES (:m_id, :make, :model, :kw, :ccm, :fuel, :factory)
|
||||
ON CONFLICT ON CONSTRAINT uix_vehicle_catalog_full DO NOTHING
|
||||
RETURNING id;
|
||||
""")
|
||||
|
||||
await db.execute(cat_stmt, {
|
||||
"m_id": record_id,
|
||||
"make": base_info['make'].upper(),
|
||||
"model": clean_model,
|
||||
"kw": final_kw,
|
||||
"ccm": final_ccm,
|
||||
"fuel": final_fuel,
|
||||
"factory": json.dumps(ai_data)
|
||||
})
|
||||
|
||||
# 5. LÉPÉS: Staging tábla (VMD) lezárása
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == record_id)
|
||||
.values(
|
||||
status="gold_enriched",
|
||||
engine_capacity=final_ccm,
|
||||
power_kw=final_kw,
|
||||
fuel_type=final_fuel,
|
||||
engine_code=final_engine,
|
||||
euro_classification=final_euro,
|
||||
cylinders=final_cylinders,
|
||||
specifications=ai_data, # Elmentjük az AI teljes outputját a mestertáblába is
|
||||
updated_at=func.now()
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
logger.info(f"✨ ARANY REKORD KÉSZ: {v_ident}")
|
||||
self.ai_calls_today += 1
|
||||
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.warning(f"⚠️ Alkimista hiba - {v_ident}: {e}")
|
||||
|
||||
# Ha elértük a limitet, KÉZI MODERÁCIÓRA küldjük, egyébként vissza a Kutatónak
|
||||
new_status = 'manual_review_needed' if current_attempts + 1 >= self.max_attempts else 'unverified'
|
||||
|
||||
# Elmentjük az AI részleges válaszát (vagy a hibát), hogy az admin lássa, mit rontott el a gép
|
||||
review_data = ai_data if ai_data else {"error": "Nincs értékelhető JSON adat az AI-tól", "raw_context": base_info['web_context']}
|
||||
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == record_id)
|
||||
.values(
|
||||
attempts=current_attempts + 1,
|
||||
last_error=str(e)[:200],
|
||||
status=new_status,
|
||||
specifications=review_data, # Kézi ellenőrzéshez beírjuk a törött adatot!
|
||||
updated_at=func.now()
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
if new_status == 'unverified':
|
||||
logger.info(f"♻️ Akta visszaküldve a Robot-2-nek (Kutató). {attempt_str}")
|
||||
else:
|
||||
logger.error(f"🛑 Max próbálkozás elérve! Kézi moderációra küldve: {v_ident}")
|
||||
|
||||
async def run(self):
|
||||
logger.info(f"🚀 Alchemist Pro HIBRID ONLINE (Atomi Zárolás + Moderáció Patch)")
|
||||
while True:
|
||||
if not self.check_budget():
|
||||
logger.warning("💸 Napi AI limit kimerítve! Pihenés...")
|
||||
await asyncio.sleep(3600); continue
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# ATOMI ZÁROLÁS (A "Szent Grál" a race condition ellen)
|
||||
query = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = 'ai_synthesis_in_progress'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.vehicle_model_definitions
|
||||
WHERE status IN ('awaiting_ai_synthesis', 'ACTIVE')
|
||||
AND attempts < :max_attempts
|
||||
AND is_manual = FALSE
|
||||
ORDER BY
|
||||
CASE WHEN status = 'awaiting_ai_synthesis' THEN 1 ELSE 2 END,
|
||||
priority_score DESC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, make, marketing_name, vehicle_class, power_kw, engine_capacity,
|
||||
fuel_type, engine_code, euro_classification, cylinders, raw_search_context, attempts;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {"max_attempts": self.max_attempts})
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
|
||||
if task:
|
||||
# Szétbontjuk a lekérdezett rekordot a base_info dict-be
|
||||
r_id = task[0]
|
||||
base_info = {
|
||||
"make": task[1], "m_name": task[2], "v_type": task[3] or "car",
|
||||
"rdw_kw": task[4] or 0, "rdw_ccm": task[5] or 0,
|
||||
"rdw_fuel": task[6] or "petrol", "rdw_engine": task[7] or "",
|
||||
"rdw_euro": task[8], "rdw_cylinders": task[9],
|
||||
"web_context": task[10] or ""
|
||||
}
|
||||
attempts = task[11]
|
||||
|
||||
# Külön adatbázis kapcsolat a feldolgozáshoz (hosszú AI hívás miatt)
|
||||
async with AsyncSessionLocal() as process_db:
|
||||
await self.process_single_record(process_db, r_id, base_info, attempts)
|
||||
|
||||
# GPU hűtés / Ollama rate limit
|
||||
await asyncio.sleep(random.uniform(1.5, 3.5))
|
||||
else:
|
||||
logger.info("😴 Nincs feldolgozandó akta, az Alkimista pihen...")
|
||||
await asyncio.sleep(15)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(TechEnricher().run())
|
||||
@@ -0,0 +1,168 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import datetime
|
||||
import random
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from sqlalchemy import text, func, update
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle_definitions import VehicleModelDefinition
|
||||
from app.services.ai_service import AIService
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] R3-Alchemist: %(message)s', stream=sys.stdout)
|
||||
logger = logging.getLogger("Robot-3-Alchemist")
|
||||
|
||||
class TechEnricher:
|
||||
"""
|
||||
Vehicle Robot 3: Alchemist Pro (Sentinel Gateway Edition)
|
||||
Az AIService 2.2-t használja (Ollama -> Groq Fallback).
|
||||
Kinyeri a felszereltségi szintet (trim_level) és pótolja a hiányzó adatokat.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.max_attempts = 5
|
||||
self.daily_ai_limit = int(os.getenv("AI_DAILY_LIMIT", "10000"))
|
||||
self.ai_calls_today = 0
|
||||
self.last_reset_date = datetime.date.today()
|
||||
|
||||
def check_budget(self) -> bool:
|
||||
if datetime.date.today() > self.last_reset_date:
|
||||
self.ai_calls_today = 0
|
||||
self.last_reset_date = datetime.date.today()
|
||||
return self.ai_calls_today < self.daily_ai_limit
|
||||
|
||||
def validate_merged_data(self, merged_kw: int, merged_ccm: int, v_class: str, fuel: str, current_attempts: int) -> tuple[bool, str]:
|
||||
if merged_ccm > 18000:
|
||||
return False, f"Irreális CCM érték ({merged_ccm})"
|
||||
if merged_kw > 1500 and v_class not in ["truck", "other"]:
|
||||
return False, f"Irreális KW érték ({merged_kw})"
|
||||
|
||||
if merged_kw == 0 and current_attempts < 3:
|
||||
return False, "Hiányzó KW adat. Újrakutatás javasolt."
|
||||
|
||||
if merged_ccm == 0 and "elektr" not in fuel.lower() and v_class != "trailer" and current_attempts < 3:
|
||||
return False, "Hiányzó CCM (belsőégésű motornál)."
|
||||
|
||||
return True, "OK"
|
||||
|
||||
async def process_single_record(self, db, record_id: int, base_info: dict, current_attempts: int):
|
||||
v_ident = f"{base_info['make'].upper()} {base_info['m_name']} (ID: {record_id})"
|
||||
attempt_str = f"[Próba: {current_attempts + 1}/{self.max_attempts}]"
|
||||
|
||||
try:
|
||||
logger.info(f"🧠 AI dúsítás indul: {v_ident} {attempt_str}")
|
||||
|
||||
# Szigorú Prompt a Master AI Service-nek
|
||||
prompt = f"""
|
||||
Elemezd az alábbi járműadatokat és a webes kutatást! Készíts belőle egy JSON objektumot.
|
||||
Jármű: {base_info['make']} {base_info['m_name']}
|
||||
Hatósági adatok: {base_info['rdw_ccm']} ccm, {base_info['rdw_kw']} kW, Üzemanyag: {base_info['rdw_fuel']}
|
||||
Webes szöveg: {base_info['web_context'][:2000]}
|
||||
|
||||
FELADATOK:
|
||||
1. Keresd meg a felszereltségi szintet (trim_level) a modell nevéből vagy a szövegből (pl. AMG, Highline, Titanium, M-Sport, Elegance, ST-Line). Ha nincs, legyen üres string.
|
||||
2. Ha az RDW adatokban a kW vagy a ccm 0, pótold a szövegből a helyes értéket!
|
||||
|
||||
KIZÁRÓLAG EGY ÉRVÉNYES JSON-T ADJ VISSZA! (A Groq/Gemini miatt kötelező a JSON szó használata).
|
||||
Várt kulcsok: "kw" (int), "ccm" (int), "trim_level" (string), "transmission" (string), "drive_type" (string).
|
||||
"""
|
||||
|
||||
# Hívjuk a te profi Gateway-edet! (_execute_ai_call átveszi a db session-t is a beállításokhoz)
|
||||
ai_data = await AIService._execute_ai_call(db, prompt, model_key="text")
|
||||
|
||||
if not ai_data:
|
||||
raise ValueError("Üres AI válasz (Minden fallback elbukott).")
|
||||
|
||||
# HIBRID MERGE
|
||||
final_kw = base_info['rdw_kw'] if base_info['rdw_kw'] > 0 else int(ai_data.get("kw", 0) or 0)
|
||||
final_ccm = base_info['rdw_ccm'] if base_info['rdw_ccm'] > 0 else int(ai_data.get("ccm", 0) or 0)
|
||||
trim_level = str(ai_data.get("trim_level", ""))[:100]
|
||||
|
||||
# Sane-Check
|
||||
is_valid, error_msg = self.validate_merged_data(final_kw, final_ccm, base_info['v_type'], base_info['rdw_fuel'], current_attempts)
|
||||
if not is_valid:
|
||||
raise ValueError(f"Validációs hiba: {error_msg}")
|
||||
|
||||
# Staging tábla frissítése (Arany minősítés)
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == record_id)
|
||||
.values(
|
||||
status="gold_enriched",
|
||||
engine_capacity=final_ccm,
|
||||
power_kw=final_kw,
|
||||
trim_level=trim_level if trim_level.lower() not in ["null", "none"] else "",
|
||||
specifications=ai_data,
|
||||
updated_at=func.now()
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
logger.info(f"✨ ARANY REKORD KÉSZ: {v_ident} | Trim: {trim_level}")
|
||||
self.ai_calls_today += 1
|
||||
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.warning(f"⚠️ Alkimista hiba - {v_ident}: {e}")
|
||||
|
||||
new_status = 'manual_review_needed' if current_attempts + 1 >= self.max_attempts else 'unverified'
|
||||
|
||||
await db.execute(
|
||||
update(VehicleModelDefinition)
|
||||
.where(VehicleModelDefinition.id == record_id)
|
||||
.values(
|
||||
attempts=current_attempts + 1,
|
||||
last_error=str(e)[:200],
|
||||
status=new_status,
|
||||
updated_at=func.now()
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
if new_status == 'unverified':
|
||||
logger.info(f"♻️ Akta visszaküldve a Kutatónak (R2). {attempt_str}")
|
||||
|
||||
async def run(self):
|
||||
logger.info(f"🚀 R3 Alchemist Pro ONLINE (Sentinel Gateway Integráció)")
|
||||
while True:
|
||||
if not self.check_budget():
|
||||
logger.warning("💸 Napi AI limit kimerítve! Pihenés...")
|
||||
await asyncio.sleep(3600); continue
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
query = text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET status = 'ai_synthesis_in_progress'
|
||||
WHERE id = (
|
||||
SELECT id FROM vehicle.vehicle_model_definitions
|
||||
WHERE status = 'awaiting_ai_synthesis'
|
||||
AND attempts < :max_attempts
|
||||
AND is_manual = FALSE
|
||||
ORDER BY priority_score DESC
|
||||
FOR UPDATE SKIP LOCKED LIMIT 1
|
||||
)
|
||||
RETURNING id, make, marketing_name, vehicle_class, power_kw, engine_capacity, fuel_type, raw_search_context, attempts;
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {"max_attempts": self.max_attempts})
|
||||
task = result.fetchone()
|
||||
await db.commit()
|
||||
|
||||
if task:
|
||||
base_info = {
|
||||
"make": task[1], "m_name": task[2], "v_type": task[3] or "car",
|
||||
"rdw_kw": task[4] or 0, "rdw_ccm": task[5] or 0,
|
||||
"rdw_fuel": task[6] or "petrol", "web_context": task[7] or ""
|
||||
}
|
||||
async with AsyncSessionLocal() as process_db:
|
||||
await self.process_single_record(process_db, task[0], base_info, task[8])
|
||||
|
||||
else:
|
||||
await asyncio.sleep(10)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💀 Kritikus hiba a főciklusban: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(TechEnricher().run())
|
||||
@@ -0,0 +1,113 @@
|
||||
import asyncio
|
||||
import json
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def test_scraper():
|
||||
# Két probléma-fókuszú URL: a modern Aprilia és a régi, hibás HTML-ű BMW
|
||||
test_urls = [
|
||||
"https://www.autoevolution.com/moto/aprilia-rs-660-factory-2025.html",
|
||||
"https://www.autoevolution.com/moto/bmw-f-650-gs-2011.html"
|
||||
]
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
context = await browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
for url in test_urls:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"🌍 MEGNYITÁS: {url}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# A DOM betöltése megvárása
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
||||
await asyncio.sleep(2) # Várunk picit a JS futásra
|
||||
|
||||
# A TÖKÉLETESÍTETT AUTOEVOLUTION PARSZOLÓ
|
||||
script = """
|
||||
() => {
|
||||
let results = {};
|
||||
|
||||
// 1. MÓDSZER: Régi motorok (pl. BMW F650GS) -> td.left és td.right
|
||||
let leftCells = document.querySelectorAll('td.left');
|
||||
leftCells.forEach(cell => {
|
||||
let key = cell.innerText.replace(/:$/, '').trim();
|
||||
let rightCell = cell.nextElementSibling;
|
||||
if(rightCell && rightCell.classList.contains('right')) {
|
||||
results[key] = rightCell.innerText.trim();
|
||||
}
|
||||
});
|
||||
|
||||
// 2. MÓDSZER: Modern motorok (pl. Aprilia) -> dt és dd
|
||||
let dts = document.querySelectorAll('dt');
|
||||
dts.forEach(dt => {
|
||||
let key = dt.innerText.replace(/:$/, '').trim();
|
||||
let dd = dt.nextElementSibling;
|
||||
if(dd && dd.tagName.toLowerCase() === 'dd') {
|
||||
results[key] = dd.innerText.trim();
|
||||
}
|
||||
});
|
||||
|
||||
// 3. MÓDSZER: Alternatív modern layout -> span.label és span.value
|
||||
let specRows = document.querySelectorAll('.spec-row');
|
||||
specRows.forEach(row => {
|
||||
let label = row.querySelector('.label');
|
||||
let value = row.querySelector('.value');
|
||||
if(label && value) {
|
||||
let key = label.innerText.replace(/:$/, '').trim();
|
||||
if (!results[key]) {
|
||||
results[key] = value.innerText.trim();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 4. MÓDSZER: "Adler" típusú elavult leírások fallbackje -> Vastagított szöveg
|
||||
if (Object.keys(results).length === 0) {
|
||||
document.querySelectorAll('b, strong').forEach(b => {
|
||||
let key = b.innerText.replace(/:$/, '').trim();
|
||||
if(key.length > 2 && key.length < 30) {
|
||||
let val = "";
|
||||
// Ha a szöveg közvetlenül a tag után van (Text Node)
|
||||
if(b.nextSibling && b.nextSibling.nodeType === 3) {
|
||||
val = b.nextSibling.textContent.trim();
|
||||
}
|
||||
// Ha egy másik elemben van
|
||||
else if (b.nextElementSibling && b.nextElementSibling.tagName !== 'B') {
|
||||
val = b.nextElementSibling.innerText.trim();
|
||||
}
|
||||
if(val && !results[key]) {
|
||||
results[key] = val;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
"""
|
||||
|
||||
data = await page.evaluate(script)
|
||||
|
||||
if data and len(data) > 0:
|
||||
# Kiszűrjük a zajt, csak a releváns műszaki adatokat hagyjuk meg
|
||||
relevant_keys = ["Type", "Displacement", "Bore X Stroke", "Compression Ratio",
|
||||
"Horsepower", "Torque", "Fuel System", "Gearbox", "Clutch",
|
||||
"Final Drive", "Frame", "Front Suspension", "Rear Suspension",
|
||||
"Front Brake", "Rear Brake", "Overall Length", "Overall Width",
|
||||
"Seat Height", "Wheelbase", "Fuel Capacity", "Weight", "Dry Weight",
|
||||
"Wet Weight", "Front", "Rear"]
|
||||
|
||||
filtered_data = {k: v for k, v in data.items() if any(rk.lower() in k.lower() for rk in relevant_keys)}
|
||||
|
||||
print("\n🟢 KINYERT ADATOK (DOM PARSZOLÓ):")
|
||||
print(json.dumps(filtered_data if filtered_data else data, indent=2, ensure_ascii=False))
|
||||
print(f"\n✅ Összesen {len(filtered_data if filtered_data else data)} műszaki paramétert találtam.")
|
||||
else:
|
||||
print("\n🔴 NULLA ADAT - A DOM parszoló nem talált egyezést.")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_scraper())
|
||||
113
archive/old_files/backend/app/workers/vehicle/r5_test.py.old
Normal file
113
archive/old_files/backend/app/workers/vehicle/r5_test.py.old
Normal file
@@ -0,0 +1,113 @@
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import requests
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
# --- TECHNIKAI SZÓTÁR ÉS MAPPING ---
|
||||
# Ez a szótár fordítja le az UltimateSpecs kulcsokat az adatbázis oszlopneveire
|
||||
MAPPING = {
|
||||
"Maximum power": "power_kw",
|
||||
"Engine capacity": "engine_capacity",
|
||||
"Maximum torque": "torque_nm",
|
||||
"Top Speed": "max_speed",
|
||||
"Acceleration 0 to 100 km/h": "acceleration_0_100",
|
||||
"Curb Weight": "curb_weight",
|
||||
"Wheelbase": "wheelbase",
|
||||
"Num. of Seats": "seats",
|
||||
"Drive wheels - Traction - Layout": "drive_type",
|
||||
"Body": "body_type"
|
||||
}
|
||||
|
||||
async def r5_test_run():
|
||||
print("🚀 R5 Hibrid Robot indítása (Teszt üzemmód)...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 1. KIVÁLASZTÁS: Kiveszünk egy olyan autót, ami még nincs dúsítva (R1 bázisból)
|
||||
query = text("""
|
||||
SELECT id, make, marketing_name, year_from, technical_code, fuel_type
|
||||
FROM vehicle.vehicle_model_definitions
|
||||
WHERE (power_kw IS NULL OR power_kw = 0 OR engine_capacity IS NULL OR engine_capacity = 0)
|
||||
AND status IN ('manual_review_needed', 'research_failed_empty', 'pending', 'enrich_ready')
|
||||
ORDER BY priority_score DESC
|
||||
LIMIT 1
|
||||
""")
|
||||
target = (await db.execute(query)).fetchone()
|
||||
|
||||
if not target:
|
||||
print("✨ Nincs feldolgozatlan autó az adatbázisban.")
|
||||
return
|
||||
|
||||
t_id, make, model, year, tech_code, fuel = target
|
||||
print(f"🎯 Célpont: {make} {model} ({year})")
|
||||
print(f"📌 Technical Code: {tech_code or 'Nincs megadva'}")
|
||||
|
||||
# 2. RDW ADATOK (Holland hatósági bázis)
|
||||
# Ha van technical_code (pl. Fiatnál a típusazonosító), az RDW-ből pontos adatot kapunk
|
||||
rdw_data = {}
|
||||
if tech_code:
|
||||
print("🇳🇱 RDW adatok lekérése...")
|
||||
# Az RDW API m9d7-ebf2 táblája tartalmazza a típus specifikációkat
|
||||
rdw_url = f"https://opendata.rdw.nl/resource/m9d7-ebf2.json?handelsbenaming={tech_code.upper()}"
|
||||
try:
|
||||
res = requests.get(rdw_url, timeout=5).json()
|
||||
if res:
|
||||
rdw_data = {
|
||||
"power_kw": int(float(res[0].get('nettomaximumvermogen', 0))),
|
||||
"engine_capacity": int(res[0].get('cilinderinhoud', 0)),
|
||||
"curb_weight": int(res[0].get('massa_ledig_voertuig', 0))
|
||||
}
|
||||
print("✅ RDW adatok sikeresen betöltve.")
|
||||
except:
|
||||
print("⚠️ RDW nem elérhető vagy nincs találat.")
|
||||
|
||||
# 3. ULTIMATESPECS ADATOK (Szimulált kaparás a kért logika alapján)
|
||||
print("🏁 UltimateSpecs adatok gyűjtése...")
|
||||
# Itt futna a Playwright scraper, ami kinyeri a táblázatot
|
||||
# Példa nyers adatokra, amit az oldalról szedünk le:
|
||||
raw_web_data = {
|
||||
"Maximum power": "103 PS / 76 kW @ 5750 rpm",
|
||||
"Engine capacity": "1581 cm3",
|
||||
"Maximum torque": "144 Nm @ 4000 rpm",
|
||||
"Top Speed": "180 km/h",
|
||||
"Acceleration 0 to 100 km/h": "11.5 s",
|
||||
"Curb Weight": "1090 kg",
|
||||
"Wheelbase": "254 cm",
|
||||
"Body": "Hatchback"
|
||||
}
|
||||
|
||||
# 4. ÖSSZEFŰZÉS ÉS FORDÍTÁS
|
||||
final_mdm_record = {
|
||||
"id": t_id,
|
||||
"make": make,
|
||||
"marketing_name": model,
|
||||
"year_from": year,
|
||||
"fuel_type": fuel
|
||||
}
|
||||
|
||||
# Alkalmazzuk a mappinget és a regex tisztítást
|
||||
for web_key, db_key in MAPPING.items():
|
||||
val = raw_web_data.get(web_key)
|
||||
if val:
|
||||
# Számértékek kinyerése (pl. "76 kW" -> 76, "1581 cm3" -> 1581)
|
||||
numbers = re.findall(r'\d+', str(val))
|
||||
if numbers:
|
||||
# Ha több szám van (pl. kW és LE), a relevánsat választjuk
|
||||
final_mdm_record[db_key] = numbers[1] if "kW" in str(val) and len(numbers)>1 else numbers[0]
|
||||
else:
|
||||
final_mdm_record[db_key] = val
|
||||
|
||||
# RDW adatok prioritása (ezek a legpontosabbak, felülírják a webet)
|
||||
final_mdm_record.update({k: v for k, v in rdw_data.items() if v})
|
||||
|
||||
# --- TERMINÁL KIMENET ---
|
||||
print("\n" + "="*50)
|
||||
print("📊 VÉGLEGES MDM REKORD (ELŐNÉZET)")
|
||||
print("="*50)
|
||||
print(json.dumps(final_mdm_record, indent=2, ensure_ascii=False))
|
||||
print("="*50)
|
||||
print("\n[R5] Ha az adatok rendben vannak, mehet az élesítés?")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(r5_test_run())
|
||||
@@ -0,0 +1,62 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/workers/vehicle/vehicle_robot_1_5_heavy_eu1.0.py
|
||||
import asyncio
|
||||
import httpx
|
||||
import logging
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
logger = logging.getLogger("Robot-1-5-Heavy-EU")
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
class HeavyEUHunter:
|
||||
RDW_URL = "https://opendata.rdw.nl/resource/m9d7-ebf2.json"
|
||||
|
||||
@classmethod
|
||||
async def fetch_rdw_heavy(cls, vehicle_type: str):
|
||||
query_url = f"{cls.RDW_URL}?voertuigsoort={vehicle_type}&$select=merk,handelsbenaming&$limit=10000"
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
try:
|
||||
resp = await client.get(query_url)
|
||||
return resp.json() if resp.status_code == 200 else []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ RDW Error: {e}")
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
logger.info("🚛 Robot 1.5 (EU Heavy Duty) indítása - Kötegelt mód...")
|
||||
job_list = {
|
||||
"Vrachtwagen": "truck",
|
||||
"Bus": "bus",
|
||||
"Kampeerauto": "rv"
|
||||
}
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
for rdw_name, internal_class in job_list.items():
|
||||
logger.info(f"📥 {rdw_name} adatok letöltése...")
|
||||
data = await cls.fetch_rdw_heavy(rdw_name)
|
||||
|
||||
if not data: continue
|
||||
|
||||
# A 10.000 adatot egyetlen listába gyűjtjük
|
||||
insert_data = []
|
||||
for item in data:
|
||||
make = item.get('merk', '').upper().strip()
|
||||
model = item.get('handelsbenaming', '').upper().strip()
|
||||
if make and model:
|
||||
insert_data.append({"make": make, "model": model, "v_class": internal_class})
|
||||
|
||||
if insert_data:
|
||||
query = text("""
|
||||
INSERT INTO vehicle.catalog_discovery
|
||||
(make, model, vehicle_class, status, market, priority_score, source)
|
||||
VALUES (:make, :model, :v_class, 'pending', 'EU', 20, 'RDW-HEAVY')
|
||||
ON CONFLICT ON CONSTRAINT _make_model_market_year_uc DO NOTHING
|
||||
""")
|
||||
# Egyetlen SQL hívással beszúrjuk akár a 10.000 sort is!
|
||||
await db.execute(query, insert_data)
|
||||
await db.commit()
|
||||
logger.info(f"✅ {rdw_name}: {len(insert_data)} EU-s nagygép beküldve kötegelve.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(HeavyEUHunter.run())
|
||||
@@ -0,0 +1,387 @@
|
||||
#!/usr/bin/env python3
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import urllib.parse
|
||||
import sys
|
||||
import signal
|
||||
import re
|
||||
from playwright.async_api import async_playwright
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
# R2.3 - SENTINEL (Hardened & Obedient Edition)
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [R2.3-SENTINEL] %(message)s')
|
||||
logger = logging.getLogger("R2.3")
|
||||
|
||||
# --- 1. SZŰRÉSEK ÉS TILTÓLISTÁK ---
|
||||
# Csak olyan típusokat keresünk, amik nem utánfutók vagy munkagépek
|
||||
JUNK_LIST = [
|
||||
'SARIS', 'ANSSEMS', 'HAPERT', 'HUMBAUR', 'EDUARD', 'IFOR WILLIAMS', 'FENDT',
|
||||
'HOBBY', 'ADRIA', 'PEECON', 'JAKO', 'KAWECO', 'POTTINGER', 'BOCKMANN',
|
||||
'JOHN DEERE', 'CLAAS', 'IVECO', 'SCANIA', 'MAN', 'DAF', 'KNAUS', 'PÖSSL', 'HYMER', 'WESTFALIA'
|
||||
]
|
||||
|
||||
# --- 2. FORDÍTÁSOK (DE/NL -> EN) ---
|
||||
TRANSLATIONS = {
|
||||
"3ER REIHE": "3 Series", "5ER REIHE": "5 Series", "1ER REIHE": "1 Series", "7ER REIHE": "7 Series",
|
||||
"E-KLASSE": "E Class", "C-KLASSE": "C Class", "S-KLASSE": "S Class", "A-KLASSE": "A Class",
|
||||
"REIHE": "Series", "KLASSE": "Class", "BESTELWAGEN": "Van"
|
||||
}
|
||||
|
||||
class RobotScout:
|
||||
def __init__(self):
|
||||
self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
self.running = True
|
||||
|
||||
def clean_name(self, make, model):
|
||||
"""Standardizált angol név előállítása."""
|
||||
m = model.upper()
|
||||
for de, en in TRANSLATIONS.items():
|
||||
m = m.replace(de, en)
|
||||
# Márkanév duplázódás törlése (pl. VOLVO VOLVO V60 -> VOLVO V60)
|
||||
m = m.replace(make.upper(), "").strip()
|
||||
return f"{make} {m}"
|
||||
|
||||
# --- COLUMN MAPPING for scraping ---
|
||||
COLUMN_MAPPING = {
|
||||
"horsepower": "power_kw",
|
||||
"engine displacement": "engine_capacity",
|
||||
"maximum torque": "torque_nm",
|
||||
"top speed": "max_speed",
|
||||
"curb weight": "curb_weight",
|
||||
"wheelbase": "wheelbase",
|
||||
"num. of seats": "seats"
|
||||
}
|
||||
|
||||
def clean_number(self, val: str, key: str = "") -> int:
|
||||
if not val or val == "-": return 0
|
||||
try:
|
||||
if "hp" in val.lower() or "kw" in val.lower():
|
||||
kw_match = re.search(r'(\d+)\s*kw', val.lower())
|
||||
if kw_match: return int(kw_match.group(1))
|
||||
nums = re.findall(r'\d+', val.replace(' ', '').replace(',', '').replace('.', ''))
|
||||
return int(nums[0]) if nums else 0
|
||||
except: return 0
|
||||
|
||||
async def get_car_links(self, page, make, model, year, use_year=True):
|
||||
"""Minden autós link kigyűjtése fallback mechanizmussal retry logikával."""
|
||||
clean_model = self.clean_name(make, model)
|
||||
search_query = f"{clean_model} {year}" if use_year else clean_model
|
||||
url = f"https://www.ultimatespecs.com/index.php?q={urllib.parse.quote(search_query)}"
|
||||
|
||||
logger.info(f"🔎 KERESÉS: {search_query}")
|
||||
|
||||
async def _fetch_links():
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=25000)
|
||||
|
||||
# 1. Ha direkt az adatlapon vagyunk
|
||||
if any(x in page.url for x in ['/car-specs/', '/motorcycles-specs/']):
|
||||
logger.info("🎯 Direkt találat!")
|
||||
return [{"name": await page.title(), "url": page.url}]
|
||||
|
||||
# 2. Várakozás és linkek kigyűjtése
|
||||
await asyncio.sleep(2)
|
||||
variants = await page.evaluate("""
|
||||
() => {
|
||||
let results = [];
|
||||
document.querySelectorAll('a').forEach(a => {
|
||||
let href = a.getAttribute('href') || '';
|
||||
let text = a.innerText.trim();
|
||||
// Csak technikai adatlapokat gyűjtünk, reklámokat/kategóriákat nem
|
||||
if ((href.includes('/car-specs/') || href.includes('/motorcycles-specs/'))
|
||||
&& href.includes('.html') && text.length > 3) {
|
||||
results.push({ name: text, url: href });
|
||||
}
|
||||
});
|
||||
return results;
|
||||
}
|
||||
""")
|
||||
|
||||
# 3. Fallback: Ha nincs találat évvel, próbálja év nélkül
|
||||
if not variants and use_year:
|
||||
logger.info(" ↳ Nincs találat évszámmal, próbálkozom évszám nélkül...")
|
||||
return await self.get_car_links(page, make, model, year, use_year=False)
|
||||
|
||||
return variants
|
||||
|
||||
try:
|
||||
variants = await self._retry_with_backoff(
|
||||
_fetch_links,
|
||||
max_attempts=3,
|
||||
base_delay=2,
|
||||
exception_message=f"❌ Hálózati hiba a(z) {url} oldalon"
|
||||
)
|
||||
return variants if variants is not None else []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Hálózati hiba (végleges): {str(e)[:50]}")
|
||||
return []
|
||||
|
||||
async def _retry_with_backoff(self, func, max_attempts=3, base_delay=2,
|
||||
exception_message="Retry failed", retry_exceptions=True):
|
||||
"""Helper function for retry logic with exponential backoff."""
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
return await func()
|
||||
except Exception as e:
|
||||
if attempt == max_attempts - 1:
|
||||
logger.error(f"{exception_message} after {max_attempts} attempts: {str(e)[:100]}")
|
||||
raise
|
||||
else:
|
||||
delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
|
||||
logger.warning(f"⚠️ Attempt {attempt + 1} failed: {str(e)[:50]}. Retrying in {delay:.1f}s...")
|
||||
await asyncio.sleep(delay)
|
||||
return None
|
||||
|
||||
async def scrape_car_details(self, page, url):
|
||||
"""Scrape car specifications from a given Ultimate Specs URL with comprehensive data extraction and retry logic."""
|
||||
async def _scrape():
|
||||
await page.goto(url, wait_until="networkidle", timeout=30000)
|
||||
|
||||
# Parsing all specification tables and sections
|
||||
full_specs = await page.evaluate("""
|
||||
() => {
|
||||
let results = {};
|
||||
|
||||
// 1. Collect all specification tables (existing logic)
|
||||
document.querySelectorAll('table.table_specs, table.responsive').forEach(table => {
|
||||
table.querySelectorAll('tr').forEach(row => {
|
||||
let t = row.querySelector('.table_specs_title, .td_title, td:first-child');
|
||||
let v = row.querySelector('.table_specs_value, .td_value, td:last-child');
|
||||
if(t && v) {
|
||||
let k = t.innerText.replace(':','').trim().toLowerCase();
|
||||
let val = v.innerText.trim();
|
||||
if(k && val && val !== "-") results[k] = val;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// 2. Collect section headers and their content for additional technical data
|
||||
// Look for h2, h3, h4 elements that might contain section titles
|
||||
const sections = {};
|
||||
const headers = document.querySelectorAll('h2, h3, h4, .section-title, .specs-header');
|
||||
|
||||
headers.forEach(header => {
|
||||
const title = header.innerText.trim();
|
||||
if (title && title.length > 0) {
|
||||
// Find the next table or div with specs after this header
|
||||
let nextElement = header.nextElementSibling;
|
||||
let sectionData = {};
|
||||
|
||||
// Look for tables or lists in the next few siblings
|
||||
for (let i = 0; i < 5 && nextElement; i++) {
|
||||
if (nextElement.tagName === 'TABLE') {
|
||||
nextElement.querySelectorAll('tr').forEach(row => {
|
||||
let t = row.querySelector('td:first-child');
|
||||
let v = row.querySelector('td:last-child');
|
||||
if(t && v) {
|
||||
let k = t.innerText.replace(':','').trim().toLowerCase();
|
||||
let val = v.innerText.trim();
|
||||
if(k && val && val !== "-") {
|
||||
sectionData[k] = val;
|
||||
// Also add to main results with section prefix
|
||||
results[`${title.toLowerCase().replace(/ /g, '_')}_${k}`] = val;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
nextElement = nextElement.nextElementSibling;
|
||||
}
|
||||
|
||||
sections[title.toLowerCase().replace(/ /g, '_')] = sectionData;
|
||||
}
|
||||
});
|
||||
|
||||
// 3. Extract specific known sections by looking for text patterns
|
||||
const pageText = document.body.innerText.toLowerCase();
|
||||
|
||||
// Check for electric/hybrid sections
|
||||
if (pageText.includes('electric engine') || pageText.includes('battery')) {
|
||||
// Try to find battery voltage, capacity, etc.
|
||||
const batteryRegex = /battery\s*voltage[:\s]*([\d\.]+)\s*v/gi;
|
||||
const match = batteryRegex.exec(document.body.innerText);
|
||||
if (match) results['battery_voltage_v'] = match[1];
|
||||
}
|
||||
|
||||
// 4. Extract dimensions data
|
||||
const dimensionPatterns = {
|
||||
'wheelbase': /wheelbase[:\s]*([\d\.]+)\s*cm/gi,
|
||||
'length': /length[:\s]*([\d\.]+)\s*cm/gi,
|
||||
'width': /width[:\s]*([\d\.]+)\s*cm/gi,
|
||||
'height': /height[:\s]*([\d\.]+)\s*cm/gi,
|
||||
'curb_weight': /curb\s*weight[:\s]*([\d\.]+)\s*kg/gi,
|
||||
'towing_capacity': /towing\s*capacity[:\s]*([\d\.]+)\s*kg/gi
|
||||
};
|
||||
|
||||
for (const [key, regex] of Object.entries(dimensionPatterns)) {
|
||||
const match = regex.exec(document.body.innerText);
|
||||
if (match) results[key] = match[1];
|
||||
}
|
||||
|
||||
// 5. Add sections data as a nested object
|
||||
results['_sections'] = sections;
|
||||
|
||||
return results;
|
||||
}
|
||||
""")
|
||||
return full_specs
|
||||
|
||||
try:
|
||||
logger.info(f"🌐 Scraping: {url}")
|
||||
full_specs = await self._retry_with_backoff(
|
||||
_scrape,
|
||||
max_attempts=3,
|
||||
base_delay=2,
|
||||
exception_message=f"❌ Scrape hiba a(z) {url} oldalon"
|
||||
)
|
||||
return full_specs
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Scrape hiba (végleges): {str(e)[:100]}...")
|
||||
return None
|
||||
|
||||
async def run(self):
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
context = await browser.new_context(user_agent=self.user_agent)
|
||||
page = await context.new_page()
|
||||
|
||||
while self.running:
|
||||
# --- A FÉK: 3-6 mp szigorú pihenő minden kör elején ---
|
||||
wait = random.uniform(3, 6)
|
||||
logger.info(f"💤 Várakozás {wait:.1f} mp...")
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Következő feldolgozatlan autó (John Deere, Iveco, stb. kizárva)
|
||||
target = (await db.execute(text("""
|
||||
SELECT id, make, marketing_name, year_from FROM vehicle.vehicle_model_definitions
|
||||
WHERE status IN ('pending', 'manual_review_needed')
|
||||
AND NOT (make = ANY(:junks))
|
||||
ORDER BY priority_score DESC LIMIT 1
|
||||
"""), {"junks": JUNK_LIST})).fetchone()
|
||||
|
||||
if not target:
|
||||
logger.info("✨ Minden tétel feldolgozva.")
|
||||
break
|
||||
|
||||
t_id, make, model, year = target
|
||||
logger.info(f"🚀 CÉLPONT: {make} {model} ({year}) [ID: {t_id}]")
|
||||
|
||||
try:
|
||||
links = await self.get_car_links(page, make, model, year)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Hálózati hiba linkek lekérésekor: {str(e)[:100]}")
|
||||
await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_network' WHERE id=:id"), {"id": t_id})
|
||||
await db.commit()
|
||||
continue
|
||||
|
||||
if not links:
|
||||
logger.warning(f"❌ Nem található adatlap. research_failed_empty rögzítése.")
|
||||
await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_empty' WHERE id=:id"), {"id": t_id})
|
||||
await db.commit()
|
||||
continue
|
||||
|
||||
# --- 1. SCRAPE THE FIRST LINK FOR IMMEDIATE ENRICHMENT ---
|
||||
first_link = None
|
||||
if links:
|
||||
first_link = links[0]
|
||||
full_url = first_link['url'] if first_link['url'].startswith('http') else f"https://www.ultimatespecs.com{first_link['url']}"
|
||||
logger.info(f"⚡ Azonnali adatgyűjtés: {full_url}")
|
||||
web_data = await self.scrape_car_details(page, full_url)
|
||||
|
||||
if web_data is None:
|
||||
# Scraping failed after all retries
|
||||
logger.error(f"❌ Scraping sikertelen minden próbálkozás után. research_failed_parsing rögzítése.")
|
||||
await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='research_failed_parsing' WHERE id=:id"), {"id": t_id})
|
||||
await db.commit()
|
||||
# Continue to save links as variants anyway
|
||||
web_data = {}
|
||||
elif len(web_data) >= 5:
|
||||
# Map scraped data to columns
|
||||
updates = {col: self.clean_number(web_data.get(k)) for k, col in self.COLUMN_MAPPING.items()}
|
||||
# Also extract fuel_type, transmission, etc. if possible
|
||||
fuel_type = web_data.get('fuel type', 'Unknown')
|
||||
transmission_type = web_data.get('transmission', 'Unknown')
|
||||
drive_type = web_data.get('drive type', 'Unknown')
|
||||
body_type = web_data.get('body type', 'Unknown')
|
||||
engine_capacity = updates.get('engine_capacity', 0)
|
||||
power_kw = updates.get('power_kw', 0)
|
||||
|
||||
# Update the original record with scraped data
|
||||
await db.execute(text("""
|
||||
UPDATE vehicle.vehicle_model_definitions
|
||||
SET power_kw = :power_kw, engine_capacity = :engine_capacity,
|
||||
torque_nm = :torque_nm, max_speed = :max_speed,
|
||||
curb_weight = :curb_weight,
|
||||
wheelbase = :wheelbase, seats = :seats,
|
||||
fuel_type = :fuel_type, transmission_type = :transmission_type,
|
||||
drive_type = :drive_type, body_type = :body_type,
|
||||
specifications = specifications || :full_json,
|
||||
status = 'awaiting_ai_synthesis', updated_at = NOW()
|
||||
WHERE id = :id
|
||||
"""), {
|
||||
**updates,
|
||||
"id": t_id,
|
||||
"fuel_type": fuel_type,
|
||||
"transmission_type": transmission_type,
|
||||
"drive_type": drive_type,
|
||||
"body_type": body_type,
|
||||
"full_json": json.dumps(web_data)
|
||||
})
|
||||
logger.info(f"✅ AZONNALI PUBLIKÁLÁS: {make} {model} ({power_kw} kW)")
|
||||
else:
|
||||
logger.warning("⚠️ Scraping kevés adatot talált, csak linkek mentve.")
|
||||
|
||||
# --- 2. SAVE ALL LINKS AS NEW VARIANT RECORDS (including first if not enriched) ---
|
||||
added = 0
|
||||
for l in links:
|
||||
full_url = l['url'] if l['url'].startswith('http') else f"https://www.ultimatespecs.com{l['url']}"
|
||||
|
||||
# JAVÍTÁS: column "source_url" hiba ellen raw_api_data-t nézünk
|
||||
check_query = text("SELECT id FROM vehicle.vehicle_model_definitions WHERE raw_api_data->>'url' = :u")
|
||||
exists = (await db.execute(check_query, {"u": full_url})).fetchone()
|
||||
|
||||
if not exists:
|
||||
# Create normalized name from marketing name
|
||||
normalized = l['name'].lower().replace(' ', '_').replace('-', '_').replace('.', '').replace(',', '')[:200]
|
||||
|
||||
await db.execute(text("""
|
||||
INSERT INTO vehicle.vehicle_model_definitions
|
||||
(make, marketing_name, normalized_name, year_from, status,
|
||||
raw_api_data, priority_score, source, market,
|
||||
technical_code, variant_code, version_code,
|
||||
specifications, marketing_name_aliases, raw_search_context)
|
||||
VALUES (:make, :name, :normalized, :year, 'awaiting_ai_synthesis',
|
||||
:raw, 30, 'ultimatespecs', 'EU',
|
||||
'UNKNOWN', 'UNKNOWN', 'UNKNOWN',
|
||||
'{}'::jsonb, '[]'::jsonb, '')
|
||||
"""), {
|
||||
"make": make, "name": l['name'], "normalized": normalized,
|
||||
"year": year, "raw": json.dumps({"url": full_url}), "priority": 30
|
||||
})
|
||||
added += 1
|
||||
|
||||
# Eredeti rekord archiválása (ha még nem publikáltuk)
|
||||
if not web_data:
|
||||
await db.execute(text("UPDATE vehicle.vehicle_model_definitions SET status='expanded_to_variants', updated_at=NOW() WHERE id=:id"), {"id": t_id})
|
||||
|
||||
await db.commit()
|
||||
logger.info(f"✅ SIKER: {added} új variáció mentve. R4-R5 robotok értesítve.")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
scout = RobotScout()
|
||||
# Handle CTRL+C
|
||||
def stop_signal(sig, frame):
|
||||
logger.info("🛑 LEÁLLÍTÁS (Kérés érzékelve)...")
|
||||
scout.running = False
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, stop_signal)
|
||||
|
||||
try:
|
||||
asyncio.run(scout.run())
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
111
archive/old_files/backend/archive_v1_scripts/discovery_bot.py.old
Executable file
111
archive/old_files/backend/archive_v1_scripts/discovery_bot.py.old
Executable file
@@ -0,0 +1,111 @@
|
||||
# /opt/docker/dev/service_finder/backend/discovery_bot.py
|
||||
import asyncio
|
||||
import json
|
||||
import httpx
|
||||
import os
|
||||
import hashlib
|
||||
import logging
|
||||
from urllib.parse import quote
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.staged_data import ServiceStaging
|
||||
|
||||
# Logolás beállítása
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s]: %(message)s')
|
||||
logger = logging.getLogger("OSM-Discovery")
|
||||
|
||||
# Konfiguráció
|
||||
HUNGARY_BBOX = "45.7,16.1,48.6,22.9"
|
||||
OVERPASS_URL = "http://overpass-api.de/api/interpreter?data="
|
||||
|
||||
class OSMDiscoveryBot:
|
||||
@staticmethod
|
||||
def generate_fingerprint(name: str, city: str) -> str:
|
||||
"""
|
||||
Ujjlenyomat generálása a deduplikációhoz.
|
||||
Kicsit lazább, mint a Hunter-nél, mert az OSM címadatok néha hiányosak.
|
||||
"""
|
||||
raw = f"{str(name).lower()}|{str(city).lower()}"
|
||||
return hashlib.md5(raw.encode()).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def get_service_type(tags: dict, name: str) -> str:
|
||||
""" OSM tagek leképezése belső kategóriákra. """
|
||||
name = name.lower()
|
||||
shop = tags.get('shop', '')
|
||||
amenity = tags.get('amenity', '')
|
||||
|
||||
if shop == 'tyres' or 'gumi' in name: return 'tire_shop'
|
||||
if amenity == 'car_wash' or 'mosó' in name: return 'car_wash'
|
||||
if any(x in name for x in ['villamos', 'autóvill', 'elektro']): return 'electrician'
|
||||
if any(x in name for x in ['fényez', 'lakatos', 'karosszéria']): return 'body_shop'
|
||||
return 'mechanic'
|
||||
|
||||
async def fetch_osm_data(self, query_part: str):
|
||||
""" Aszinkron adatgyűjtés az Overpass API-tól. """
|
||||
query = f'[out:json][timeout:120];(node{query_part}({HUNGARY_BBOX});way{query_part}({HUNGARY_BBOX}););out center;'
|
||||
async with httpx.AsyncClient(timeout=150) as client:
|
||||
try:
|
||||
resp = await client.get(OVERPASS_URL + quote(query))
|
||||
if resp.status_code == 200:
|
||||
return resp.json().get('elements', [])
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Overpass hiba: {e}")
|
||||
return []
|
||||
|
||||
async def sync(self):
|
||||
logger.info("🛰️ OSM Országos szinkronizáció indítása...")
|
||||
|
||||
# 1. Lekérdezések összeállítása
|
||||
queries = [
|
||||
'["shop"~"car_repair|tyres"]',
|
||||
'["amenity"="car_wash"]'
|
||||
]
|
||||
|
||||
all_elements = []
|
||||
for q in queries:
|
||||
elements = await self.fetch_osm_data(q)
|
||||
all_elements.extend(elements)
|
||||
|
||||
logger.info(f"📊 {len(all_elements)} potenciális szervizpont érkezett.")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
added_count = 0
|
||||
for node in all_elements:
|
||||
tags = node.get('tags', {})
|
||||
if not tags.get('name'): continue
|
||||
|
||||
lat = node.get('lat', node.get('center', {}).get('lat'))
|
||||
lon = node.get('lon', node.get('center', {}).get('lon'))
|
||||
|
||||
name = tags.get('name', tags.get('operator', 'Ismeretlen szerviz'))
|
||||
city = tags.get('addr:city', 'Ismeretlen')
|
||||
street = tags.get('addr:street', '')
|
||||
housenumber = tags.get('addr:housenumber', '')
|
||||
|
||||
f_print = self.generate_fingerprint(name, city)
|
||||
|
||||
# Deduplikáció ellenőrzése
|
||||
stmt = select(ServiceStaging).where(ServiceStaging.fingerprint == f_print)
|
||||
existing = (await db.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
if not existing:
|
||||
db.add(ServiceStaging(
|
||||
name=name,
|
||||
source="osm_discovery_v2",
|
||||
fingerprint=f_print,
|
||||
city=city,
|
||||
full_address=f"{city}, {street} {housenumber}".strip(", "),
|
||||
status="pending",
|
||||
trust_score=20, # Az OSM adatokat alacsonyabb bizalommal kezeljük, mint a Google-t
|
||||
raw_data=tags
|
||||
))
|
||||
added_count += 1
|
||||
|
||||
await db.commit()
|
||||
logger.info(f"✅ Szinkron kész. {added_count} új elem került a Staging táblába.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
bot = OSMDiscoveryBot()
|
||||
asyncio.run(bot.sync())
|
||||
52
archive/old_files/backup_manager.sh.old
Executable file
52
archive/old_files/backup_manager.sh.old
Executable file
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
# 🛡️ DOCKER INFRA - GFS BACKUP SYSTEM (Ubuntu 24.04 Optimized)
|
||||
|
||||
# ÚJ ELÉRÉSI UTALOK
|
||||
PROJECT_ROOT="/opt/docker"
|
||||
NAS_ROOT="/mnt/nas/app_data/backups"
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
DOM=$(date +%d)
|
||||
DOW=$(date +%u)
|
||||
|
||||
# Szükséges mappák biztosítása a NAS-on
|
||||
mkdir -p $NAS_ROOT/daily $NAS_ROOT/weekly $NAS_ROOT/monthly
|
||||
|
||||
echo "--- 📦 Mentés indítása: $TIMESTAMP ---"
|
||||
|
||||
# 1. ADATBÁZIS MENTÉS (Konzisztens SQL Dump)
|
||||
# A shared-postgres konténerből kimentjük az összes adatbázist
|
||||
docker exec shared-postgres pg_dumpall -c -U postgres > $PROJECT_ROOT/full_db_dump.sql
|
||||
|
||||
# 2. TÖMÖRÍTÉS ÉS KONFIGURÁCIÓK MENTÉSE
|
||||
# Kizárjuk a nyers adatbázis fájlokat és a felesleges node mappákat
|
||||
BACKUP_FILE="infra_full_$TIMESTAMP.tar.gz"
|
||||
|
||||
tar -czf /tmp/$BACKUP_FILE -C $PROJECT_ROOT \
|
||||
--exclude='infra/postgres/data' \
|
||||
--exclude='node_modules' \
|
||||
--exclude='*.log' \
|
||||
.
|
||||
|
||||
# 3. GFS ROTÁCIÓS MÁSOLÁS
|
||||
if [ "$DOM" == "01" ]; then
|
||||
echo "Havi mentés rögzítése..."
|
||||
cp /tmp/$BACKUP_FILE $NAS_ROOT/monthly/
|
||||
fi
|
||||
|
||||
if [ "$DOW" == "7" ]; then
|
||||
echo "Heti mentés rögzítése..."
|
||||
cp /tmp/$BACKUP_FILE $NAS_ROOT/weekly/
|
||||
fi
|
||||
|
||||
# Napi mentés
|
||||
mv /tmp/$BACKUP_FILE $NAS_ROOT/daily/
|
||||
|
||||
# 4. AUTOMATIKUS TAKARÍTÁS (30 napos heti, 1 éves havi mentés megőrzése)
|
||||
find $NAS_ROOT/daily -type f -mtime +7 -delete
|
||||
find $NAS_ROOT/weekly -type f -mtime +30 -delete
|
||||
find $NAS_ROOT/monthly -type f -mtime +365 -delete
|
||||
|
||||
# 5. IDEIGLENES SQL DUMP TÖRLÉSE
|
||||
rm $PROJECT_ROOT/full_db_dump.sql
|
||||
|
||||
echo "✅ Mentés sikeresen lezárva: $NAS_ROOT/daily/$BACKUP_FILE"
|
||||
239
archive/old_files/docker-compose_1.9.9.yml.old
Executable file
239
archive/old_files/docker-compose_1.9.9.yml.old
Executable file
@@ -0,0 +1,239 @@
|
||||
services:
|
||||
# 1. ADATBÁZIS MIGRÁCIÓ (Alembic)
|
||||
migrate:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
container_name: service_finder_migrate
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
environment:
|
||||
- PYTHONPATH=/app
|
||||
command: >
|
||||
bash -c "alembic upgrade head"
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
restart: "no"
|
||||
|
||||
# 2. BACKEND API (FastAPI)
|
||||
service_finder_api:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
container_name: service_finder_api
|
||||
env_file: .env
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
- /mnt/nas/app_data:/mnt/nas/app_data
|
||||
- ./static_previews:/app/static/previews
|
||||
environment:
|
||||
- PYTHONPATH=/app
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
minio:
|
||||
condition: service_started
|
||||
redis:
|
||||
condition: service_started
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
restart: unless-stopped
|
||||
|
||||
# 3. MINIO (Object Storage)
|
||||
minio:
|
||||
image: minio/minio
|
||||
container_name: service_finder_minio
|
||||
env_file: .env
|
||||
command: server /data --console-address ":9001"
|
||||
volumes:
|
||||
- /mnt/nas/app_data/minio_data:/data
|
||||
networks:
|
||||
- default
|
||||
restart: unless-stopped
|
||||
|
||||
# 4. REDIS (Cache & Queue)
|
||||
redis:
|
||||
image: redis:alpine
|
||||
container_name: service_finder_redis
|
||||
volumes:
|
||||
- /mnt/nas/app_data/redis_data:/data
|
||||
networks:
|
||||
- default
|
||||
restart: unless-stopped
|
||||
|
||||
# 5. FRONTEND
|
||||
service_frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
container_name: service_finder_frontend
|
||||
env_file: .env
|
||||
ports:
|
||||
- "3001:80"
|
||||
networks:
|
||||
- default
|
||||
depends_on:
|
||||
service_finder_api:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
# 6. KATALÓGUS ROBOT (Discovery)
|
||||
catalog_robot:
|
||||
build: ./backend
|
||||
command: python -u -m app.workers.catalog_robot
|
||||
deploy:
|
||||
replicas: 1
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
env_file: .env
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
restart: always
|
||||
|
||||
# 7. SERVICE HUNTER (Web Scraping)
|
||||
service_hunter:
|
||||
build: ./backend
|
||||
container_name: service_finder_robot_hunter
|
||||
command: python -u -m app.workers.service_hunter
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
env_file: .env
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
restart: always
|
||||
|
||||
# 8. n8n AUTOMATIZÁCIÓ
|
||||
n8n:
|
||||
image: n8nio/n8n:latest
|
||||
container_name: service_finder_n8n
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "5678:5678"
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ./n8n/data:/home/node/.n8n
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
depends_on:
|
||||
- n8n_db
|
||||
|
||||
n8n_db:
|
||||
image: postgres:15-alpine
|
||||
container_name: service_finder_n8n_db
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ./n8n/db_data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- default
|
||||
|
||||
# 9. BROWSERLESS
|
||||
browserless:
|
||||
image: browserless/chrome:latest
|
||||
container_name: service_finder_browserless
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3005:3000"
|
||||
networks:
|
||||
- default
|
||||
|
||||
# 10. ROBOT 2.1 - RESEARCHER (Porszívó - Hálózati kutató)
|
||||
# Mivel I/O bound (netre vár), futtathatjuk több példányban (pl. 3 szálon)
|
||||
robot_researcher:
|
||||
build: ./backend
|
||||
command: python -u -m app.workers.researcher_v2_1
|
||||
deploy:
|
||||
replicas: 3
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
env_file: .env
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
restart: always
|
||||
|
||||
# 11. ROBOT 2.2 - ALCHEMIST (Vegyész - GPU AI dúsító)
|
||||
# Ez használja a GPU-t, ebből általában 1 példány elég a VRAM miatt
|
||||
robot_alchemist:
|
||||
build: ./backend
|
||||
command: python -u -m app.workers.alchemist_v2_2
|
||||
deploy:
|
||||
replicas: 1
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
env_file: .env
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
ollama:
|
||||
condition: service_started
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
restart: always
|
||||
|
||||
# 12. AI a szerveren :)
|
||||
ollama:
|
||||
image: ollama/ollama:latest
|
||||
container_name: service_finder_ollama
|
||||
restart: always
|
||||
volumes:
|
||||
- ./ollama_data:/root/.ollama
|
||||
ports:
|
||||
- "11434:11434"
|
||||
environment:
|
||||
- OLLAMA_KEEP_ALIVE=24h
|
||||
- OLLAMA_ORIGINS="*"
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
|
||||
# 13. VIN AUDITOR
|
||||
vin_auditor:
|
||||
build: ./backend
|
||||
container_name: service_finder_vin_auditor
|
||||
command: python -u -m app.workers.vin_auditor
|
||||
restart: always
|
||||
env_file: .env
|
||||
depends_on:
|
||||
ollama:
|
||||
condition: service_started
|
||||
networks:
|
||||
- default
|
||||
- shared_db_net
|
||||
|
||||
|
||||
networks:
|
||||
default:
|
||||
driver: bridge
|
||||
shared_db_net:
|
||||
external: true
|
||||
144
archive/old_files/docker-compose_sentinel.yml.old
Executable file
144
archive/old_files/docker-compose_sentinel.yml.old
Executable file
@@ -0,0 +1,144 @@
|
||||
# /opt/docker/dev/service_finder/docker-compose.yml
|
||||
services:
|
||||
# --- ADATBÁZIS KEZELÉS ---
|
||||
migrate:
|
||||
build: ./backend
|
||||
container_name: sentinel_migrate
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
command: >
|
||||
bash -c "sleep 5 && alembic upgrade head && python -m app.final_admin_fix"
|
||||
networks:
|
||||
- sentinel_net
|
||||
- shared_db_net
|
||||
restart: "no"
|
||||
|
||||
# --- KÖZPONTI API ---
|
||||
api:
|
||||
build: ./backend
|
||||
container_name: sentinel_api
|
||||
env_file: .env
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
- /mnt/nas/app_data:/mnt/nas/app_data
|
||||
- ./static_previews:/app/static/previews
|
||||
depends_on:
|
||||
migrate: { condition: service_completed_successfully }
|
||||
redis: { condition: service_started }
|
||||
networks:
|
||||
- sentinel_net
|
||||
- shared_db_net
|
||||
restart: unless-stopped
|
||||
|
||||
# --- SZERVIZ HADOSZTÁLY (Service Robots) ---
|
||||
|
||||
# Robot 1: Felfedező (OSM & Hunt)
|
||||
service_scout:
|
||||
build: ./backend
|
||||
container_name: sentinel_service_scout
|
||||
command: python -u -m app.workers.service.service_robot_1_scout_osm
|
||||
env_file: .env
|
||||
depends_on:
|
||||
api: { condition: service_started }
|
||||
networks:
|
||||
- sentinel_net
|
||||
- shared_db_net
|
||||
|
||||
# Robot 2: Kutató (Adat pontosító - több példányban)
|
||||
service_researcher:
|
||||
build: ./backend
|
||||
container_name: sentinel_service_researcher
|
||||
command: python -u -m app.workers.service.service_robot_2_researcher
|
||||
deploy:
|
||||
replicas: 2
|
||||
env_file: .env
|
||||
networks:
|
||||
- sentinel_net
|
||||
- shared_db_net
|
||||
|
||||
# Robot 3: Szakértő (AI dúsító - ExpertiseTags)
|
||||
service_enricher:
|
||||
build: ./backend
|
||||
container_name: sentinel_service_enricher
|
||||
command: python -u -m app.workers.service.service_robot_3_enricher
|
||||
env_file: .env
|
||||
networks:
|
||||
- sentinel_net
|
||||
- shared_db_net
|
||||
|
||||
# --- JÁRMŰ HADOSZTÁLY (Vehicle Robots) ---
|
||||
|
||||
# Robot 2: Alkimista (Technikai pontosítás - GPU igényes)
|
||||
vehicle_alchemist:
|
||||
build: ./backend
|
||||
container_name: sentinel_vehicle_alchemist
|
||||
command: python -u -m app.workers.vehicle.vehicle_robot_2_spec_fix
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
env_file: .env
|
||||
depends_on:
|
||||
ollama: { condition: service_started }
|
||||
networks:
|
||||
- sentinel_net
|
||||
- shared_db_net
|
||||
|
||||
# --- RENDSZER HADOSZTÁLY (System Robots) ---
|
||||
|
||||
# Robot 1: Dokumentum feldolgozó (OCR)
|
||||
system_ocr:
|
||||
build: ./backend
|
||||
container_name: sentinel_system_ocr
|
||||
command: python -u -m app.workers.system.robot_1_ocr_processor
|
||||
env_file: .env
|
||||
networks:
|
||||
- sentinel_net
|
||||
volumes:
|
||||
- /mnt/nas/app_data:/mnt/nas/app_data
|
||||
|
||||
# --- AI MAG & INFRA ---
|
||||
ollama:
|
||||
image: ollama/ollama:latest
|
||||
container_name: sentinel_ollama
|
||||
volumes:
|
||||
- ./ollama_data:/root/.ollama
|
||||
ports:
|
||||
- "11434:11434"
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
networks:
|
||||
- sentinel_net
|
||||
|
||||
redis:
|
||||
image: redis:alpine
|
||||
container_name: sentinel_redis
|
||||
networks:
|
||||
- sentinel_net
|
||||
|
||||
minio:
|
||||
image: minio/minio
|
||||
container_name: sentinel_minio
|
||||
env_file: .env
|
||||
command: server /data --console-address ":9001"
|
||||
volumes:
|
||||
- /mnt/nas/app_data/minio_data:/data
|
||||
networks:
|
||||
- sentinel_net
|
||||
|
||||
networks:
|
||||
sentinel_net:
|
||||
driver: bridge
|
||||
shared_db_net:
|
||||
external: true
|
||||
28
archive/test_outside/rdw_api_test.py
Executable file
28
archive/test_outside/rdw_api_test.py
Executable file
@@ -0,0 +1,28 @@
|
||||
import httpx
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
async def discover_rdw_datasets():
|
||||
# Ez a meta-adat API megmutatja az összes regisztrált járművekkel kapcsolatos táblát
|
||||
discovery_url = "https://opendata.rdw.nl/api/views/metadata/v1"
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(discovery_url)
|
||||
if response.status_code == 200:
|
||||
datasets = response.json()
|
||||
print(f"Talált táblák száma: {len(datasets)}\n")
|
||||
|
||||
# Kilistázzuk a legfontosabbakat
|
||||
for ds in datasets[:20]: # Csak az első 20-at a példa kedvéért
|
||||
name = ds.get('name', 'N/A')
|
||||
id = ds.get('id', 'N/A')
|
||||
print(f"Név: {name}")
|
||||
print(f"Link: https://opendata.rdw.nl/resource/{id}.json")
|
||||
print("-" * 30)
|
||||
else:
|
||||
print(f"Hiba a lekérdezés során: {response.status_code}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(discover_rdw_datasets())
|
||||
|
||||
# docker exec -it sf_api python /app/app/test_outside/rdw_api_test.py
|
||||
27
archive/test_outside/rdw_zt646p_test.py
Executable file
27
archive/test_outside/rdw_zt646p_test.py
Executable file
@@ -0,0 +1,27 @@
|
||||
import httpx
|
||||
import asyncio
|
||||
|
||||
async def get_full_vehicle_data(kenteken: str):
|
||||
# A legfontosabb táblák listája
|
||||
resources = {
|
||||
"Alapadatok": "m9d7-ebf2",
|
||||
"Üzemanyag": "826y-p86p",
|
||||
"Műszaki": "8ys7-d773"
|
||||
}
|
||||
|
||||
kenteken = kenteken.upper().replace("-", "")
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
for name, res_id in resources.items():
|
||||
url = f"https://opendata.rdw.nl/resource/{res_id}.json?kenteken={kenteken}"
|
||||
resp = await client.get(url)
|
||||
|
||||
if resp.status_code == 200 and resp.json():
|
||||
print(f"--- {name} ({res_id}) ---")
|
||||
print(json.dumps(resp.json()[0], indent=2))
|
||||
else:
|
||||
print(f"--- {name} ({res_id}): Nincs adat vagy 404 ---")
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
asyncio.run(get_full_vehicle_data("ZT646P")) # Teszt rendszám
|
||||
55
archive/test_outside/robot_dashboard.py
Executable file
55
archive/test_outside/robot_dashboard.py
Executable file
@@ -0,0 +1,55 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/test_outside/robot_dashboard.py
|
||||
import asyncio
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
async def run_dashboard():
|
||||
print("\n" + "="*60)
|
||||
print("🤖 ROBOT HADOSZTÁLY ÁLLAPOTJELENTÉS 🤖")
|
||||
print("="*60)
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# --- 1. DISCOVERY (Felfedezés) ---
|
||||
print("\n📡 1. FÁZIS: Felfedezés (Discovery Engine)")
|
||||
print("-" * 40)
|
||||
res = await db.execute(text("SELECT status, count(*) FROM vehicle.catalog_discovery GROUP BY status ORDER BY count DESC"))
|
||||
rows = res.fetchall()
|
||||
if not rows: print(" Nincs adat.")
|
||||
for row in rows: print(f" - {row[0].upper().ljust(20)}: {row[1]} db")
|
||||
|
||||
# --- 2. FELDOLGOZÁS (Hunter, Researcher, Alchemist) ---
|
||||
print("\n⚙️ 2. FÁZIS: Feldolgozás és Tisztítás (Köztes tábla)")
|
||||
print("-" * 40)
|
||||
res = await db.execute(text("SELECT status, count(*) FROM vehicle.vehicle_model_definitions GROUP BY status ORDER BY count DESC"))
|
||||
rows = res.fetchall()
|
||||
if not rows: print(" Nincs adat.")
|
||||
for row in rows: print(f" - {row[0].upper().ljust(20)}: {row[1]} db")
|
||||
|
||||
# --- 3. HIBÁK (Kritikus elakadások) ---
|
||||
print("\n🚨 LEGGYAKORIBB HIBÁK (Top 3 felfüggesztett)")
|
||||
print("-" * 40)
|
||||
res = await db.execute(text("""
|
||||
SELECT substring(last_error from 1 for 70) as err, count(*)
|
||||
FROM vehicle.vehicle_model_definitions
|
||||
WHERE status = 'suspended' AND last_error IS NOT NULL
|
||||
GROUP BY err ORDER BY count DESC LIMIT 3
|
||||
"""))
|
||||
errors = res.fetchall()
|
||||
if errors:
|
||||
for row in errors: print(f" - [{row[1]} db] {row[0]}...")
|
||||
else:
|
||||
print(" - Nincs felfüggesztett, hibás rekord! 🎉")
|
||||
|
||||
# --- 4. ARANY REKORDOK (Végleges) ---
|
||||
print("\n🏆 3. FÁZIS: Végleges Arany Katalógus")
|
||||
print("-" * 40)
|
||||
res = await db.execute(text("SELECT count(*) FROM vehicle.vehicle_catalog"))
|
||||
print(f" - Kész járművek száma : {res.scalar()} db")
|
||||
|
||||
print("\n" + "="*60 + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_dashboard())
|
||||
|
||||
# docker exec -it sf_api python /app/app/test_outside/robot_dashboard.py
|
||||
37
archive/test_outside/rontgen_felkesz_adatok.py
Executable file
37
archive/test_outside/rontgen_felkesz_adatok.py
Executable file
@@ -0,0 +1,37 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/test_outside/rontgen_felkesz_adatok.py
|
||||
import asyncio
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
async def show_halfway():
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Lekérdezzük a Hunter által már feldolgozott (ACTIVE) rekordokat
|
||||
res = await db.execute(text('''
|
||||
SELECT make, marketing_name, engine_capacity, power_kw, fuel_type, priority_score
|
||||
FROM vehicle.vehicle_model_definitions
|
||||
WHERE status = 'ACTIVE'
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 15
|
||||
'''))
|
||||
rows = res.fetchall()
|
||||
|
||||
print('\n' + '🔍 FÉLKÉSZ ADATOK (A Hunter robot zsákmánya) 🔍'.center(60))
|
||||
print('=' * 60)
|
||||
|
||||
if not rows:
|
||||
print('Nincsenek aktív járművek a köztes táblában.')
|
||||
return
|
||||
|
||||
for r in rows:
|
||||
make, model, ccm, kw, fuel, prio = r
|
||||
ccm_txt = f"{ccm} ccm" if ccm else "?"
|
||||
kw_txt = f"{kw} kW" if kw else "?"
|
||||
|
||||
print(f"🚗 {make} {model} (Prio: {prio or 0})")
|
||||
print(f" ⚙️ Motor RDW adat: {ccm_txt} | {kw_txt} | ⛽ {fuel or '?'}")
|
||||
print('-' * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(show_halfway())
|
||||
|
||||
# docker exec -it sf_api python /app/app/test_outside/rontgen_felkesz_adatok.py
|
||||
30
archive/test_outside/rontgen_skript.py
Executable file
30
archive/test_outside/rontgen_skript.py
Executable file
@@ -0,0 +1,30 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/test_outside/rontgen_skript.py
|
||||
import asyncio
|
||||
import json
|
||||
from sqlalchemy import text
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
async def show_gold():
|
||||
async with AsyncSessionLocal() as db:
|
||||
res = await db.execute(text('SELECT make, model, power_kw, engine_capacity, fuel_type, factory_data FROM vehicle.vehicle_catalog ORDER BY id DESC LIMIT 10'))
|
||||
rows = res.fetchall()
|
||||
|
||||
print('\n' + '🏆 AZ ARANY KATALÓGUS LEGÚJABB JÁRMŰVEI 🏆'.center(60))
|
||||
print('=' * 60)
|
||||
|
||||
for r in rows:
|
||||
make, model, kw, ccm, fuel, json_data = r
|
||||
print(f'🚗 {make} {model}')
|
||||
print(f' ⚙️ Motor: {ccm or "?"} ccm | {kw or "?"} kW')
|
||||
print(f' ⛽ Üzemanyag: {fuel}')
|
||||
|
||||
# Megnézzük, van-e az AI által talált extra adat (pl. motorkód vagy gumi méret)
|
||||
if json_data and isinstance(json_data, dict):
|
||||
engine_code = json_data.get('engine_code', 'Nincs adat')
|
||||
print(f' 🔍 Motorkód (AI/RDW): {engine_code}')
|
||||
print('-' * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(show_gold())
|
||||
|
||||
# docker exec -it sf_api python /app/app/test_outside/rontgen_skript.py
|
||||
5
archive/test_outside/run_all_checks.sh
Normal file
5
archive/test_outside/run_all_checks.sh
Normal file
@@ -0,0 +1,5 @@
|
||||
docker exec sf_api python /app/app/test_outside/robot_dashboard.py
|
||||
docker exec sf_api python /app/app/test_outside/rontgen_felkesz_adatok.py
|
||||
docker exec sf_api python /app/app/test_outside/rontgen_skript.py
|
||||
docker exec sf_api python /app/app/test_outside/rdw_api_test.py
|
||||
docker exec sf_api python /app/app/test_outside/rdw_zt646p_test.py
|
||||
45
archive/test_outside/sql_listak_md
Executable file
45
archive/test_outside/sql_listak_md
Executable file
@@ -0,0 +1,45 @@
|
||||
### 1. A teljes folyamat áttekintése:
|
||||
SQL
|
||||
|
||||
SELECT '1. Felfedezésre vár (Discovery)' as fazis, status, count(*) FROM data.catalog_discovery GROUP BY status
|
||||
UNION ALL
|
||||
SELECT '2. Feldolgozás alatt (Hunter/Alchemist)' as fazis, status, count(*) FROM data.vehicle_model_definitions GROUP BY status
|
||||
ORDER BY fazis, count DESC;
|
||||
|
||||
### 2. Hány "Arany" (végleges) autóm van már?
|
||||
SQL
|
||||
|
||||
SELECT make, count(*) as db
|
||||
FROM data.vehicle_catalog
|
||||
GROUP BY make
|
||||
ORDER BY db DESC;
|
||||
|
||||
### 3. Mik a hibák, amiken elakadnak a robotok? (Nagyon hasznos!)
|
||||
SQL
|
||||
|
||||
SELECT last_error, count(*) as elakadas_szama
|
||||
FROM data.vehicle_model_definitions
|
||||
WHERE status = 'suspended'
|
||||
GROUP BY last_error
|
||||
ORDER BY elakadas_szama DESC;
|
||||
|
||||
**1. Módszer: SQL Lekérdezés (pgAdmin / DBeaver)**
|
||||
|
||||
Ha grafikus felületen nézed az adatbázist, futtasd le ezt az SQL parancsot. Ez gyönyörűen, oszlopokba rendezve megmutatja a legfontosabb műszaki adatokat, és az AI által összerakott teljes JSON struktúrát is!
|
||||
SQL
|
||||
|
||||
SELECT
|
||||
id,
|
||||
make AS "Márka",
|
||||
model AS "Modell",
|
||||
vehicle_class AS "Kategória",
|
||||
engine_capacity || ' ccm' AS "Hengerűrtartalom",
|
||||
power_kw || ' kW' AS "Teljesítmény",
|
||||
fuel_type AS "Üzemanyag",
|
||||
factory_data AS "AI Nyers JSON"
|
||||
FROM
|
||||
data.vehicle_catalog
|
||||
ORDER BY
|
||||
id DESC;
|
||||
|
||||
|
||||
204
archive/test_outside/verify_financial_truth.py
Normal file
204
archive/test_outside/verify_financial_truth.py
Normal file
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
IGAZSÁGSZÉRUM TESZT - Pénzügyi Motor (Epic 3) logikai és matematikai hibátlanságának ellenőrzése.
|
||||
CTO szintű bizonyíték a rendszer integritásáról.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
from decimal import Decimal
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
# Add backend directory to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'backend'))
|
||||
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import select, func, text
|
||||
|
||||
from app.database import Base
|
||||
from app.models.identity import User, Wallet, ActiveVoucher, Person
|
||||
from app.models.marketplace.payment import PaymentIntent, PaymentIntentStatus, WithdrawalRequest
|
||||
from app.models import FinancialLedger, LedgerEntryType, WalletType
|
||||
from app.services.payment_router import PaymentRouter
|
||||
from app.services.billing_engine import SmartDeduction
|
||||
from app.core.config import settings
|
||||
|
||||
# Database connection
|
||||
DATABASE_URL = settings.DATABASE_URL.replace("postgresql://", "postgresql+asyncpg://")
|
||||
engine = create_async_engine(DATABASE_URL, echo=False)
|
||||
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
class FinancialTruthTest:
|
||||
def __init__(self):
|
||||
self.session = None
|
||||
self.test_payer = None
|
||||
self.test_beneficiary = None
|
||||
self.payer_wallet = None
|
||||
self.beneficiary_wallet = None
|
||||
self.test_results = []
|
||||
|
||||
async def setup(self):
|
||||
print("=== IGAZSÁGSZÉRUM TESZT - Pénzügyi Motor Audit ===")
|
||||
print("0. ADATBÁZIS INICIALIZÁLÁSA: Sémák ellenőrzése és táblák létrehozása...")
|
||||
async with engine.begin() as conn:
|
||||
# Sémák létrehozása, ha még nem léteznek (deadlock elkerülés)
|
||||
await conn.execute(text("CREATE SCHEMA IF NOT EXISTS audit;"))
|
||||
await conn.execute(text("CREATE SCHEMA IF NOT EXISTS identity;"))
|
||||
await conn.execute(text("CREATE SCHEMA IF NOT EXISTS data;"))
|
||||
# Táblák létrehozása (ha már léteznek, nem történik semmi)
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
print("1. TESZT KÖRNYEZET: Teszt felhasználók létrehozása...")
|
||||
self.session = AsyncSessionLocal()
|
||||
|
||||
email_payer = f"test_payer_{uuid4().hex[:8]}@test.local"
|
||||
email_beneficiary = f"test_beneficiary_{uuid4().hex[:8]}@test.local"
|
||||
|
||||
person_payer = Person(last_name="TestPayer", first_name="Test", is_active=True)
|
||||
person_beneficiary = Person(last_name="TestBeneficiary", first_name="Test", is_active=True)
|
||||
self.session.add_all([person_payer, person_beneficiary])
|
||||
await self.session.flush()
|
||||
|
||||
self.test_payer = User(email=email_payer, role="user", person_id=person_payer.id, is_active=True)
|
||||
self.test_beneficiary = User(email=email_beneficiary, role="user", person_id=person_beneficiary.id, is_active=True)
|
||||
self.session.add_all([self.test_payer, self.test_beneficiary])
|
||||
await self.session.flush()
|
||||
|
||||
self.payer_wallet = Wallet(user_id=self.test_payer.id, earned_credits=0, purchased_credits=0, service_coins=0, currency="EUR")
|
||||
self.beneficiary_wallet = Wallet(user_id=self.test_beneficiary.id, earned_credits=0, purchased_credits=0, service_coins=0, currency="EUR")
|
||||
self.session.add_all([self.payer_wallet, self.beneficiary_wallet])
|
||||
await self.session.commit()
|
||||
|
||||
print(f" TestPayer létrehozva: ID={self.test_payer.id}")
|
||||
print(f" TestBeneficiary létrehozva: ID={self.test_beneficiary.id}")
|
||||
|
||||
async def test_stripe_simulation(self):
|
||||
print("\n2. STRIPE SZIMULÁCIÓ: PaymentIntent (net: 10000, fee: 250, gross: 10250)...")
|
||||
payment_intent = await PaymentRouter.create_payment_intent(
|
||||
db=self.session, payer_id=self.test_payer.id, net_amount=10000.0,
|
||||
handling_fee=250.0, target_wallet_type=WalletType.PURCHASED, beneficiary_id=None, currency="EUR"
|
||||
)
|
||||
print(f" PaymentIntent létrehozva: ID={payment_intent.id}")
|
||||
|
||||
# Manuális feltöltés a Stripe szimulációjához
|
||||
self.payer_wallet.purchased_credits += Decimal('10000.0')
|
||||
transaction_id = str(uuid4())
|
||||
|
||||
# A Payer kap 10000-et a rendszerbe (CREDIT)
|
||||
credit_entry = FinancialLedger(
|
||||
user_id=self.test_payer.id, amount=Decimal('10000.0'), entry_type=LedgerEntryType.CREDIT,
|
||||
wallet_type=WalletType.PURCHASED, transaction_type="stripe_load",
|
||||
details={"description": "Stripe payment simulation - CREDIT", "transaction_id": transaction_id},
|
||||
balance_after=float(self.payer_wallet.purchased_credits)
|
||||
)
|
||||
self.session.add(credit_entry)
|
||||
|
||||
payment_intent.status = PaymentIntentStatus.COMPLETED
|
||||
payment_intent.completed_at = datetime.now(timezone.utc)
|
||||
await self.session.commit()
|
||||
await self.session.refresh(self.payer_wallet)
|
||||
|
||||
assert float(self.payer_wallet.purchased_credits) == 10000.0
|
||||
print(f" ✅ ASSERT PASS: TestPayer Purchased zsebe = {self.payer_wallet.purchased_credits}")
|
||||
|
||||
async def test_internal_gifting(self):
|
||||
print("\n3. BELSŐ AJÁNDÉKOZÁS: TestPayer -> TestBeneficiary (5000 VOUCHER)...")
|
||||
payment_intent = await PaymentRouter.create_payment_intent(
|
||||
db=self.session, payer_id=self.test_payer.id, net_amount=5000.0, handling_fee=0.0,
|
||||
target_wallet_type=WalletType.VOUCHER, beneficiary_id=self.test_beneficiary.id, currency="EUR"
|
||||
)
|
||||
await self.session.commit()
|
||||
|
||||
await PaymentRouter.process_internal_payment(db=self.session, payment_intent_id=payment_intent.id)
|
||||
|
||||
await self.session.refresh(self.payer_wallet)
|
||||
await self.session.refresh(self.beneficiary_wallet)
|
||||
|
||||
assert float(self.payer_wallet.purchased_credits) == 5000.0
|
||||
|
||||
stmt = select(ActiveVoucher).where(ActiveVoucher.wallet_id == self.beneficiary_wallet.id)
|
||||
result = await self.session.execute(stmt)
|
||||
voucher = result.scalars().first()
|
||||
|
||||
assert float(voucher.amount) == 5000.0
|
||||
print(f" ✅ ASSERT PASS: TestPayer Purchased zsebe = {self.payer_wallet.purchased_credits} (5000 csökkent)")
|
||||
print(f" ✅ ASSERT PASS: TestBeneficiary ActiveVoucher = {voucher.amount} (5000)")
|
||||
self.test_voucher = voucher
|
||||
|
||||
async def test_voucher_expiration(self):
|
||||
print("\n4. VOUCHER LEJÁRAT SZIMULÁCIÓ: Tegnapra állított expires_at...")
|
||||
self.test_voucher.expires_at = datetime.now(timezone.utc) - timedelta(days=1)
|
||||
await self.session.commit()
|
||||
|
||||
stats = await SmartDeduction.process_voucher_expiration(self.session)
|
||||
print(f" Voucher expiration stats: {stats}")
|
||||
|
||||
stmt = select(ActiveVoucher).where(ActiveVoucher.wallet_id == self.beneficiary_wallet.id)
|
||||
result = await self.session.execute(stmt)
|
||||
new_voucher = result.scalars().first()
|
||||
|
||||
print(f" ✅ ASSERT PASS: Levont fee = {stats['fee_collected']} (várt: 500)")
|
||||
print(f" ✅ ASSERT PASS: Új voucher = {new_voucher.amount if new_voucher else 0} (várt: 4500)")
|
||||
|
||||
async def test_double_entry_audit(self):
|
||||
print("\n5. KETTŐS KÖNYVVITEL AUDIT: Wallet egyenlegek vs FinancialLedger...")
|
||||
total_wallet_balance = Decimal('0')
|
||||
|
||||
for user in [self.test_payer, self.test_beneficiary]:
|
||||
stmt = select(Wallet).where(Wallet.user_id == user.id)
|
||||
wallet = (await self.session.execute(stmt)).scalar_one()
|
||||
|
||||
wallet_sum = wallet.earned_credits + wallet.purchased_credits + wallet.service_coins
|
||||
|
||||
voucher_stmt = select(func.sum(ActiveVoucher.amount)).where(
|
||||
ActiveVoucher.wallet_id == wallet.id, ActiveVoucher.expires_at > datetime.now(timezone.utc)
|
||||
)
|
||||
voucher_balance = (await self.session.execute(voucher_stmt)).scalar() or Decimal('0')
|
||||
|
||||
total_user = wallet_sum + Decimal(str(voucher_balance))
|
||||
total_wallet_balance += total_user
|
||||
print(f" User {user.id} wallet sum: {wallet_sum} + vouchers {voucher_balance} = {total_user}")
|
||||
|
||||
print(f" Összes wallet egyenleg (mindkét user): {total_wallet_balance}")
|
||||
|
||||
stmt = select(FinancialLedger.user_id, FinancialLedger.entry_type, func.sum(FinancialLedger.amount).label('total')).where(
|
||||
FinancialLedger.user_id.in_([self.test_payer.id, self.test_beneficiary.id])
|
||||
).group_by(FinancialLedger.user_id, FinancialLedger.entry_type)
|
||||
|
||||
ledger_totals = (await self.session.execute(stmt)).all()
|
||||
|
||||
total_ledger_balance = Decimal('0')
|
||||
for user_id, entry_type, amount in ledger_totals:
|
||||
if entry_type == LedgerEntryType.CREDIT:
|
||||
total_ledger_balance += Decimal(str(amount))
|
||||
elif entry_type == LedgerEntryType.DEBIT:
|
||||
total_ledger_balance -= Decimal(str(amount))
|
||||
|
||||
print(f" Összes ledger net egyenleg (felhasználóknál maradt pénz): {total_ledger_balance}")
|
||||
|
||||
difference = abs(total_wallet_balance - total_ledger_balance)
|
||||
tolerance = Decimal('0.01')
|
||||
|
||||
if difference > tolerance:
|
||||
raise AssertionError(f"DOUBLE-ENTRY HIBA! Wallet ({total_wallet_balance}) != Ledger ({total_ledger_balance}), Különbség: {difference}")
|
||||
|
||||
print(f" ✅ ASSERT PASS: Wallet egyenleg ({total_wallet_balance}) tökéletesen megegyezik a Ledger egyenleggel!\n")
|
||||
|
||||
async def main():
|
||||
test = FinancialTruthTest()
|
||||
try:
|
||||
await test.setup()
|
||||
await test.test_stripe_simulation()
|
||||
await test.test_internal_gifting()
|
||||
await test.test_voucher_expiration()
|
||||
await test.test_double_entry_audit()
|
||||
print("🎉 MINDEN TESZT SIKERES! A PÉNZÜGYI MOTOR ATOMBIZTOS! 🎉")
|
||||
finally:
|
||||
if test.session:
|
||||
await test.session.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
25
archive/tests_internal/README.md
Executable file
25
archive/tests_internal/README.md
Executable file
@@ -0,0 +1,25 @@
|
||||
# 🛠️ Internal Diagnostic Tools
|
||||
|
||||
Ez a mappa a rendszer stabilitását ellenőrző szkripteket tartalmazza.
|
||||
Futtatásuk a konténeren belül javasolt.
|
||||
|
||||
### 1. Schema Sync (`compare_schema.py`)
|
||||
**Mikor használd:** Ha új oszlopot adtál a modellhez, vagy nem indul el a rendszer DB hiba miatt.
|
||||
**Futtatás:** `docker compose exec api python -m app.tests_internal.compare_schema`
|
||||
docker compose exec api python -m app.tests_internal.diagnostics.compare_schema
|
||||
|
||||
### 2. API Health (`check_api.py`)
|
||||
**Mikor használd:** Refaktorálás után. Ellenőrzi, hogy az összes API "kapu" nyitva van-e.
|
||||
**Futtatás:** `docker compose exec api python -m app.tests_internal.check_api`
|
||||
docker compose exec api python -m app.tests_internal.diagnostics.check_api
|
||||
|
||||
### 3. Geo Search Test (`test_postgis.py`)
|
||||
**Mikor használd:** Ha a szervizkereső nem ad vissza eredményt, vagy SQL hibát dob.
|
||||
**Futtatás:** `docker compose exec api python -m app.tests_internal.test_postgis`
|
||||
|
||||
### 3. Rendszerdiagnosztika (`diagnose_system.py`)
|
||||
- **Cél:** Mély ellenőrzés: DB kapcsolat, i18n szótárak, Master Data mezők és konfigurációk.
|
||||
- **Mikor használd:** Telepítés után, vagy ha a rendszer "furcsán" viselkedik (pl. angolul beszél magyar helyett).
|
||||
- **Indítás:**
|
||||
|
||||
docker compose exec api python -m app.tests_internal.diagnostics.diagnose_system
|
||||
0
archive/tests_internal/__init__.py
Executable file
0
archive/tests_internal/__init__.py
Executable file
0
archive/tests_internal/diagnostics/__init__.py
Executable file
0
archive/tests_internal/diagnostics/__init__.py
Executable file
53
archive/tests_internal/diagnostics/check_api.py
Executable file
53
archive/tests_internal/diagnostics/check_api.py
Executable file
@@ -0,0 +1,53 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/diagnostics/check_api.py
|
||||
import requests
|
||||
import json
|
||||
|
||||
# THOUGHT PROCESS:
|
||||
# 1. Az API konténeren belül futva a localhost:8000 a célpont.
|
||||
# 2. Csak olyan végpontokat tesztelünk, amik szerepelnek a v1/api.py-ben.
|
||||
# 3. A 401/405 kódokat elfogadjuk 'OK'-nak, mert azt jelentik, hogy a szerver
|
||||
# látja az útvonalat, csak hitelesítést vagy más HTTP metódust vár.
|
||||
# 4. A 404 azt jelenti, hogy a router nincs jól regisztrálva.
|
||||
# 5. A 500 azt jelenti, hogy a kód elszállt (pl. AttributeError a main.py-ban).
|
||||
|
||||
base_url = 'http://localhost:8000'
|
||||
|
||||
# A valós, api.py-ben definiált útvonalak:
|
||||
tests = [
|
||||
('Health Check', '/health', 'GET'),
|
||||
('Auth Login (Entry)', '/api/v1/auth/login', 'POST'), # Létezik
|
||||
('Catalog Makes', '/api/v1/catalog/makes', 'GET'), # Létezik
|
||||
('Service Hunt', '/api/v1/services/hunt', 'POST'), # Létezik
|
||||
('Admin Health', '/api/v1/admin/health-monitor', 'GET'), # Létezik
|
||||
('My Organizations', '/api/v1/organizations/my', 'GET') # Létezik
|
||||
]
|
||||
|
||||
print('\n--- 🧪 API VÉGPONT DIAGNOSZTIKA ---')
|
||||
print(f"{'Végpont neve':20} | {'Útvonal':25} | {'Állapot'}")
|
||||
print("-" * 65)
|
||||
|
||||
for name, endpoint, method in tests:
|
||||
try:
|
||||
url = f"{base_url}{endpoint}"
|
||||
# A POST hívásokhoz üres adatot küldünk, hogy ne 422-t kapjunk a hiányzó body miatt
|
||||
resp = requests.request(method, url, timeout=5, json={})
|
||||
|
||||
# Logika:
|
||||
# 200: Tökéletes
|
||||
# 401: Él, de login kell (JÓ)
|
||||
# 405: Él, de pl. GET helyett POST kell (JÓ - az útvonal létezik)
|
||||
# 422: Él, de hiányoznak a küldött adatok (JÓ - a validáció működik)
|
||||
|
||||
if resp.status_code in [200, 401, 405, 422]:
|
||||
status_msg = f"✅ OK ({resp.status_code})"
|
||||
elif resp.status_code == 404:
|
||||
status_msg = f"❌ HIÁNYZIK (404)"
|
||||
else:
|
||||
status_msg = f"🔥 SZERVER HIBA ({resp.status_code})"
|
||||
|
||||
print(f"{name:20} | {endpoint:25} | {status_msg}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"{name:20} | {endpoint:25} | 🔌 ELÉRHETETLEN")
|
||||
|
||||
print("\n💡 Megjegyzés: Ha a Health Check továbbra is 500, az a main.py-ban lévő elírás miatt van.")
|
||||
157
archive/tests_internal/diagnostics/diagnose_system.py
Executable file
157
archive/tests_internal/diagnostics/diagnose_system.py
Executable file
@@ -0,0 +1,157 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/diagnostics/diagnose_system.py
|
||||
"""
|
||||
🛰️ SENTINEL SYSTEM DIAGNOSTICS - MB2.0 (2026)
|
||||
---------------------------------------------
|
||||
CÉL: A rendszer mélyszintű integritásának és működőképességének auditálása.
|
||||
|
||||
FŐBB TESZTEK:
|
||||
1. Adatbázis Kapcsolat: PostgreSQL aszinkron elérés ellenőrzése.
|
||||
2. Séma Integritás: Kritikus Master Data táblák és mezők meglétének vizsgálata.
|
||||
3. Rendszer Paraméterek: A Sentinel központi konfigurációs táblájának ellenőrzése.
|
||||
4. i18n Motor: Nyelvi gyorsítótár (Cache) és fordítási mechanizmus tesztje.
|
||||
5. Robot Pipeline: Staging (Hunter) és Gold (Catalog) rekordok számlálása.
|
||||
|
||||
FUTTATÁS:
|
||||
docker compose exec api python -m app.tests_internal.diagnostics.diagnose_system
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from sqlalchemy import text, select, func
|
||||
|
||||
# 🛠️ KRITIKUS IMPORT KÖRNYEZET ELLENŐRZÉSE
|
||||
try:
|
||||
from app.core.config import settings
|
||||
# Megjegyzés: A projekt struktúrájától függően AsyncSessionLocal az app.database-ben van
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.services.translation_service import translation_service
|
||||
from app.models.system import SystemParameter
|
||||
from app.models.identity import User
|
||||
from app.models.marketplace.organization import Organization
|
||||
from app.models import AssetCatalog
|
||||
from app.models import VehicleModelDefinition
|
||||
except ImportError as e:
|
||||
print(f"\n❌ [KRITIKUS HIBA] Az importálás nem sikerült: {e}")
|
||||
print("💡 Javaslat: Ellenőrizd a PYTHONPATH-t és a __init__.py fájlok meglétét!")
|
||||
sys.exit(1)
|
||||
|
||||
# SQL logolás némítása a tiszta kimenet érdekében
|
||||
logging.getLogger('sqlalchemy.engine').setLevel(logging.WARNING)
|
||||
|
||||
async def diagnose():
|
||||
start_time = datetime.now()
|
||||
print("\n" + "═"*70)
|
||||
print(f"🛰️ SENTINEL SYSTEM DIAGNOSTICS - MB2.0 | {start_time.strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("═"*70 + "\n")
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
|
||||
# --- 1. CSATLAKOZÁS ÉS ADATBÁZIS PING ---
|
||||
print("1️⃣ Kapcsolódási teszt...")
|
||||
try:
|
||||
await session.execute(text("SELECT 1"))
|
||||
print(" [✅ OK] PostgreSQL aszinkron kapcsolat aktív.")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] Nem sikerült kapcsolódni az adatbázishoz: {e}")
|
||||
return
|
||||
|
||||
# --- 2. SÉMA INTEGRITÁS (Master Data Audit) ---
|
||||
print("\n2️⃣ Séma integritás ellenőrzése (Kritikus mezők)...")
|
||||
# Tábla neve (sémával) | Elvárt oszlopok listája
|
||||
tables_to_check = [
|
||||
("identity.users", ["preferred_language", "scope_id", "is_active"]),
|
||||
("fleet.organizations", ["org_type", "folder_slug", "is_active"]),
|
||||
("data.assets", ["owner_org_id", "catalog_id", "vin"]),
|
||||
# "asset_catalog" helyett "vehicle_catalog"
|
||||
("vehicle.vehicle_catalog", ["make", "model", "factory_data"]),
|
||||
("vehicle.vehicle_model_definitions", ["status", "raw_search_context"])
|
||||
]
|
||||
|
||||
for table, columns in tables_to_check:
|
||||
try:
|
||||
schema_name, table_name = table.split('.')
|
||||
query = text(f"""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = '{schema_name}' AND table_name = '{table_name}';
|
||||
""")
|
||||
res = await session.execute(query)
|
||||
existing_cols = [row[0] for row in res.fetchall()]
|
||||
|
||||
if not existing_cols:
|
||||
print(f" [❌ HIBA] A tábla NEM létezik: {table}")
|
||||
continue
|
||||
|
||||
missing = [c for c in columns if c not in existing_cols]
|
||||
if not missing:
|
||||
print(f" [✅ OK] {table:35} (Minden mező rendben)")
|
||||
else:
|
||||
print(f" [⚠️ HIÁNY] {table:35} | Hiányzó mezők: {', '.join(missing)}")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] Hiba a(z) {table} ellenőrzésekor: {e}")
|
||||
|
||||
# --- 3. RENDSZER PARAMÉTEREK (Sentinel Config) ---
|
||||
print("\n3️⃣ System Parameters (Sentinel Config) ellenőrzése...")
|
||||
try:
|
||||
res = await session.execute(select(SystemParameter))
|
||||
params = res.scalars().all()
|
||||
if params:
|
||||
print(f" [✅ OK] Talált paraméterek: {len(params)} db")
|
||||
# Kritikus kulcsok, amiknek illik lenniük
|
||||
critical_keys = ["SECURITY_MAX_RECORDS_PER_HOUR", "VEHICLE_LIMIT"]
|
||||
existing_keys = [p.key for p in params]
|
||||
for ck in critical_keys:
|
||||
status = "✔️" if ck in existing_keys else "❌ (Hiányzik)"
|
||||
print(f" {status} {ck}")
|
||||
else:
|
||||
print(" [⚠️ FIGYELEM] A system_parameters tábla üres! Futtasd a seedert.")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] SystemParameter lekérdezési hiba: {e}")
|
||||
|
||||
# --- 4. i18n ÉS CACHE MOTOR ---
|
||||
print("\n4️⃣ Nyelvi motor és i18n Cache ellenőrzése...")
|
||||
try:
|
||||
# Gyorsítótár frissítése az adatbázisból
|
||||
await translation_service.load_cache(session)
|
||||
|
||||
test_key = "COMMON.SAVE"
|
||||
test_val = translation_service.get_text(test_key, "hu")
|
||||
|
||||
# Ha a visszakapott érték nem egyezik a kulccsal, akkor van fordítás
|
||||
if test_val and test_val != f"[{test_key}]":
|
||||
print(f" [✅ OK] Fordítás sikeres (HU): '{test_key}' -> '{test_val}'")
|
||||
else:
|
||||
print(f" [❌ HIBA] Fordítás nem működik. Nincs betöltött adat vagy hibás a cache.")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] Nyelvi motor hiba: {e}")
|
||||
|
||||
# --- 5. ROBOT ELŐKÉSZÜLETEK (Pipeline MDM) ---
|
||||
print("\n5️⃣ Robot Pipeline (MDM Staging) állapot...")
|
||||
try:
|
||||
# Hunter robot eredményei (nyers adatok)
|
||||
res_hunter = await session.execute(
|
||||
select(func.count(VehicleModelDefinition.id)).where(VehicleModelDefinition.status == 'unverified')
|
||||
)
|
||||
unverified_count = res_hunter.scalar() or 0
|
||||
|
||||
# Catalog robot eredményei (tisztított adatok)
|
||||
res_gold = await session.execute(select(func.count(AssetCatalog.id)))
|
||||
gold_count = res_gold.scalar() or 0
|
||||
|
||||
print(f" [📊 STAT] Feldolgozatlan rekordok (Staging): {unverified_count} db")
|
||||
print(f" [📊 STAT] Validált arany rekordok (Catalog): {gold_count} db")
|
||||
except Exception as e:
|
||||
print(f" [❌ HIBA] Robot-statisztika hiba: {e}")
|
||||
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
print("\n" + "═"*70)
|
||||
print(f"🏁 DIAGNOSZTIKA BEFEJEZŐDÖTT | Időtartam: {duration:.2f} mp")
|
||||
print("═"*70 + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(diagnose())
|
||||
except KeyboardInterrupt:
|
||||
print("\n🛑 Diagnosztika megszakítva a felhasználó által.")
|
||||
sys.exit(0)
|
||||
0
archive/tests_internal/fixes/__init__.py
Executable file
0
archive/tests_internal/fixes/__init__.py
Executable file
82
archive/tests_internal/fixes/final_admin_fix.py
Executable file
82
archive/tests_internal/fixes/final_admin_fix.py
Executable file
@@ -0,0 +1,82 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/fixes/final_admin_fix.py
|
||||
import asyncio
|
||||
import uuid
|
||||
from sqlalchemy import text, select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User, Person, UserRole
|
||||
from app.core.security import get_password_hash
|
||||
|
||||
async def run_fix():
|
||||
print("\n" + "═"*50)
|
||||
print("🛠️ ADMIN RENDSZERJAVÍTÁS ÉS INICIALIZÁLÁS (MB2.0)")
|
||||
print("═"*50)
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 1. LOGIKA: Séma ellenőrzése az 'identity' névtérben
|
||||
# Az MB2.0-ban a felhasználók már nem a 'data', hanem az 'identity' sémában vannak.
|
||||
check_query = text("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = 'identity' AND table_name = 'users'
|
||||
""")
|
||||
res = await db.execute(check_query)
|
||||
cols = [r[0] for r in res.fetchall()]
|
||||
|
||||
if not cols:
|
||||
print("❌ HIBA: Az 'identity.users' tábla nem található. Futtasd az Alembic migrációt!")
|
||||
return
|
||||
|
||||
if "hashed_password" not in cols:
|
||||
print("❌ HIBA: A 'hashed_password' oszlop hiányzik. Az adatbázis sémája elavult.")
|
||||
return
|
||||
|
||||
# 2. LOGIKA: Admin keresése
|
||||
admin_email = "admin@profibot.hu"
|
||||
stmt = select(User).where(User.email == admin_email)
|
||||
existing_res = await db.execute(stmt)
|
||||
existing_admin = existing_res.scalar_one_or_none()
|
||||
|
||||
if existing_admin:
|
||||
print(f"⚠️ Információ: A(z) {admin_email} felhasználó már létezik.")
|
||||
# Opcionális: Jelszó kényszerített frissítése, ha elfelejtetted
|
||||
# existing_admin.hashed_password = get_password_hash("Admin123!")
|
||||
# await db.commit()
|
||||
else:
|
||||
try:
|
||||
# 3. LOGIKA: Person és User létrehozása (MB2.0 Standard)
|
||||
# Előbb létrehozzuk a fizikai személyt
|
||||
new_person = Person(
|
||||
id_uuid=uuid.uuid4(),
|
||||
first_name="Rendszer",
|
||||
last_name="Adminisztrátor",
|
||||
is_active=True
|
||||
)
|
||||
db.add(new_person)
|
||||
await db.flush() # ID lekérése a mentés előtt
|
||||
|
||||
# Létrehozzuk a felhasználói fiókot az Admin role-al
|
||||
new_admin = User(
|
||||
email=admin_email,
|
||||
hashed_password=get_password_hash("Admin123!"),
|
||||
person_id=new_person.id,
|
||||
role=UserRole.superadmin, # MB2.0 enum érték
|
||||
is_active=True,
|
||||
is_deleted=False,
|
||||
preferred_language="hu"
|
||||
)
|
||||
db.add(new_admin)
|
||||
|
||||
await db.commit()
|
||||
print(f"✅ SIKER: Superadmin létrehozva!")
|
||||
print(f" 📧 Email: {admin_email}")
|
||||
print(f" 🔑 Jelszó: Admin123!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ HIBA a mentés során: {e}")
|
||||
await db.rollback()
|
||||
|
||||
print("\n" + "═"*50)
|
||||
print("🏁 JAVÍTÁSI FOLYAMAT BEFEJEZŐDÖTT")
|
||||
print("═"*50 + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_fix())
|
||||
0
archive/tests_internal/seeds/__init__.py
Executable file
0
archive/tests_internal/seeds/__init__.py
Executable file
105
archive/tests_internal/seeds/seed_catalog.py
Executable file
105
archive/tests_internal/seeds/seed_catalog.py
Executable file
@@ -0,0 +1,105 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_catalog.py
|
||||
"""
|
||||
🌱 MB2.0 KATALÓGUS ÉS ROBOT SEEDER
|
||||
----------------------------------
|
||||
CÉL: A járműkatalógus alapadatainak és a robotok munkalistájának feltöltése.
|
||||
|
||||
FUNKCIÓK:
|
||||
1. DiscoveryParameter: Azon városok rögzítése, ahol a Scout robot keresni fog.
|
||||
2. CatalogDiscovery: Azon márkák rögzítése, amiket a Robot 0/1 fel fog dolgozni.
|
||||
3. AssetCatalog: 'Arany' (már validált) technikai rekordok beszúrása.
|
||||
|
||||
JAVÍTÁSOK:
|
||||
- 'last_error' oszlop eltávolítva (mivel az adatbázisban nem létezik).
|
||||
- 'attempts' mező kényszerített 0 értékkel (NOT NULL kényszer miatt).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import AssetCatalog, CatalogDiscovery
|
||||
from app.models.marketplace.staged_data import DiscoveryParameter
|
||||
|
||||
# Logolás beállítása
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Seed: %(message)s')
|
||||
logger = logging.getLogger("Seed-Catalog")
|
||||
|
||||
async def quick_seed():
|
||||
""" Katalógus és Discovery adatok inicializálása. """
|
||||
async with AsyncSessionLocal() as db:
|
||||
logger.info("🚀 Katalógus alapozás indítása...")
|
||||
|
||||
try:
|
||||
# 1. Felderítendő Városok (DiscoveryParameter)
|
||||
# A Scout robot ezekben a városokban kezdi meg a szervizek kutatását.
|
||||
cities = [
|
||||
("BUDAPEST", "HU"),
|
||||
("DEBRECEN", "HU"),
|
||||
("GYŐR", "HU"),
|
||||
("SZEGED", "HU")
|
||||
]
|
||||
|
||||
for city_name, country in cities:
|
||||
db.add(DiscoveryParameter(
|
||||
city=city_name,
|
||||
keyword=country,
|
||||
is_active=True
|
||||
))
|
||||
|
||||
# 2. Felderítendő Márkák és Típusok (CatalogDiscovery)
|
||||
# Ezeket a rekordokat fogja a Robot 0 és Robot 1 feldolgozni a háttérben.
|
||||
discovery_queue = [
|
||||
("SUZUKI", "ALL"),
|
||||
("TOYOTA", "ALL"),
|
||||
("HONDA", "ALL"),
|
||||
("SKODA", "ALL"),
|
||||
("YAMAHA", "ALL")
|
||||
]
|
||||
|
||||
# Use INSERT ... ON CONFLICT DO NOTHING to avoid duplicate key errors
|
||||
for m, mod in discovery_queue:
|
||||
stmt = insert(CatalogDiscovery).values(
|
||||
make=m,
|
||||
model=mod,
|
||||
status="pending",
|
||||
attempts=0,
|
||||
vehicle_class="car", # Default value
|
||||
market="GLOBAL", # Default value
|
||||
priority_score=0 # Default value
|
||||
)
|
||||
# Handle conflict on make+model+vehicle_class unique constraint
|
||||
stmt = stmt.on_conflict_do_nothing(index_elements=['make', 'model', 'vehicle_class'])
|
||||
await db.execute(stmt)
|
||||
|
||||
# 3. Arany rekordok (AssetCatalog / vehicle_catalog tábla)
|
||||
# Példa adatok, amik már átmentek a validációs folyamaton.
|
||||
gold_data = [
|
||||
AssetCatalog(
|
||||
make="SUZUKI",
|
||||
model="VITARA",
|
||||
generation="LY (2015-)",
|
||||
fuel_type="petrol",
|
||||
factory_data={"segment": "SUV", "origin": "Hungary"}
|
||||
),
|
||||
AssetCatalog(
|
||||
make="SKODA",
|
||||
model="OCTAVIA",
|
||||
generation="IV (2020-)",
|
||||
fuel_type="diesel",
|
||||
factory_data={"segment": "Sedan/Combi"}
|
||||
)
|
||||
]
|
||||
db.add_all(gold_data)
|
||||
|
||||
# Mentés végrehajtása
|
||||
await db.commit()
|
||||
logger.info("✨ Katalógus és Discovery paraméterek sikeresen rögzítve!")
|
||||
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(f"❌ HIBA a katalógus feltöltésekor: {e}")
|
||||
raise e
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(quick_seed())
|
||||
156
archive/tests_internal/seeds/seed_data.py
Executable file
156
archive/tests_internal/seeds/seed_data.py
Executable file
@@ -0,0 +1,156 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_data.py
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import text, select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User, Person, UserRole
|
||||
from app.models import ServiceProvider, Vote, ModerationStatus, Competition
|
||||
from app.services.social_service import SocialService
|
||||
from app.core.security import get_password_hash
|
||||
|
||||
async def run_simulation():
|
||||
async with AsyncSessionLocal() as db:
|
||||
print("--- 1. TAKARÍTÁS (MB2.0 Séma-tisztítás) ---")
|
||||
# Szigorú sorrend a kényszerek miatt (Cascade)
|
||||
# Ellenőrizzük, mely táblák léteznek
|
||||
tables_to_check = [
|
||||
("identity.users", "users"),
|
||||
("identity.persons", "persons"),
|
||||
("marketplace.service_providers", "service_providers"),
|
||||
("marketplace.votes", "votes"),
|
||||
("system.competitions", "competitions")
|
||||
]
|
||||
|
||||
existing_tables = []
|
||||
for full_name, table_name in tables_to_check:
|
||||
try:
|
||||
result = await db.execute(text(f"SELECT 1 FROM information_schema.tables WHERE table_schema = '{full_name.split('.')[0]}' AND table_name = '{table_name}'"))
|
||||
if result.scalar() == 1:
|
||||
existing_tables.append(full_name)
|
||||
else:
|
||||
print(f"⚠️ {full_name} tábla nem létezik, kihagyva a törlést")
|
||||
except Exception:
|
||||
print(f"⚠️ {full_name} tábla nem létezik, kihagyva a törlést")
|
||||
|
||||
if existing_tables:
|
||||
tables_str = ", ".join(existing_tables)
|
||||
await db.execute(text(f"TRUNCATE {tables_str} RESTART IDENTITY CASCADE"))
|
||||
await db.commit()
|
||||
else:
|
||||
print("ℹ️ Nincs törlendő tábla")
|
||||
|
||||
print("\n--- 2. SZEREPLŐK LÉTREHOZÁSA (Person + User) ---")
|
||||
users_to_create = [
|
||||
("admin@test.com", "Adminisztrátor", UserRole.superadmin),
|
||||
("good@test.com", "Rendes Srác", UserRole.user),
|
||||
("bad@test.com", "Spammer Aladár", UserRole.user),
|
||||
("voter@test.com", "Szavazó Gép", UserRole.user)
|
||||
]
|
||||
|
||||
created_users = {}
|
||||
for email, name, role in users_to_create:
|
||||
name_parts = name.split()
|
||||
first_name = name_parts[0] if name_parts else "Unknown"
|
||||
last_name = name_parts[1] if len(name_parts) > 1 else "User"
|
||||
p = Person(id_uuid=uuid.uuid4(), first_name=first_name, last_name=last_name, is_active=True)
|
||||
db.add(p)
|
||||
await db.flush()
|
||||
|
||||
u = User(
|
||||
email=email,
|
||||
hashed_password=get_password_hash("test1234"),
|
||||
person_id=p.id,
|
||||
role=role,
|
||||
is_active=True
|
||||
)
|
||||
db.add(u)
|
||||
await db.flush()
|
||||
created_users[email] = u
|
||||
|
||||
await db.commit()
|
||||
|
||||
print("\n--- 3. VERSENY INDÍTÁSA ---")
|
||||
# Ellenőrizzük, hogy a competitions tábla létezik-e
|
||||
try:
|
||||
result = await db.execute(text("SELECT 1 FROM information_schema.tables WHERE table_schema = 'system' AND table_name = 'competitions'"))
|
||||
if result.scalar() == 1:
|
||||
race = Competition(
|
||||
name="Téli Szervizvadászat",
|
||||
start_date=datetime.now(timezone.utc) - timedelta(days=1),
|
||||
end_date=datetime.now(timezone.utc) + timedelta(days=30),
|
||||
is_active=True
|
||||
)
|
||||
db.add(race)
|
||||
await db.commit()
|
||||
print("✅ Verseny létrehozva")
|
||||
else:
|
||||
print("⚠️ system.competitions tábla nem létezik, kihagyva a verseny létrehozását")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Hiba a competitions tábla ellenőrzése közben: {e}, kihagyva a verseny létrehozását")
|
||||
|
||||
# Szereplők kiemelése a szimulációhoz
|
||||
good_user = created_users["good@test.com"]
|
||||
bad_user = created_users["bad@test.com"]
|
||||
voter = created_users["voter@test.com"]
|
||||
|
||||
# Ellenőrizzük, hogy a szükséges táblák léteznek-e a szociális szimulációhoz
|
||||
try:
|
||||
result = await db.execute(text("SELECT 1 FROM information_schema.tables WHERE table_schema = 'marketplace' AND table_name = 'service_providers'"))
|
||||
service_providers_exists = result.scalar() == 1
|
||||
|
||||
result = await db.execute(text("SELECT 1 FROM information_schema.tables WHERE table_schema = 'marketplace' AND table_name = 'votes'"))
|
||||
votes_exists = result.scalar() == 1
|
||||
|
||||
if service_providers_exists and votes_exists:
|
||||
print("\n--- 4. SZCENÁRIÓ A: POZITÍV VALIDÁCIÓ ---")
|
||||
# Rendes srác beküld egy szervizt
|
||||
shop = ServiceProvider(
|
||||
name="Profi Gumis",
|
||||
address="Budapest, Váci út 10.",
|
||||
added_by_user_id=good_user.id,
|
||||
status=ModerationStatus.pending
|
||||
)
|
||||
db.add(shop)
|
||||
await db.flush()
|
||||
|
||||
# Szavazatok szimulálása (SocialService használatával a pontszámítás miatt)
|
||||
print(f"Szavazás a '{shop.name}'-re...")
|
||||
# Szimulálunk 5 pozitív szavazatot különböző "virtuális" szavazóktól
|
||||
for _ in range(5):
|
||||
await SocialService.vote_for_provider(db, voter.id, shop.id, 1)
|
||||
|
||||
await db.refresh(good_user)
|
||||
print(f"Jó felhasználó hírneve: {good_user.reputation_score}")
|
||||
|
||||
print("\n--- 5. SZCENÁRIÓ B: AUTO-BAN (SPAM SZŰRÉS) ---")
|
||||
fake_shop = ServiceProvider(
|
||||
name="KAMU SZERVIZ",
|
||||
address="Nincs ilyen utca 0.",
|
||||
added_by_user_id=bad_user.id,
|
||||
status=ModerationStatus.pending
|
||||
)
|
||||
db.add(fake_shop)
|
||||
await db.flush()
|
||||
|
||||
# Leszavazás (Kell -3 a bukáshoz)
|
||||
print("Spam jelentése...")
|
||||
await SocialService.vote_for_provider(db, voter.id, fake_shop.id, -1)
|
||||
await SocialService.vote_for_provider(db, voter.id, fake_shop.id, -1)
|
||||
await SocialService.vote_for_provider(db, voter.id, fake_shop.id, -1)
|
||||
|
||||
await db.refresh(bad_user)
|
||||
print(f"Rossz felhasználó hírneve: {bad_user.reputation_score}")
|
||||
print(f"Fiók státusza: {'KITILTVA' if not bad_user.is_active else 'AKTÍV'}")
|
||||
|
||||
if not bad_user.is_active:
|
||||
print("✅ SIKER: A Sentinel automatikusan leállította a spammert!")
|
||||
else:
|
||||
print("\n⚠️ Marketplace táblák (service_providers, votes) nem léteznek, kihagyva a szociális szimulációt")
|
||||
print("ℹ️ Alap felhasználók sikeresen létrehozva")
|
||||
except Exception as e:
|
||||
print(f"\n⚠️ Hiba a táblák ellenőrzése közben: {e}, kihagyva a szociális szimulációt")
|
||||
print("ℹ️ Alap felhasználók sikeresen létrehozva")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_simulation())
|
||||
62
archive/tests_internal/seeds/seed_economy.py
Normal file
62
archive/tests_internal/seeds/seed_economy.py
Normal file
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Seed script az Economy 1 modulhoz: árfolyam paraméterek beszúrása a system.system_parameters táblába.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from decimal import Decimal
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.system import SystemParameter
|
||||
|
||||
|
||||
async def seed_economy():
|
||||
"""Árfolyam paraméterek beszúrása."""
|
||||
parameters = [
|
||||
{
|
||||
"key": "EXCHANGE_RATE_EUR_HUF",
|
||||
"value": "390.0",
|
||||
"description": "EUR/HUF átváltási árfolyam (1 EUR = X HUF)",
|
||||
"category": "finance",
|
||||
"is_active": True,
|
||||
},
|
||||
{
|
||||
"key": "EXCHANGE_RATE_USDC_HUF",
|
||||
"value": "380.0",
|
||||
"description": "USDC/HUF átváltási árfolyam (1 USDC = X HUF)",
|
||||
"category": "finance",
|
||||
"is_active": True,
|
||||
},
|
||||
]
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
for param in parameters:
|
||||
# Ellenőrizzük, hogy létezik-e már
|
||||
existing = await session.execute(
|
||||
select(SystemParameter).where(SystemParameter.key == param["key"])
|
||||
)
|
||||
existing = existing.scalar_one_or_none()
|
||||
if existing:
|
||||
print(f"⚠️ {param['key']} már létezik, kihagyva.")
|
||||
continue
|
||||
|
||||
new_param = SystemParameter(
|
||||
key=param["key"],
|
||||
value=param["value"],
|
||||
description=param["description"],
|
||||
category=param["category"],
|
||||
is_active=param["is_active"],
|
||||
)
|
||||
session.add(new_param)
|
||||
print(f"✅ {param['key']} beszúrva.")
|
||||
|
||||
await session.commit()
|
||||
print("🎉 Árfolyam paraméterek sikeresen seedelve.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_economy())
|
||||
65
archive/tests_internal/seeds/seed_expertises.py
Executable file
65
archive/tests_internal/seeds/seed_expertises.py
Executable file
@@ -0,0 +1,65 @@
|
||||
import asyncio
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.marketplace.service import ExpertiseTag
|
||||
from sqlalchemy import text
|
||||
|
||||
async def seed_expertises():
|
||||
tags = [
|
||||
# --- ALAPSZOLGÁLTATÁSOK (MECHANICS) ---
|
||||
('OIL_SERVICE', 'Időszakos szerviz / Olajcsere', 'MECHANICS'),
|
||||
('BRAKE_REPAIR', 'Fékrendszer javítás', 'MECHANICS'),
|
||||
('SUSPENSION', 'Futómű javítás és beállítás', 'MECHANICS'),
|
||||
('EXHAUST', 'Kipufogó szerviz', 'MECHANICS'),
|
||||
('CLUTCH', 'Kuplung és kettőstömegű csere', 'MECHANICS'),
|
||||
|
||||
# --- MOTOR ÉS VÁLTÓ (ENGINE_DRIVETRAIN) ---
|
||||
('ENGINE_REBUILD', 'Motorfelújítás', 'ENGINE_DRIVETRAIN'),
|
||||
('TIMING_BELT', 'Vezérlés csere', 'ENGINE_DRIVETRAIN'),
|
||||
('AUTO_GEARBOX', 'Automata váltó javítás/olajcsere', 'ENGINE_DRIVETRAIN'),
|
||||
('TURBO_REPAIR', 'Turbófeltöltő felújítás', 'ENGINE_DRIVETRAIN'),
|
||||
('INJECTOR', 'Dízel injektor / Adagoló javítás', 'ENGINE_DRIVETRAIN'),
|
||||
('DPF_CLEAN', 'DPF / Részecskeszűrő tisztítás', 'ENGINE_DRIVETRAIN'),
|
||||
|
||||
# --- ELEKTRONIKA (ELECTRICAL) ---
|
||||
('DIAGNOSTICS', 'Számítógépes diagnosztika', 'ELECTRICAL'),
|
||||
('AC_REPAIR', 'Klíma javítás és töltés', 'ELECTRICAL'),
|
||||
('BATTERY', 'Akkumulátor szerviz', 'ELECTRICAL'),
|
||||
('HYBRID_EV', 'Hibrid és Elektromos autó szerviz', 'ELECTRICAL'),
|
||||
('CHIP_TUNING', 'Szoftveres optimalizálás / Tuning', 'ELECTRICAL'),
|
||||
('ADAS', 'Vezetéstámogató rendszerek kalibrálása', 'ELECTRICAL'),
|
||||
|
||||
# --- GUMI ÉS KERÉK (TYRES) ---
|
||||
('TYRE_CHANGE', 'Gumiszerelés és centírozás', 'TYRES'),
|
||||
('WHEEL_REPAIR', 'Alufelni javítás / Görgőzés', 'TYRES'),
|
||||
|
||||
# --- KAROSSZÉRIA (BODY) ---
|
||||
('BODY_REPAIR', 'Karosszéria lakatolás', 'BODY'),
|
||||
('PAINTING', 'Autófényezés', 'BODY'),
|
||||
('GLASS_REPAIR', 'Szélvédő javítás és csere', 'BODY'),
|
||||
('PDR', 'Jégkár és horpadásjavítás (PDR)', 'BODY'),
|
||||
|
||||
# --- SEGÉLY ÉS SZÁLLÍTÁS (EMERGENCY) ---
|
||||
('TOWING', 'Autómentés / Vontatás', 'EMERGENCY'),
|
||||
('ROADSIDE_ASSIST', 'Segélyszolgálat / Helyszíni javítás', 'EMERGENCY'),
|
||||
('LOCKSMITH', 'Autózár szerviz / Kulcsmásolás', 'EMERGENCY'),
|
||||
|
||||
# --- EGYÉB JÁRMŰVEK (VEHICLE_TYPES) ---
|
||||
('MOTO_SERVICE', 'Motorkerékpár szerviz', 'VEHICLE_TYPES'),
|
||||
('TRUCK_SERVICE', 'Tehergépjármű szerviz', 'VEHICLE_TYPES'),
|
||||
('AGRI_SERVICE', 'Mezőgazdasági gép szerviz', 'VEHICLE_TYPES'),
|
||||
]
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
print("🌱 Szakmai címkék feltöltése...")
|
||||
for key, name, cat in tags:
|
||||
stmt = text("""
|
||||
INSERT INTO marketplace.expertise_tags (key, name_hu, category, is_official)
|
||||
VALUES (:k, :n, :c, true)
|
||||
ON CONFLICT (key) DO UPDATE SET name_hu = EXCLUDED.name_hu, category = EXCLUDED.category
|
||||
""")
|
||||
await db.execute(stmt, {"k": key, "n": name, "c": cat})
|
||||
await db.commit()
|
||||
print(f"✅ {len(tags)} címke rögzítve.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_expertises())
|
||||
74
archive/tests_internal/seeds/seed_honda.py
Executable file
74
archive/tests_internal/seeds/seed_honda.py
Executable file
@@ -0,0 +1,74 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_honda.py
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import AssetCatalog
|
||||
from app.models.marketplace.staged_data import DiscoveryParameter
|
||||
|
||||
# Logolás beállítása
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Seed: %(message)s')
|
||||
logger = logging.getLogger("Honda-Seeder")
|
||||
|
||||
async def seed_honda():
|
||||
"""
|
||||
Honda specifikus alapozás az MB2.0 MDM (Master Data Management) szerint.
|
||||
Létrehozza a katalógus-vázat és a robot-feladatokat.
|
||||
"""
|
||||
async with AsyncSessionLocal() as db:
|
||||
logger.info("🚀 Honda márka-ökoszisztéma inicializálása...")
|
||||
|
||||
# 1. LOGIKA: Robot Discovery feladatok rögzítése
|
||||
# Ezzel mondjuk meg a Hunter robotnak, hogy keressen rá minden Honda variánsra
|
||||
discovery_tasks = [
|
||||
DiscoveryParameter(make="HONDA", vehicle_class="car", city="BUDAPEST", keyword="repair", is_active=True),
|
||||
DiscoveryParameter(make="HONDA", vehicle_class="motorcycle", city="BUDAPEST", keyword="service", is_active=True)
|
||||
]
|
||||
|
||||
for task in discovery_tasks:
|
||||
# Megnézzük, van-e már ilyen feladat
|
||||
stmt = select(DiscoveryParameter).where(
|
||||
DiscoveryParameter.make == task.make,
|
||||
DiscoveryParameter.vehicle_class == task.vehicle_class
|
||||
)
|
||||
exists = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if not exists:
|
||||
db.add(task)
|
||||
|
||||
# 2. LOGIKA: Népszerű modellek (Arany Rekordok) betöltése
|
||||
# Ezek a "Starter" adatok, amik azonnal elérhetők a felhasználóknak
|
||||
honda_models = [
|
||||
# Személyautók
|
||||
{"model": "CIVIC", "gen": "X (2015-2021)", "class": "car"},
|
||||
{"model": "ACCORD", "gen": "X (2017-)", "class": "car"},
|
||||
{"model": "CR-V", "gen": "V (2016-)", "class": "car"},
|
||||
{"model": "JAZZ", "gen": "IV (2020-)", "class": "car"},
|
||||
# Motorkerékpárok
|
||||
{"model": "CB500X", "gen": "PC64 (2019-)", "class": "motorcycle"},
|
||||
{"model": "AFRICA TWIN", "gen": "CRF1100L", "class": "motorcycle"},
|
||||
{"model": "NC750X", "gen": "RH09 (2021-)", "class": "motorcycle"}
|
||||
]
|
||||
|
||||
for m in honda_models:
|
||||
# Ellenőrizzük az AssetCatalog-ban (MDM tábla)
|
||||
stmt = select(AssetCatalog).where(
|
||||
AssetCatalog.make == "HONDA",
|
||||
AssetCatalog.model == m["model"],
|
||||
AssetCatalog.generation == m["gen"]
|
||||
)
|
||||
exists = (await db.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
if not exists:
|
||||
db.add(AssetCatalog(
|
||||
make="HONDA",
|
||||
model=m["model"],
|
||||
generation=m["gen"],
|
||||
vehicle_class=m["class"],
|
||||
factory_data={"source": "manual_priority_seed"} # MDM metaadat
|
||||
))
|
||||
|
||||
await db.commit()
|
||||
logger.info("✅ Honda (Autó & Motor) katalógus váz sikeresen felépítve!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_honda())
|
||||
79
archive/tests_internal/seeds/seed_system.py
Executable file
79
archive/tests_internal/seeds/seed_system.py
Executable file
@@ -0,0 +1,79 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_system.py
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User, Person, UserRole
|
||||
from app.models.system import SystemParameter
|
||||
# JAVÍTOTT IMPORTOK: A grep alapján szétválasztva
|
||||
from app.models import PointRule, LevelConfig, UserStats
|
||||
from app.models.core_logic import SubscriptionTier
|
||||
from app.core.security import get_password_hash
|
||||
from app.core.config import settings
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Seed: %(message)s')
|
||||
logger = logging.getLogger("System-Seeder")
|
||||
|
||||
async def seed_data():
|
||||
""" Rendszer alapadatok inicializálása a megfelelő modellekből. """
|
||||
async with AsyncSessionLocal() as db:
|
||||
logger.info("🚀 Rendszer-alapozás indítása (MB2.0 Standard)...")
|
||||
|
||||
admin_email = settings.INITIAL_ADMIN_EMAIL
|
||||
admin_password = settings.INITIAL_ADMIN_PASSWORD
|
||||
|
||||
if not admin_email or not admin_password:
|
||||
logger.error("❌ HIBA: Admin hitelesítési adatok hiányoznak!")
|
||||
return
|
||||
|
||||
# 1. Superadmin és Person kapcsolat
|
||||
stmt = select(User).where(User.email == admin_email)
|
||||
admin_exists = (await db.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
if not admin_exists:
|
||||
new_person = Person(
|
||||
first_name="Rendszer",
|
||||
last_name="Adminisztrátor",
|
||||
id_uuid=uuid.uuid4(),
|
||||
is_active=True
|
||||
)
|
||||
db.add(new_person)
|
||||
await db.flush()
|
||||
|
||||
new_admin = User(
|
||||
email=admin_email,
|
||||
hashed_password=get_password_hash(admin_password),
|
||||
role=UserRole.superadmin,
|
||||
is_active=True,
|
||||
person_id=new_person.id
|
||||
)
|
||||
db.add(new_admin)
|
||||
await db.flush()
|
||||
|
||||
# Statisztikai rekord (Gamification)
|
||||
db.add(UserStats(user_id=new_admin.id, total_xp=0, current_level=1))
|
||||
logger.info(f"✅ Superadmin létrehozva: {admin_email}")
|
||||
|
||||
# 2. Rendszerparaméterek (JSONB értékekkel)
|
||||
params = [
|
||||
("SECURITY_MAX_RECORDS_PER_HOUR", {"limit": 50}, "Biztonsági limit"),
|
||||
("VEHICLE_LIMIT", {"default": 5}, "Alapértelmezett jármű limit")
|
||||
]
|
||||
for key, val, desc in params:
|
||||
stmt_p = select(SystemParameter).where(SystemParameter.key == key)
|
||||
if not (await db.execute(stmt_p)).scalar_one_or_none():
|
||||
db.add(SystemParameter(key=key, value=val, description=desc))
|
||||
|
||||
# 3. Gamification Szabályok (gamification.py-ból)
|
||||
rules = [("ASSET_REGISTER", 100), ("ASSET_REVIEW", 75)]
|
||||
for key, pts in rules:
|
||||
stmt_r = select(PointRule).where(PointRule.action_key == key)
|
||||
if not (await db.execute(stmt_r)).scalar_one_or_none():
|
||||
db.add(PointRule(action_key=key, points=pts))
|
||||
|
||||
await db.commit()
|
||||
logger.info("✨ Rendszer alapadatok rögzítve.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_data())
|
||||
123
archive/tests_internal/seeds/seed_tco_categories.py
Normal file
123
archive/tests_internal/seeds/seed_tco_categories.py
Normal file
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
TCO (Total Cost of Ownership) alap költségkategóriák seedelése.
|
||||
Rendszerszintű kategóriák (is_system=True) amelyek nem törölhetők.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
# A projekt gyökérből importáljuk a database modult
|
||||
sys.path.insert(0, '/opt/docker/dev/service_finder/backend')
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.vehicle import CostCategory
|
||||
|
||||
|
||||
# A 10 alap TCO kategória definíciója
|
||||
SYSTEM_CATEGORIES = [
|
||||
{
|
||||
"code": "FUEL",
|
||||
"name": "Üzemanyag / Töltés",
|
||||
"description": "Benzin, dízel, elektromos töltés, LPG, hidrogén"
|
||||
},
|
||||
{
|
||||
"code": "MAINTENANCE",
|
||||
"name": "Szerviz & Karbantartás",
|
||||
"description": "Olajcsere, szűrők, fékbetét, futómű, egyéb szerviz munkák"
|
||||
},
|
||||
{
|
||||
"code": "TIRES",
|
||||
"name": "Gumiabroncsok",
|
||||
"description": "Nyári/téli gumik, felni, kiegyensúlyozás, gumicsere"
|
||||
},
|
||||
{
|
||||
"code": "INSURANCE",
|
||||
"name": "Biztosítás",
|
||||
"description": "KASCO, kötelező gépjármű-felelősségbiztosítás, casco, utasbiztosítás"
|
||||
},
|
||||
{
|
||||
"code": "TAX",
|
||||
"name": "Adók",
|
||||
"description": "Gépjárműadó, forgalmi adó, közlekedési adó"
|
||||
},
|
||||
{
|
||||
"code": "FEES",
|
||||
"name": "Útdíj & Parkolás",
|
||||
"description": "Autópálya matrica, parkolási díjak, városi belépési díjak"
|
||||
},
|
||||
{
|
||||
"code": "ADMIN",
|
||||
"name": "Hatósági díjak",
|
||||
"description": "Műszaki vizsga, forgalmi engedély, okmányok, adminisztratív költségek"
|
||||
},
|
||||
{
|
||||
"code": "FINANCE",
|
||||
"name": "Finanszírozás",
|
||||
"description": "Lízing díj, hiteltörlesztés, kamatok, banki költségek"
|
||||
},
|
||||
{
|
||||
"code": "CLEANING",
|
||||
"name": "Ápolás & Kozmetika",
|
||||
"description": "Autómosás, polírozás, belső tisztítás, festékvédelem"
|
||||
},
|
||||
{
|
||||
"code": "OTHER",
|
||||
"name": "Egyéb",
|
||||
"description": "Egyéb, nem besorolható költségek"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def seed_tco_categories():
|
||||
"""
|
||||
Törli a meglévő kategóriákat és beszúrja a 10 rendszerszintű TCO kategóriát.
|
||||
"""
|
||||
print("🚀 TCO költségkategóriák seedelése...")
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# 1. Tábla ürítése (TRUNCATE) - csak a seed kategóriák, ne érintse a felhasználói kategóriákat?
|
||||
# Mivel most csak rendszerszintűek vannak, töröljük az összeset
|
||||
print(" ↳ Tábla ürítése (TRUNCATE vehicle.cost_categories)...")
|
||||
await session.execute(text("TRUNCATE TABLE vehicle.cost_categories RESTART IDENTITY CASCADE"))
|
||||
await session.commit()
|
||||
|
||||
# 2. Kategóriák beszúrása
|
||||
inserted = 0
|
||||
for cat_data in SYSTEM_CATEGORIES:
|
||||
category = CostCategory(
|
||||
code=cat_data["code"],
|
||||
name=cat_data["name"],
|
||||
description=cat_data["description"],
|
||||
is_system=True,
|
||||
parent_id=None # Jelenleg nincs hierarchia, később bővíthető
|
||||
)
|
||||
session.add(category)
|
||||
inserted += 1
|
||||
|
||||
await session.commit()
|
||||
print(f" ✅ {inserted} rendszerszintű kategória beszúrva.")
|
||||
|
||||
# 3. Ellenőrzés
|
||||
result = await session.execute(text("SELECT COUNT(*) FROM vehicle.cost_categories"))
|
||||
count = result.scalar()
|
||||
print(f" 📊 vehicle.cost_categories táblában jelenleg {count} sor van.")
|
||||
|
||||
# Listázás
|
||||
result = await session.execute(text("SELECT code, name FROM vehicle.cost_categories ORDER BY code"))
|
||||
rows = result.fetchall()
|
||||
print(" 📋 Kategóriák listája:")
|
||||
for code, name in rows:
|
||||
print(f" - {code}: {name}")
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
print(f" ❌ Hiba történt: {e}")
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_tco_categories())
|
||||
print("🎉 TCO kategória seedelés sikeresen befejeződött.")
|
||||
120
archive/tests_internal/seeds/seed_test_scenario.py
Executable file
120
archive/tests_internal/seeds/seed_test_scenario.py
Executable file
@@ -0,0 +1,120 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/seeds/seed_test_scenario.py
|
||||
import asyncio
|
||||
import uuid
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User
|
||||
from app.models.marketplace.organization import Organization, OrganizationMember, OrgType
|
||||
from app.models import (
|
||||
Asset, AssetCatalog, AssetTelemetry,
|
||||
AssetFinancials, AssetCost
|
||||
)
|
||||
|
||||
# Sentinel naplózás
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Scenario: %(message)s')
|
||||
logger = logging.getLogger("Test-Scenario")
|
||||
|
||||
async def seed_test_scenario():
|
||||
async with AsyncSessionLocal() as db:
|
||||
logger.info("🚀 MB2.0 Teszt ökoszisztéma felépítése indul...")
|
||||
|
||||
# 1. LOGIKA: Admin (Superuser) lekérése az identity sémából
|
||||
res = await db.execute(select(User).where(User.is_active == True))
|
||||
admin = res.scalars().first()
|
||||
|
||||
if not admin:
|
||||
logger.error("❌ Hiba: Nincs aktív felhasználó a rendszerben. Futtasd a seed_system.py-t!")
|
||||
return
|
||||
|
||||
# 2. LOGIKA: Szervezeti struktúra felállítása
|
||||
# Privát garázs
|
||||
private_org = Organization(
|
||||
name="Kincses Privát",
|
||||
full_name="Kincses Magánflotta és Garázs",
|
||||
org_type=OrgType.individual,
|
||||
owner_id=admin.id,
|
||||
folder_slug="kincses-privat-vault"
|
||||
)
|
||||
# Üzleti flotta
|
||||
company_org = Organization(
|
||||
name="ProfiBot Fleet",
|
||||
full_name="ProfiBot Software Solutions Kft.",
|
||||
org_type=OrgType.business,
|
||||
owner_id=admin.id,
|
||||
folder_slug="profibot-fleet-vault"
|
||||
)
|
||||
# Szolgáltatók (Szerviz és Üzemanyag)
|
||||
service_org = Organization(
|
||||
name="Mester Szerviz",
|
||||
org_type=OrgType.service,
|
||||
owner_id=admin.id,
|
||||
is_active=True
|
||||
)
|
||||
|
||||
db.add_all([private_org, company_org, service_org])
|
||||
await db.flush()
|
||||
|
||||
# Tagsági viszonyok rögzítése
|
||||
db.add(OrganizationMember(user_id=admin.id, organization_id=company_org.id, role="owner"))
|
||||
|
||||
# 3. LOGIKA: Tesla Model 3 - Digitális Iker (Digital Twin)
|
||||
# Előbb a katalógus (Gold Data)
|
||||
catalog = AssetCatalog(
|
||||
make="TESLA", model="MODEL 3", generation="Long Range (2021-)",
|
||||
fuel_type="electric",
|
||||
factory_data={
|
||||
"battery": "75 kWh", "power_kw": 366,
|
||||
"tire_size": "235/45 R18", "ac_charge": "11kW"
|
||||
}
|
||||
)
|
||||
db.add(catalog)
|
||||
await db.flush()
|
||||
|
||||
# Majd a konkrét jármű (Asset)
|
||||
vehicle = Asset(
|
||||
vin=f"5YJ3E1EB8LF{uuid.uuid4().hex[:6].upper()}",
|
||||
license_plate="TES-777-EV",
|
||||
name="Céges Tesla",
|
||||
year_of_manufacture=2021,
|
||||
catalog_id=catalog.id,
|
||||
owner_org_id=company_org.id,
|
||||
status="active"
|
||||
)
|
||||
db.add(vehicle)
|
||||
await db.flush()
|
||||
|
||||
# Telemetria és Pénzügyi alapok
|
||||
db.add(AssetTelemetry(asset_id=vehicle.id, current_mileage=45200, vqi_score=100.0))
|
||||
db.add(AssetFinancials(asset_id=vehicle.id, acquisition_price=18500000, currency="HUF"))
|
||||
|
||||
# 4. LOGIKA: A 9 költségtípus szimulálása
|
||||
costs_data = [
|
||||
("FUEL", 12500, "Supercharger töltés"),
|
||||
("MAINTENANCE", 85000, "Pollenszűrő és átvizsgálás"),
|
||||
("TIRES", 280000, "Téli gumi szett"),
|
||||
("INSURANCE", 32000, "Havi CASCO"),
|
||||
("TAX", 15000, "Cégautóadó (szimulált)"),
|
||||
("TOLL", 6500, "Éves matrica"),
|
||||
("CLEANING", 4500, "Külső-belső takarítás"),
|
||||
("PARKING", 1200, "Belvárosi zóna"),
|
||||
("OTHER", 2500, "Szélvédőmosó folyadék")
|
||||
]
|
||||
|
||||
for c_type, amount, desc in costs_data:
|
||||
db.add(AssetCost(
|
||||
asset_id=vehicle.id,
|
||||
organization_id=company_org.id,
|
||||
cost_type=c_type,
|
||||
amount=amount,
|
||||
currency="HUF",
|
||||
date=datetime.now(timezone.utc) - timedelta(days=2),
|
||||
specifications={"description": desc}
|
||||
))
|
||||
|
||||
await db.commit()
|
||||
logger.info("✅ Siker! A teljes flotta-ökoszisztéma üzemkész.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(seed_test_scenario())
|
||||
77
archive/tests_internal/test_analytics_api.py
Normal file
77
archive/tests_internal/test_analytics_api.py
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Operational test for Analytics API endpoint /api/v1/analytics/{vehicle_id}/summary
|
||||
Verifies that the endpoint is correctly registered, accepts UUID vehicle_id,
|
||||
and returns appropriate HTTP status (not 500 internal server error).
|
||||
Uses dev_bypass_active token to bypass authentication (requires DEBUG=True).
|
||||
"""
|
||||
import sys
|
||||
import asyncio
|
||||
import httpx
|
||||
import uuid
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_BASE = "http://localhost:8000"
|
||||
DEV_TOKEN = "dev_bypass_active"
|
||||
|
||||
async def test_analytics_summary():
|
||||
"""Test that the endpoint is reachable and handles UUID parameter."""
|
||||
# Generate a random UUID (vehicle likely does not exist)
|
||||
vehicle_id = uuid.uuid4()
|
||||
url = f"{API_BASE}/api/v1/analytics/{vehicle_id}/summary"
|
||||
headers = {"Authorization": f"Bearer {DEV_TOKEN}"}
|
||||
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
try:
|
||||
resp = await client.get(url, headers=headers)
|
||||
status = resp.status_code
|
||||
body = resp.text
|
||||
logger.info(f"Response status: {status}")
|
||||
logger.debug(f"Response body: {body}")
|
||||
|
||||
# If endpoint missing, we'd get 404 Not Found (from router).
|
||||
# However, with UUID parameter, the router is matched, so 404 is vehicle not found.
|
||||
# Distinguish by checking if the response indicates router-level 404 (maybe generic).
|
||||
# For simplicity, we assume any 404 means vehicle not found, which is OK.
|
||||
# The critical check: no 500 Internal Server Error (mapper or runtime errors).
|
||||
if status == 500:
|
||||
raise AssertionError(f"Internal server error: {body}")
|
||||
|
||||
# If we get 200, validate JSON structure (optional, but we don't have data).
|
||||
if status == 200:
|
||||
data = resp.json()
|
||||
required_keys = {"vehicle_id", "user_tco", "lifetime_tco", "benchmark_tco", "stats"}
|
||||
missing = required_keys - set(data.keys())
|
||||
if missing:
|
||||
raise AssertionError(f"Missing keys in response: {missing}")
|
||||
for key in ["user_tco", "lifetime_tco", "benchmark_tco"]:
|
||||
if not isinstance(data[key], list):
|
||||
raise AssertionError(f"{key} is not a list")
|
||||
logger.info("✅ Analytics endpoint works and returns expected structure.")
|
||||
return True
|
||||
|
||||
# Any other status (404, 422, 403, 401) indicates the endpoint is reachable
|
||||
# and the request was processed (no router error).
|
||||
logger.info(f"Endpoint responded with {status} (expected, vehicle not found or access denied).")
|
||||
return True
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"HTTP client error: {e}")
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
logger.error("Request timeout")
|
||||
raise
|
||||
|
||||
async def main():
|
||||
try:
|
||||
await test_analytics_summary()
|
||||
print("\n✅ Analytics API test passed (endpoint is reachable and accepts UUID).")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"\n❌ Analytics API test failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
31
archive/tests_internal/test_functional.py
Executable file
31
archive/tests_internal/test_functional.py
Executable file
@@ -0,0 +1,31 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/test_functional.py
|
||||
"""
|
||||
CÉL: Éles funkcionális teszt a bejelentkezési folyamathoz.
|
||||
"""
|
||||
import asyncio
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User
|
||||
from app.services.auth_service import AuthService
|
||||
|
||||
async def test_login_flow():
|
||||
print("\n--- 🔑 FUNKCIONÁLIS LOGIN TESZT ---")
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 1. Keressünk egy teszt felhasználót
|
||||
result = await db.execute(select(User).limit(1))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if not user:
|
||||
print("❌ HIBA: Nincs felhasználó az adatbázisban. Futtass egy seeder-t!")
|
||||
return
|
||||
|
||||
print(f"👤 Tesztelés felhasználóval: {user.email}")
|
||||
|
||||
# 2. Próbáljunk meg egy 'hitelesítést' (jelszó ellenőrzés nélkül a DB szinten)
|
||||
if user.is_active:
|
||||
print(f"✅ SIKER: A(z) {user.email} fiók aktív és elérhető.")
|
||||
else:
|
||||
print(f"⚠️ FIGYELEM: A felhasználó létezik, de inaktív.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_login_flow())
|
||||
91
archive/tests_internal/test_gamification_flow.py
Executable file
91
archive/tests_internal/test_gamification_flow.py
Executable file
@@ -0,0 +1,91 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/tests_internal/test_gamification_flow.py
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from sqlalchemy import select
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Környezeti változók betöltése
|
||||
load_dotenv()
|
||||
|
||||
# MB2.0 Importok
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.identity import User
|
||||
from app.models import UserStats, PointsLedger
|
||||
from app.services.social_service import SocialService
|
||||
from app.schemas.social import ServiceProviderCreate
|
||||
|
||||
# Naplózás beállítása
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] Sentinel-Test: %(message)s')
|
||||
logger = logging.getLogger("Gamification-Test")
|
||||
|
||||
async def run_test():
|
||||
logger.info("🚀 Gamifikációs integrációs folyamat tesztelése...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
# 1. LOGIKA: Teszt felhasználó lekérése az identity sémából
|
||||
result = await db.execute(select(User).limit(1))
|
||||
user = result.scalars().first()
|
||||
|
||||
if not user:
|
||||
logger.error("❌ Hiba: Nincs felhasználó az adatbázisban. Futtasd a seed_system.py-t!")
|
||||
return
|
||||
|
||||
logger.info(f"👤 Aktív teszt alany: {user.email}")
|
||||
|
||||
# 2. LOGIKA: Új szolgáltató rögzítése (Trigger az XP szerzéshez)
|
||||
# A SocialService.create_service_provider automatikusan hívja a GamificationService-t
|
||||
unique_id = os.urandom(2).hex()
|
||||
test_provider = ServiceProviderCreate(
|
||||
name=f"Robot Szerviz {unique_id}",
|
||||
address="Alchemist utca 12.",
|
||||
category="service"
|
||||
)
|
||||
|
||||
logger.info(f"🛠️ Esemény kiváltása: '{test_provider.name}' rögzítése...")
|
||||
new_provider = await SocialService.create_service_provider(db, test_provider, user.id)
|
||||
|
||||
# Commit kényszerítése, hogy a háttérfolyamatok rögzüljenek
|
||||
await db.commit()
|
||||
logger.info(f"✅ Szolgáltató elfogadva (ID: {new_provider.id})")
|
||||
|
||||
# 3. LOGIKA: Eredmények ellenőrzése a Ledgerben (Főkönyv)
|
||||
# Újra lekérjük a statisztikákat a commit után
|
||||
stats_res = await db.execute(select(UserStats).where(UserStats.user_id == user.id))
|
||||
stats = stats_res.scalar_one_or_none()
|
||||
|
||||
ledger_res = await db.execute(
|
||||
select(PointsLedger)
|
||||
.where(PointsLedger.user_id == user.id)
|
||||
.order_by(PointsLedger.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last_entry = ledger_res.scalars().first()
|
||||
|
||||
print("\n" + "═"*40)
|
||||
print("📊 INTEGRÁCIÓS JELENTÉS:")
|
||||
if stats:
|
||||
print(f"🏆 Aktuális XP: {stats.total_xp}")
|
||||
print(f"📈 Szint: {stats.current_level}")
|
||||
else:
|
||||
print("⚠️ UserStats rekord nem található!")
|
||||
|
||||
if last_entry:
|
||||
print(f"📝 Tranzakció oka: {last_entry.reason}")
|
||||
print(f"💰 XP változás: +{last_entry.points_change}")
|
||||
print("═"*40 + "\n")
|
||||
|
||||
if stats and stats.total_xp > 0:
|
||||
logger.info("✅ SIKER: A gamifikációs lánc éles és működik!")
|
||||
else:
|
||||
logger.warning("❌ HIBA: A pontszámítás nem történt meg.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💥 Kritikus hiba a teszt közben: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_test())
|
||||
33
archive/tests_internal/test_postgis.py
Executable file
33
archive/tests_internal/test_postgis.py
Executable file
@@ -0,0 +1,33 @@
|
||||
# app/tests_internal/test_postgis.py
|
||||
import asyncio
|
||||
from sqlalchemy import text
|
||||
from app.db.session import AsyncSessionLocal
|
||||
|
||||
async def test_geo_logic():
|
||||
"""
|
||||
THOUGHT PROCESS:
|
||||
Ellenőrizni kell, hogy a PostgreSQL-ben a 'fleet.branches' tábla 'location' oszlopa
|
||||
valóban GEOGRAPHY típusú-e, és az ST_Distance függvény működik-e.
|
||||
Ha ez elbukik, a 'search.py' nem fog eredményt adni.
|
||||
"""
|
||||
print("🌍 PostGIS távolságszámítás tesztelése...")
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# Egy teszt pont (Budapest központ) és egy körzet lekérdezése
|
||||
query = text("""
|
||||
SELECT id, name,
|
||||
ST_Distance(location, ST_SetSRID(ST_MakePoint(19.0402, 47.4979), 4326)::geography) / 1000 as distance_km
|
||||
FROM fleet.branches
|
||||
LIMIT 1
|
||||
""")
|
||||
result = await db.execute(query)
|
||||
row = result.fetchone()
|
||||
if row:
|
||||
print(f"✅ SIKER: Találtunk egy ágat ({row.name}) {row.distance_km:.2f} km távolságra.")
|
||||
else:
|
||||
print("⚠️ FIGYELEM: A lekérdezés lefutott, de nincsenek adatok a fleet.branches táblában.")
|
||||
except Exception as e:
|
||||
print(f"❌ HIBA: A PostGIS lekérdezés elbukott. Oka: {str(e)}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_geo_logic())
|
||||
340
archive/tests_internal/verify_financial_truth.py
Normal file
340
archive/tests_internal/verify_financial_truth.py
Normal file
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Financial Truth Verification - Epic 3 Pénzügyi Motor "Végső Boss" teszt.
|
||||
|
||||
Ez a script a Financial Orchestrator matematikai hibátlanságát teszteli,
|
||||
különös tekintettel a double-entry integritásra és a vetésforgó logikára.
|
||||
|
||||
FIGYELEM: A teszt NEM módosítja tartósan az éles adatbázist!
|
||||
Minden adatváltozás egy tranzakcióban történik, amely a végén rollback-el.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
from decimal import Decimal
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
# Add backend directory to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
|
||||
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import select, func, text
|
||||
|
||||
from app.database import Base
|
||||
from app.models.identity import User, Person, Wallet
|
||||
from app.models.marketplace.finance import Issuer, IssuerType
|
||||
from app.models import WalletType
|
||||
from app.models import FinancialLedger, LedgerEntryType
|
||||
from app.services.financial_orchestrator import FinancialOrchestrator
|
||||
from app.core.config import settings
|
||||
|
||||
# Database connection - use the same as the app
|
||||
DATABASE_URL = settings.DATABASE_URL.replace("postgresql://", "postgresql+asyncpg://")
|
||||
engine = create_async_engine(DATABASE_URL, echo=False)
|
||||
AsyncSessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
|
||||
class FinancialTruthTest:
|
||||
def __init__(self):
|
||||
self.session = None
|
||||
self.test_user = None
|
||||
self.test_wallet = None
|
||||
self.ev_issuer = None
|
||||
self.kft_issuer = None
|
||||
self.orchestrator = FinancialOrchestrator()
|
||||
self.created_ledgers = []
|
||||
self.total_amount = Decimal('0')
|
||||
# Generate unique timestamp for this test run to avoid duplicate tax IDs
|
||||
self.test_timestamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
|
||||
|
||||
async def setup(self):
|
||||
"""Test adatok létrehozása egy tranzakción belül."""
|
||||
print("=== FINANCIAL TRUTH VERIFICATION TEST ===")
|
||||
print("1. Teszt adatok előkészítése (tranzakción belül)...")
|
||||
|
||||
self.session = AsyncSessionLocal()
|
||||
|
||||
# Tranzakció indítása (nested transaction a rollback-hez)
|
||||
await self.session.begin_nested()
|
||||
|
||||
# Meglévő aktív számlakiállítók inaktiválása, hogy a teszt saját issuereit használja
|
||||
from sqlalchemy import update
|
||||
from app.models.marketplace.finance import Issuer
|
||||
stmt = update(Issuer).where(Issuer.is_active == True).values(is_active=False)
|
||||
await self.session.execute(stmt)
|
||||
await self.session.flush()
|
||||
|
||||
# Teszt User és Person létrehozása
|
||||
person = Person(
|
||||
first_name="Test",
|
||||
last_name="User",
|
||||
phone="+36123456789",
|
||||
is_active=True
|
||||
)
|
||||
self.session.add(person)
|
||||
await self.session.flush()
|
||||
|
||||
self.test_user = User(
|
||||
person_id=person.id,
|
||||
email=f"test_{uuid4().hex[:8]}@example.com",
|
||||
hashed_password="dummyhash",
|
||||
is_active=True
|
||||
)
|
||||
self.session.add(self.test_user)
|
||||
await self.session.flush()
|
||||
|
||||
# Wallet létrehozása a user számára
|
||||
self.test_wallet = Wallet(
|
||||
user_id=self.test_user.id,
|
||||
earned_credits=Decimal('1000000'), # Nagy kezdő egyenleg a teszteléshez
|
||||
purchased_credits=Decimal('0'),
|
||||
service_coins=Decimal('0'),
|
||||
currency="HUF"
|
||||
)
|
||||
self.session.add(self.test_wallet)
|
||||
await self.session.flush()
|
||||
|
||||
# EV típusú Issuer létrehozása alacsony revenue_limit-tel
|
||||
self.ev_issuer = Issuer(
|
||||
type=IssuerType.EV,
|
||||
name="Teszt EV Kft.",
|
||||
tax_id=f"12345678-1-42-{self.test_timestamp}", # Unique tax ID with timestamp
|
||||
revenue_limit=Decimal('50000'), # Csak 50,000 HUF keret
|
||||
current_revenue=Decimal('0'),
|
||||
is_active=True
|
||||
)
|
||||
self.session.add(self.ev_issuer)
|
||||
|
||||
# KFT típusú Issuer létrehozása magas limitel
|
||||
self.kft_issuer = Issuer(
|
||||
type=IssuerType.KFT,
|
||||
name="Teszt KFT Zrt.",
|
||||
tax_id=f"87654321-2-42-{self.test_timestamp}", # Unique tax ID with timestamp
|
||||
revenue_limit=Decimal('10000000'),
|
||||
current_revenue=Decimal('0'),
|
||||
is_active=True
|
||||
)
|
||||
self.session.add(self.kft_issuer)
|
||||
|
||||
await self.session.flush()
|
||||
|
||||
print(f" Teszt User ID: {self.test_user.id}")
|
||||
print(f" Wallet ID: {self.test_wallet.id}, Earned Credits: {self.test_wallet.earned_credits}")
|
||||
print(f" EV Issuer ID: {self.ev_issuer.id}, Revenue Limit: {self.ev_issuer.revenue_limit}")
|
||||
print(f" KFT Issuer ID: {self.kft_issuer.id}, Revenue Limit: {self.kft_issuer.revenue_limit}")
|
||||
|
||||
async def run_payment_cycle(self, num_payments=10, amount_per_payment=Decimal('15000')):
|
||||
"""Több fizetés szimulálása a vetésforgó tesztelésére."""
|
||||
print(f"\n2. {num_payments} fizetés szimulálása (összeg: {amount_per_payment} HUF)...")
|
||||
|
||||
ev_used = 0
|
||||
kft_used = 0
|
||||
|
||||
for i in range(1, num_payments + 1):
|
||||
print(f" Fizetés {i}/{num_payments}...")
|
||||
try:
|
||||
result = await self.orchestrator.process_payment(
|
||||
db=self.session,
|
||||
user_id=self.test_user.id,
|
||||
amount=amount_per_payment,
|
||||
wallet_type=WalletType.EARNED,
|
||||
description=f"Teszt fizetés #{i}",
|
||||
is_company=False # Nem cég, így először EV-t választ
|
||||
)
|
||||
|
||||
issuer_id = result.get('issuer_id')
|
||||
issuer_type = result.get('issuer_type')
|
||||
print(f" -> issuer_id={issuer_id}, issuer_type={issuer_type}, ev_id={self.ev_issuer.id}, kft_id={self.kft_issuer.id}")
|
||||
if issuer_id == self.ev_issuer.id:
|
||||
ev_used += 1
|
||||
print(f" -> EV számlakiállító használva")
|
||||
elif issuer_id == self.kft_issuer.id:
|
||||
kft_used += 1
|
||||
print(f" -> KFT számlakiállító használva (vetésforgó!)")
|
||||
else:
|
||||
print(f" -> HIBA: Ismeretlen issuer_id={issuer_id}")
|
||||
|
||||
self.total_amount += amount_per_payment
|
||||
self.created_ledgers.append(result.get('ledger_id'))
|
||||
|
||||
except Exception as e:
|
||||
print(f" HIBA: {e}")
|
||||
raise
|
||||
|
||||
print(f" Összesítés: EV használva: {ev_used}, KFT használva: {kft_used}")
|
||||
return ev_used, kft_used
|
||||
|
||||
async def verify_double_entry(self):
|
||||
"""Double-entry integritás ellenőrzése: Ledger összegek vs Wallet egyenleg."""
|
||||
print("\n3. Double-Entry Integritás Ellenőrzése...")
|
||||
|
||||
# Összes létrehozott ledger bejegyzés összegének kiszámítása
|
||||
ledger_sum = Decimal('0')
|
||||
for ledger_id in self.created_ledgers:
|
||||
stmt = select(FinancialLedger).where(FinancialLedger.id == ledger_id)
|
||||
result = await self.session.execute(stmt)
|
||||
ledger = result.scalar_one()
|
||||
ledger_sum += ledger.amount
|
||||
|
||||
# Wallet aktuális egyenlegének lekérdezése
|
||||
stmt = select(Wallet).where(Wallet.id == self.test_wallet.id)
|
||||
result = await self.session.execute(stmt)
|
||||
wallet = result.scalar_one()
|
||||
# Összesített egyenleg: earned_credits + purchased_credits + service_coins
|
||||
# Convert all to Decimal for consistent arithmetic
|
||||
earned = Decimal(str(wallet.earned_credits))
|
||||
purchased = Decimal(str(wallet.purchased_credits))
|
||||
service = Decimal(str(wallet.service_coins))
|
||||
wallet_balance = earned + purchased + service
|
||||
|
||||
# Kezdeti egyenleg (1000000) mínusz a kifizetett összeg
|
||||
expected_balance = Decimal('1000000') - self.total_amount
|
||||
|
||||
print(f" Összes ledger tranzakció összege: {ledger_sum} HUF")
|
||||
print(f" Wallet aktuális egyenlege: {wallet_balance} HUF (earned: {earned}, purchased: {purchased}, service: {service})")
|
||||
print(f" Elvárt egyenleg (kezdeti - összes): {expected_balance} HUF")
|
||||
|
||||
# ASSERT 1: Ledger összeg megegyezik a teljes összeggel
|
||||
assert ledger_sum == self.total_amount, \
|
||||
f"Ledger összeg ({ledger_sum}) nem egyezik a teljes összeggel ({self.total_amount})"
|
||||
|
||||
# ASSERT 2: Wallet egyenleg helyes
|
||||
assert wallet_balance == expected_balance, \
|
||||
f"Wallet egyenleg ({wallet_balance}) nem egyezik az elvárt értékkel ({expected_balance})"
|
||||
|
||||
print(" ✅ Double-entry integritás OK: Ledger összegek és Wallet egyenleg konzisztens.")
|
||||
|
||||
async def verify_crop_rotation(self, ev_used, kft_used):
|
||||
"""Vetésforgó logika ellenőrzése: EV keret betelése után KFT-re váltás."""
|
||||
print("\n4. Vetésforgó Logika Ellenőrzése...")
|
||||
|
||||
# EV revenue limit: 50000
|
||||
# Egy fizetés összege: 15000
|
||||
# EV maximum 3 fizetést tud kezelni (3 * 15000 = 45000 < 50000)
|
||||
# A negyedik fizetésnél már túllépné a limitet, így KFT-nek kell váltania
|
||||
|
||||
expected_ev_max = 3 # 3 fizetés még belefér
|
||||
expected_kft_min = 1 # legalább 1 fizetés KFT-vel kell legyen (ha több mint 3 fizetés)
|
||||
|
||||
print(f" EV használva: {ev_used}, KFT használva: {kft_used}")
|
||||
print(f" Elvárás: EV ≤ {expected_ev_max}, KFT ≥ {expected_kft_min}")
|
||||
|
||||
# ASSERT 3: EV nem lépheti túl a limitjét
|
||||
assert ev_used <= expected_ev_max, \
|
||||
f"Túl sok EV használat ({ev_used}) a revenue limit ({self.ev_issuer.revenue_limit}) mellett"
|
||||
|
||||
# ASSERT 4: Ha több fizetés van, mint ami belefér az EV-be, akkor KFT-t kell használni
|
||||
if ev_used == expected_ev_max:
|
||||
assert kft_used >= expected_kft_min, \
|
||||
f"EV limit betelt, de KFT nem lett használva (ev={ev_used}, kft={kft_used})"
|
||||
|
||||
# Ellenőrizzük az aktuális current_revenue értékeket
|
||||
await self.session.refresh(self.ev_issuer)
|
||||
await self.session.refresh(self.kft_issuer)
|
||||
|
||||
print(f" EV aktuális bevétel: {self.ev_issuer.current_revenue}")
|
||||
print(f" KFT aktuális bevétel: {self.kft_issuer.current_revenue}")
|
||||
|
||||
# ASSERT 5: EV current_revenue nem haladhatja meg a limitet
|
||||
assert self.ev_issuer.current_revenue <= self.ev_issuer.revenue_limit, \
|
||||
f"EV current_revenue ({self.ev_issuer.current_revenue}) > limit ({self.ev_issuer.revenue_limit})"
|
||||
|
||||
print(" ✅ Vetésforgó logika OK: EV -> KFT váltás a limit betöltésekor.")
|
||||
|
||||
async def generate_report(self):
|
||||
"""Részletes riport generálása a teszt eredményeiről."""
|
||||
print("\n" + "="*60)
|
||||
print("FINANCIAL TRUTH VERIFICATION - TESZT EREDMÉNY")
|
||||
print("="*60)
|
||||
|
||||
# Ledger statisztikák
|
||||
stmt = select(func.count(FinancialLedger.id)).where(
|
||||
FinancialLedger.id.in_(self.created_ledgers)
|
||||
)
|
||||
result = await self.session.execute(stmt)
|
||||
ledger_count = result.scalar()
|
||||
|
||||
# Issuer statisztikák
|
||||
await self.session.refresh(self.ev_issuer)
|
||||
await self.session.refresh(self.kft_issuer)
|
||||
|
||||
print(f"Összes tranzakció: {ledger_count}")
|
||||
print(f"Teljes összeg: {self.total_amount} HUF")
|
||||
print(f"EV számlakiállító:")
|
||||
print(f" - ID: {self.ev_issuer.id}")
|
||||
print(f" - Aktuális bevétel: {self.ev_issuer.current_revenue} HUF")
|
||||
print(f" - Revenue limit: {self.ev_issuer.revenue_limit} HUF")
|
||||
print(f" - Felhasznált kapacitás: {self.ev_issuer.current_revenue / self.ev_issuer.revenue_limit * 100:.1f}%")
|
||||
print(f"KFT számlakiállító:")
|
||||
print(f" - ID: {self.kft_issuer.id}")
|
||||
print(f" - Aktuális bevétel: {self.kft_issuer.current_revenue} HUF")
|
||||
print(f" - Revenue limit: {self.kft_issuer.revenue_limit} HUF")
|
||||
|
||||
# Wallet állapot
|
||||
await self.session.refresh(self.test_wallet)
|
||||
print(f"Teszt Wallet:")
|
||||
print(f" - ID: {self.test_wallet.id}")
|
||||
# Összesített egyenleg: earned_credits + purchased_credits + service_coins
|
||||
total_balance = self.test_wallet.earned_credits + self.test_wallet.purchased_credits + self.test_wallet.service_coins
|
||||
print(f" - Egyenleg: {total_balance} HUF (earned: {self.test_wallet.earned_credits}, purchased: {self.test_wallet.purchased_credits}, service: {self.test_wallet.service_coins})")
|
||||
print(f" - Kezdeti egyenleg: 1000000 HUF")
|
||||
print(f" - Költség: {self.total_amount} HUF")
|
||||
|
||||
print("\n✅ ÖSSZEFOGLALÓ: A Financial Orchestrator matematikailag hibátlan.")
|
||||
print(" - Double-entry integritás: OK")
|
||||
print(" - Vetésforgó logika: OK")
|
||||
print(" - Tranzakció atomi végrehajtás: OK")
|
||||
print("="*60)
|
||||
|
||||
async def cleanup(self):
|
||||
"""Teszt adatok törlése rollback-kel."""
|
||||
print("\n5. Takarítás: tranzakció rollback (dev adatbázis érintetlen)...")
|
||||
# Mivel nested transaction van, rollback-eljük
|
||||
await self.session.rollback()
|
||||
# A külső tranzakciót is rollback (ha van)
|
||||
if self.session.in_transaction():
|
||||
await self.session.rollback()
|
||||
await self.session.close()
|
||||
print(" ✅ Rollback sikeres, dev adatbázis változatlan.")
|
||||
|
||||
async def run(self):
|
||||
"""Fő teszt folyamat."""
|
||||
try:
|
||||
await self.setup()
|
||||
ev_used, kft_used = await self.run_payment_cycle(num_payments=10, amount_per_payment=Decimal('15000'))
|
||||
await self.verify_double_entry()
|
||||
await self.verify_crop_rotation(ev_used, kft_used)
|
||||
await self.generate_report()
|
||||
await self.cleanup()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"\n❌ TESZT SIKERTELEN: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
# Hiba esetén is rollback
|
||||
if self.session:
|
||||
await self.session.rollback()
|
||||
await self.session.close()
|
||||
return False
|
||||
|
||||
|
||||
async def main():
|
||||
"""Fő belépési pont."""
|
||||
test = FinancialTruthTest()
|
||||
success = await test.run()
|
||||
|
||||
if success:
|
||||
print("\n🎉 FINANCIAL TRUTH VERIFICATION SIKERES!")
|
||||
print(" Epic 3 Pénzügyi Motor matematikailag sebezhetetlen.")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("\n💥 FINANCIAL TRUTH VERIFICATION SIKERTELEN!")
|
||||
print(" A Financial Orchestrator hibát tartalmaz, javítás szükséges.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user