kincses/service-finder
1
0
Fork 0
Code Issues 7 Pull Requests Packages Projects 2 Releases Wiki Activity

refaktorálás javításai

Browse Source
This commit is contained in:
Roo
2026-03-13 10:22:41 +00:00
parent 2d8d23f469
commit f53e0b53df
140 changed files with 7316 additions and 4579 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
backend/app/workers/monitor_dashboard.py
View File

@@ -61,29 +61,29 @@ async def get_ollama_models():
async def get_stats(engine):
async with engine.connect() as conn:
# 1. Sebesség adatok
res_hr = await conn.execute(text("SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '1 hour'"))
res_hr = await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '1 hour'"))
hr_rate = res_hr.scalar() or 0
res_day = await conn.execute(text("SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '24 hours'"))
res_day = await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '24 hours'"))
day_rate = res_day.scalar() or 0
# 2. Pipeline
res_pipe = await conn.execute(text("""
SELECT
(SELECT count(*) FROM data.catalog_discovery WHERE status = 'pending') as r1,
(SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'unverified') as r2,
(SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'awaiting_ai_synthesis') as r3,
(SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'gold_enriched') as r4
(SELECT count(*) FROM vehicle.catalog_discovery WHERE status = 'pending') as r1,
(SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'unverified') as r2,
(SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'awaiting_ai_synthesis') as r3,
(SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched') as r4
"""))
r_counts = res_pipe.fetchone()
# 3. TOP 7
res_top = await conn.execute(text("SELECT make, count(*) as qty FROM data.vehicle_model_definitions GROUP BY make ORDER BY qty DESC LIMIT 7"))
res_top = await conn.execute(text("SELECT make, count(*) as qty FROM vehicle.vehicle_model_definitions GROUP BY make ORDER BY qty DESC LIMIT 7"))
top_makes = res_top.fetchall()
# 4. AKTIVITÁS (3 példány per robot)
res_r4 = await conn.execute(text("SELECT make, marketing_name FROM data.vehicle_model_definitions WHERE status = 'gold_enriched' ORDER BY updated_at DESC LIMIT 5"))
res_r3 = await conn.execute(text("SELECT make, marketing_name FROM data.vehicle_model_definitions WHERE status = 'ai_synthesis_in_progress' ORDER BY updated_at DESC LIMIT 5"))
res_r12 = await conn.execute(text("SELECT make, model FROM data.catalog_discovery WHERE status = 'processing' ORDER BY updated_at DESC LIMIT 5"))
res_r4 = await conn.execute(text("SELECT make, marketing_name FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' ORDER BY updated_at DESC LIMIT 5"))
res_r3 = await conn.execute(text("SELECT make, marketing_name FROM vehicle.vehicle_model_definitions WHERE status = 'ai_synthesis_in_progress' ORDER BY updated_at DESC LIMIT 5"))
res_r12 = await conn.execute(text("SELECT make, model FROM vehicle.catalog_discovery WHERE status = 'processing' ORDER BY updated_at DESC LIMIT 5"))
hw = await get_hardware_stats()
ai = await get_ollama_models()

4
backend/app/workers/service/service_robot_2_researcher.py
View File

@@ -76,10 +76,10 @@ class ServiceResearcher:
async with AsyncSessionLocal() as db:
# ATOMI ZÁROLÁS
query = text("""
UPDATE data.service_staging
UPDATE marketplace.service_staging
SET status = 'research_in_progress'
WHERE id = (
SELECT id FROM data.service_staging
SELECT id FROM marketplace.service_staging
WHERE status = 'pending'
FOR UPDATE SKIP LOCKED
LIMIT 1

10
backend/app/workers/service/service_robot_3_enricher.py
View File

@@ -59,10 +59,10 @@ class ServiceEnricher:
async with AsyncSessionLocal() as db:
# 1. Zárolunk egy "enrich_ready" szervizt a Staging táblából
query = text("""
UPDATE data.service_staging
UPDATE marketplace.service_staging
SET status = 'enriching'
WHERE id = (
SELECT id FROM data.service_staging
SELECT id FROM marketplace.service_staging
WHERE status = 'enrich_ready'
FOR UPDATE SKIP LOCKED
LIMIT 1
@@ -81,7 +81,7 @@ class ServiceEnricher:
try:
# 2. Áttesszük a végleges ServiceProfile táblába (mert már van elég adatunk a webről)
profile_stmt = text("""
INSERT INTO data.service_profiles
INSERT INTO marketplace.service_profiles
(fingerprint, status, trust_score, location, is_verified, bio)
VALUES (:fp, 'active', 40, ST_SetSRID(ST_MakePoint(19.04, 47.49), 4326), false, :bio)
ON CONFLICT (fingerprint) DO UPDATE SET bio = EXCLUDED.bio
@@ -96,13 +96,13 @@ class ServiceEnricher:
await cls.match_expertise_to_service(process_db, profile_id, web_context)
# 4. Lezárjuk a Staging feladatot
await process_db.execute(text("UPDATE data.service_staging SET status = 'processed' WHERE id = :id"), {"id": s_id})
await process_db.execute(text("UPDATE marketplace.service_staging SET status = 'processed' WHERE id = :id"), {"id": s_id})
await process_db.commit()
except Exception as e:
await process_db.rollback()
logger.error(f"Hiba a dúsítás során ({s_id}): {e}")
await process_db.execute(text("UPDATE data.service_staging SET status = 'error' WHERE id = :id"), {"id": s_id})
await process_db.execute(text("UPDATE marketplace.service_staging SET status = 'error' WHERE id = :id"), {"id": s_id})
await process_db.commit()
else:
await asyncio.sleep(15)

8
backend/app/workers/service/service_robot_4_validator_google.py
View File

@@ -158,11 +158,11 @@ class GoogleValidator:
async with AsyncSessionLocal() as db:
# ATOMI ZÁROLÁS: Keresünk egy aktív, de még nem validált szervizt
query = text("""
UPDATE data.service_profiles
UPDATE marketplace.service_profiles
SET status = 'validation_in_progress'
WHERE id = (
SELECT id FROM data.service_profiles
WHERE is_verified = false
SELECT id FROM marketplace.service_profiles
WHERE is_verified = false
AND status NOT IN ('validation_in_progress', 'ghost')
FOR UPDATE SKIP LOCKED
LIMIT 1
@@ -181,7 +181,7 @@ class GoogleValidator:
# Ha API hiba volt, visszaállítjuk az eredeti állapotot
if status == "ERROR":
await process_db.execute(text("UPDATE data.service_profiles SET status = 'active' WHERE id = :id"), {"id": p_id})
await process_db.execute(text("UPDATE marketplace.service_profiles SET status = 'active' WHERE id = :id"), {"id": p_id})
await process_db.commit()
if status == "QUOTA_EXCEEDED":

32
backend/app/workers/vehicle/robot_report.py
View File

@@ -18,37 +18,37 @@ async def get_data():
async with engine.connect() as conn:
# Pipeline adatok (R1-R4)
pipe = await conn.execute(text("""
SELECT
(SELECT count(*) FROM data.catalog_discovery WHERE status = 'pending') as r1,
(SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'unverified') as r2,
(SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'awaiting_ai_synthesis') as r3,
(SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'gold_enriched') as r4
SELECT
(SELECT count(*) FROM vehicle.catalog_discovery WHERE status = 'pending') as r1,
(SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'unverified') as r2,
(SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'awaiting_ai_synthesis') as r3,
(SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched') as r4
"""))
p_res = pipe.fetchone()
# AI Termelés
ai_hr = await conn.execute(text("SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '1 hour'"))
ai_day = await conn.execute(text("SELECT count(*) FROM data.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '24 hours'"))
ai_hr = await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '1 hour'"))
ai_day = await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '24 hours'"))
# Market Matrix (1.3)
market_res = await conn.execute(text("SELECT vehicle_class, market, count(*) FROM data.catalog_discovery GROUP BY 1, 2"))
market_res = await conn.execute(text("SELECT vehicle_class, market, count(*) FROM vehicle.catalog_discovery GROUP BY 1, 2"))
m_data = market_res.fetchall()
# Robot Top listák (2.1 - 2.3)
r1_top = await conn.execute(text("SELECT make, count(*) FROM data.catalog_discovery WHERE market = 'RDW' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
r12_top = await conn.execute(text("SELECT make, count(*) FROM data.catalog_discovery WHERE market = 'USA_IMPORT' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
r14_top = await conn.execute(text("SELECT make, count(*) FROM data.catalog_discovery WHERE vehicle_class = 'motorcycle' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
r1_top = await conn.execute(text("SELECT make, count(*) FROM vehicle.catalog_discovery WHERE market = 'RDW' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
r12_top = await conn.execute(text("SELECT make, count(*) FROM vehicle.catalog_discovery WHERE market = 'USA_IMPORT' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
r14_top = await conn.execute(text("SELECT make, count(*) FROM vehicle.catalog_discovery WHERE vehicle_class = 'motorcycle' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
# Általános Top (3.1 - 3.3)
pending_top = await conn.execute(text("SELECT make, count(*) FROM data.catalog_discovery WHERE status = 'pending' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
gold_top = await conn.execute(text("SELECT make, count(*) FROM data.vehicle_model_definitions WHERE status = 'gold_enriched' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
status_stats = await conn.execute(text("SELECT status, count(*) FROM data.vehicle_model_definitions GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
pending_top = await conn.execute(text("SELECT make, count(*) FROM vehicle.catalog_discovery WHERE status = 'pending' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
gold_top = await conn.execute(text("SELECT make, count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
status_stats = await conn.execute(text("SELECT status, count(*) FROM vehicle.vehicle_model_definitions GROUP BY 1 ORDER BY 2 DESC LIMIT 5"))
# Kategória Top (4.1 - 4.3)
cat_tops = {}
for c in ['car', 'motorcycle', 'truck']:
res = await conn.execute(text(f"SELECT make, count(*) FROM data.catalog_discovery WHERE vehicle_class = '{c}' GROUP BY 1 ORDER BY 2 DESC LIMIT 4"))
total = await conn.execute(text(f"SELECT count(*) FROM data.catalog_discovery WHERE vehicle_class = '{c}'"))
res = await conn.execute(text(f"SELECT make, count(*) FROM vehicle.catalog_discovery WHERE vehicle_class = '{c}' GROUP BY 1 ORDER BY 2 DESC LIMIT 4"))
total = await conn.execute(text(f"SELECT count(*) FROM vehicle.catalog_discovery WHERE vehicle_class = '{c}'"))
cat_tops[c] = {"list": res.fetchall(), "total": total.scalar() or 0}
return {

4
backend/app/workers/vehicle/vehicle_data_loader.py
View File

@@ -40,7 +40,7 @@ class VehicleDataLoader:
def map_source_data(self, source_name, raw_data):
"""
Mapping Layer: Átfordítja a különböző források JSON szerkezetét
a mi egységes data.reference_lookup sémánkra.
a mi egységes vehicle.reference_lookup sémánkra.
"""
unified_entries = []
@@ -89,7 +89,7 @@ class VehicleDataLoader:
async with AsyncSessionLocal() as db:
stmt = text("""
INSERT INTO data.reference_lookup (make, model, year, specs, source, source_id)
INSERT INTO vehicle.reference_lookup (make, model, year, specs, source, source_id)
VALUES (:make, :model, :year, :specs, :source, :source_id)
ON CONFLICT ON CONSTRAINT _ref_lookup_uc
DO UPDATE SET specs = EXCLUDED.specs, updated_at = NOW()

26
backend/app/workers/vehicle/vehicle_robot_0_discovery_engine.py
View File

@@ -31,14 +31,14 @@ class DiscoveryEngine:
try:
async with AsyncSessionLocal() as db:
# A) Hunter takarítás (visszaállítás pending-re, ha a Hunter lefagyott)
res1 = await db.execute(text("UPDATE data.catalog_discovery SET status = 'pending' WHERE status = 'processing' RETURNING id;"))
res1 = await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'pending' WHERE status = 'processing' RETURNING id;"))
hunter_resets = len(res1.fetchall())
if hunter_resets > 0:
logger.warning(f"🔄 {hunter_resets} db beragadt Hunter feladat (processing) visszaállítva 'pending'-re.")
# B) AI Robotok takarítása (2 órás timeout)
query2 = text("""
UPDATE data.vehicle_model_definitions
UPDATE vehicle.vehicle_model_definitions
SET status = CASE
WHEN status = 'research_in_progress' THEN 'unverified'
WHEN status = 'ai_synthesis_in_progress' THEN 'awaiting_ai_synthesis'
@@ -133,22 +133,22 @@ class DiscoveryEngine:
# A MÁGIA: Különbözeti Szinkronizáció SQL + Explicit Type Casting
query = text("""
INSERT INTO data.catalog_discovery (make, model, vehicle_class, status, priority_score)
SELECT
CAST(:make AS VARCHAR),
CAST(:model AS VARCHAR),
CAST(:v_class AS VARCHAR),
'pending',
INSERT INTO vehicle.catalog_discovery (make, model, vehicle_class, status, priority_score)
SELECT
CAST(:make AS VARCHAR),
CAST(:model AS VARCHAR),
CAST(:v_class AS VARCHAR),
'pending',
:priority
WHERE NOT EXISTS (
SELECT 1 FROM data.vehicle_model_definitions
WHERE make = CAST(:make AS VARCHAR)
AND marketing_name = CAST(:model AS VARCHAR)
SELECT 1 FROM vehicle.vehicle_model_definitions
WHERE make = CAST(:make AS VARCHAR)
AND marketing_name = CAST(:model AS VARCHAR)
AND status = 'gold_enriched'
)
ON CONFLICT (make, model)
ON CONFLICT (make, model)
DO UPDATE SET priority_score = EXCLUDED.priority_score
WHERE data.catalog_discovery.status != 'processed'
WHERE vehicle.catalog_discovery.status != 'processed'
RETURNING xmax;
""")

6
backend/app/workers/vehicle/vehicle_robot_0_gb_discovery.py
View File

@@ -29,7 +29,7 @@ class GBDiscoveryEngine:
# Létrehozzuk a GB várólistát (ha még nem létezne)
async with AsyncSessionLocal() as db:
await db.execute(text("""
CREATE TABLE IF NOT EXISTS data.gb_catalog_discovery (
CREATE TABLE IF NOT EXISTS vehicle.gb_catalog_discovery (
id SERIAL PRIMARY KEY,
vrm VARCHAR(20) UNIQUE NOT NULL,
make VARCHAR(100),
@@ -51,10 +51,10 @@ class GBDiscoveryEngine:
# Szűrünk: Csak akkor tesszük be, ha ez az autó még nincs gold_enriched állapotban!
query = text("""
INSERT INTO data.gb_catalog_discovery (vrm, make, model)
INSERT INTO vehicle.gb_catalog_discovery (vrm, make, model)
SELECT :vrm, :make, :model
WHERE NOT EXISTS (
SELECT 1 FROM data.vehicle_model_definitions
SELECT 1 FROM vehicle.vehicle_model_definitions
WHERE make = :make AND marketing_name = :model AND status = 'gold_enriched'
)
ON CONFLICT (vrm) DO NOTHING;

10
backend/app/workers/vehicle/vehicle_robot_0_strategist.py
View File

@@ -63,7 +63,7 @@ class Robot0Strategist:
# --- SÉMA ELLENŐRZÉS (Golyóálló megoldás) ---
async with AsyncSessionLocal() as db:
try:
await db.execute(text("ALTER TABLE data.catalog_discovery ADD COLUMN IF NOT EXISTS priority_score INTEGER DEFAULT 0;"))
await db.execute(text("ALTER TABLE vehicle.catalog_discovery ADD COLUMN IF NOT EXISTS priority_score INTEGER DEFAULT 0;"))
await db.commit()
logger.info("✅ Adatbázis séma rendben (priority_score aktív).")
except Exception as e:
@@ -88,11 +88,11 @@ class Robot0Strategist:
try:
# UPSERT: Beállítjuk a prioritást, de nem bántjuk a már kész rekordokat
query = text("""
INSERT INTO data.catalog_discovery (make, model, vehicle_class, status, source, attempts, priority_score)
INSERT INTO vehicle.catalog_discovery (make, model, vehicle_class, status, source, attempts, priority_score)
VALUES (:make, 'ALL_VARIANTS', :class, 'pending', 'STRATEGIST-V2', 0, :score)
ON CONFLICT (make, model, vehicle_class)
DO UPDATE SET priority_score = :score
WHERE data.catalog_discovery.status NOT IN ('processed', 'in_progress');
ON CONFLICT (make, model, vehicle_class)
DO UPDATE SET priority_score = :score
WHERE vehicle.catalog_discovery.status NOT IN ('processed', 'in_progress');
""")
await db.execute(query, {"make": make_name, "class": v_class, "score": count})

4
backend/app/workers/vehicle/vehicle_robot_1_2_nhtsa_fetcher.py
View File

@@ -16,7 +16,7 @@ class NHTSAFetcher:
"""Lekéri azokat a márkákat, amik már benne vannak az adatbázisban EU-s forrásból."""
async with AsyncSessionLocal() as db:
# Csak azokat a márkákat keressük az USA-ban, amiket az EU-ban (RDW) már láttunk
query = text("SELECT DISTINCT make FROM data.catalog_discovery WHERE market = 'EU' OR source = 'RDW'")
query = text("SELECT DISTINCT make FROM vehicle.catalog_discovery WHERE market = 'EU' OR source = 'RDW'")
res = await db.execute(query)
return [row[0] for row in res.fetchall()]
@@ -47,7 +47,7 @@ class NHTSAFetcher:
model_name = m.get("Model_Name").upper().strip()
# USA_IMPORT jelölés, de csak EU-s márkákhoz!
query = text("""
INSERT INTO data.catalog_discovery
INSERT INTO vehicle.catalog_discovery
(make, model, vehicle_class, status, market, model_year, priority_score, source)
VALUES (:make, :model, 'car', 'pending', 'USA_IMPORT', :year, 5, 'NHTSA-EU-FILTERED')
ON CONFLICT ON CONSTRAINT _make_model_market_year_uc DO NOTHING

2
backend/app/workers/vehicle/vehicle_robot_1_4_bike_hunter.py
View File

@@ -35,7 +35,7 @@ class BikeHunter:
model_name = m.get("Model_Name").upper().strip()
# TISZTA SQL - Nincs Simon!
query = text("""
INSERT INTO data.catalog_discovery
INSERT INTO vehicle.catalog_discovery
(make, model, vehicle_class, status, market, model_year, priority_score, source)
VALUES (:make, :model, 'motorcycle', 'pending', 'USA_IMPORT', :year, 8, 'NHTSA-V1-BIKE')
ON CONFLICT ON CONSTRAINT _make_model_market_year_uc DO NOTHING

2
backend/app/workers/vehicle/vehicle_robot_1_5_heavy_eu.py
View File

@@ -51,7 +51,7 @@ class HeavyEUHunter:
# Szűrés a kért EU márkákra + amik jönnek az RDW-ből
query = text("""
INSERT INTO data.catalog_discovery
INSERT INTO vehicle.catalog_discovery
(make, model, vehicle_class, status, market, priority_score, source)
VALUES (:make, :model, :v_class, 'pending', 'EU', 20, 'RDW-HEAVY')
ON CONFLICT ON CONSTRAINT _make_model_market_year_uc DO NOTHING

6
backend/app/workers/vehicle/vehicle_robot_1_catalog_hunter.py
View File

@@ -173,7 +173,7 @@ class CatalogHunter:
await asyncio.sleep(0.5) # Lassítjuk kicsit a terhelést
# Discovery státusz frissítése
await db.execute(text("UPDATE data.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task_id})
await db.execute(text("UPDATE vehicle.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task_id})
await db.commit()
@classmethod
@@ -184,10 +184,10 @@ class CatalogHunter:
# ATOMI ZÁROLÁS (Race condition ellenszere)
# Keresünk egy pending feladatot, azonnal zároljuk és átállítjuk processingre!
query = text("""
UPDATE data.catalog_discovery
UPDATE vehicle.catalog_discovery
SET status = 'processing'
WHERE id = (
SELECT id FROM data.catalog_discovery
SELECT id FROM vehicle.catalog_discovery
WHERE status = 'pending'
ORDER BY priority_score DESC
FOR UPDATE SKIP LOCKED

14
backend/app/workers/vehicle/vehicle_robot_1_gb_hunter.py
View File

@@ -101,13 +101,13 @@ class GBHunter:
if data == "NOT_FOUND":
# Hibás volt a CSV rendszám, lezárjuk a feladatot
await db.execute(text("UPDATE data.gb_catalog_discovery SET status = 'invalid_vrm' WHERE id = :id"), {"id": record_id})
await db.execute(text("UPDATE vehicle.gb_catalog_discovery SET status = 'invalid_vrm' WHERE id = :id"), {"id": record_id})
await db.commit()
return "DONE"
if not data:
# Hálózati hiba, visszateszük a sorba
await db.execute(text("UPDATE data.gb_catalog_discovery SET status = 'pending' WHERE id = :id"), {"id": record_id})
await db.execute(text("UPDATE vehicle.gb_catalog_discovery SET status = 'pending' WHERE id = :id"), {"id": record_id})
await db.commit()
return "ERROR"
@@ -121,7 +121,7 @@ class GBHunter:
# Beszúrás a Mestertáblába (A hiányzó lóerőt majd az Alkimista megszerzi!)
query_vmd = text("""
INSERT INTO data.vehicle_model_definitions
INSERT INTO vehicle.vehicle_model_definitions
(make, marketing_name, vehicle_class, fuel_type, engine_capacity, co2_emissions_combined, year_from, type_approval_number, status, source)
VALUES (:make, :model, 'car', :fuel, :ccm, :co2, :year, :approval, 'ACTIVE', 'GB-DVLA-API')
ON CONFLICT (make, normalized_name, variant_code, version_code, fuel_type) DO NOTHING;
@@ -139,14 +139,14 @@ class GBHunter:
})
# Pipáljuk a feladatot
await db.execute(text("UPDATE data.gb_catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": record_id})
await db.execute(text("UPDATE vehicle.gb_catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": record_id})
await db.commit()
logger.info(f"✅ GB Rekord mentve a VMD táblába: {dvla_make} {model_csv} ({ccm}cc {fuel})")
return "DONE"
except Exception as e:
await db.rollback()
logger.error(f"🚨 Adatbázis hiba mentéskor: {e}")
await db.execute(text("UPDATE data.gb_catalog_discovery SET status = 'pending' WHERE id = :id"), {"id": record_id})
await db.execute(text("UPDATE vehicle.gb_catalog_discovery SET status = 'pending' WHERE id = :id"), {"id": record_id})
await db.commit()
return "ERROR"
@@ -160,10 +160,10 @@ class GBHunter:
async with AsyncSessionLocal() as db:
# ATOMI ZÁROLÁS
query = text("""
UPDATE data.gb_catalog_discovery
UPDATE vehicle.gb_catalog_discovery
SET status = 'processing'
WHERE id = (
SELECT id FROM data.gb_catalog_discovery
SELECT id FROM vehicle.gb_catalog_discovery
WHERE status = 'pending'
FOR UPDATE SKIP LOCKED
LIMIT 1

52
backend/app/workers/vehicle/vehicle_robot_2_researcher.py
View File

@@ -82,6 +82,45 @@ class VehicleResearcher:
logger.debug(f"Keresési hiba ({label}): {e}")
return f"[SOURCE: {label}]\nKERESÉSI HIBA.\n"
def extract_specs_from_text(self, text: str) -> dict:
""" Regex alapú kinyerés a nyers szövegből: ccm, kW, motoradatok. """
import re
specs = {}
# CCM (köbcentiméter) minta: 1998 cc, 2.0 L, 2000 cm³
ccm_pattern = r'(\d{3,4})\s*(?:cc|ccm|cm³|cm3|cc\.)'
match = re.search(ccm_pattern, text, re.IGNORECASE)
if match:
specs['ccm'] = int(match.group(1))
else:
# Alternatív minta: 2.0 liter -> 2000 cc
liter_pattern = r'(\d+\.?\d*)\s*(?:L|liter|ℓ)'
match = re.search(liter_pattern, text, re.IGNORECASE)
if match:
liters = float(match.group(1))
specs['ccm'] = int(liters * 1000)
# KW (kilowatt) minta: 150 kW, 150kW, 150 KW
kw_pattern = r'(\d{2,4})\s*(?:kW|kw|KW)'
match = re.search(kw_pattern, text, re.IGNORECASE)
if match:
specs['kw'] = int(match.group(1))
else:
# Le (lóerő) átváltás: 150 LE -> 110 kW (kb)
hp_pattern = r'(\d{2,4})\s*(?:HP|hp|LE|le|Ps)'
match = re.search(hp_pattern, text, re.IGNORECASE)
if match:
hp = int(match.group(1))
specs['kw'] = int(hp * 0.7355) # hozzávetőleges átváltás
# Motor kód minta: motor kód: 1.8 TSI, engine code: N47
engine_pattern = r'(?:motor\s*kód|engine\s*code|motor\s*code)[:\s]+([A-Z0-9\.\- ]+)'
match = re.search(engine_pattern, text, re.IGNORECASE)
if match:
specs['engine_code'] = match.group(1).strip()
return specs
async def research_vehicle(self, db, vehicle_id: int, make: str, model: str, engine: str, year: str, current_attempts: int):
""" Egy jármű átvilágítása és a strukturált 'Akta' elkészítése a GPU számára. """
engine_safe = engine or ""
@@ -111,6 +150,9 @@ class VehicleResearcher:
if len(full_context) > 2500:
full_context = full_context[:2500] + "\n...[TRUNCATED TO SAVE GPU TOKENS]"
# Regex alapú specifikáció kinyerés
extracted_specs = self.extract_specs_from_text(full_context)
try:
if len(full_context.strip()) > 150: # Csökkentettük az elvárást, mert a célzott keresés tömörebb
await db.execute(
@@ -118,6 +160,7 @@ class VehicleResearcher:
.where(VehicleModelDefinition.id == vehicle_id)
.values(
raw_search_context=full_context,
research_metadata=extracted_specs,
status='awaiting_ai_synthesis', # Kész az Akta, mehet az Alkimistának!
last_research_at=func.now(),
attempts=current_attempts + 1
@@ -155,14 +198,15 @@ class VehicleResearcher:
async with AsyncSessionLocal() as db:
# ATOMI ZÁROLÁS
query = text("""
UPDATE data.vehicle_model_definitions
UPDATE vehicle.vehicle_model_definitions
SET status = 'research_in_progress'
WHERE id = (
SELECT id FROM data.vehicle_model_definitions
SELECT id FROM vehicle.vehicle_model_definitions
WHERE status IN ('unverified', 'awaiting_research', 'ACTIVE')
AND attempts < :max_attempts
ORDER BY
CASE WHEN make = 'TOYOTA' THEN 1 ELSE 2 END,
AND is_manual = FALSE
ORDER BY
CASE WHEN make = 'TOYOTA' THEN 1 ELSE 2 END,
attempts ASC
FOR UPDATE SKIP LOCKED
LIMIT 1

11
backend/app/workers/vehicle/vehicle_robot_3_alchemist_pro.py
View File

@@ -97,7 +97,7 @@ class TechEnricher:
clean_model = str(ai_data.get("marketing_name", base_info['m_name']))[:50].upper()
cat_stmt = text("""
INSERT INTO data.vehicle_catalog
INSERT INTO vehicle.vehicle_catalog
(master_definition_id, make, model, power_kw, engine_capacity, fuel_type, factory_data)
VALUES (:m_id, :make, :model, :kw, :ccm, :fuel, :factory)
ON CONFLICT ON CONSTRAINT uix_vehicle_catalog_full DO NOTHING
@@ -173,19 +173,20 @@ class TechEnricher:
async with AsyncSessionLocal() as db:
# ATOMI ZÁROLÁS (A "Szent Grál" a race condition ellen)
query = text("""
UPDATE data.vehicle_model_definitions
UPDATE vehicle.vehicle_model_definitions
SET status = 'ai_synthesis_in_progress'
WHERE id = (
SELECT id FROM data.vehicle_model_definitions
SELECT id FROM vehicle.vehicle_model_definitions
WHERE status IN ('awaiting_ai_synthesis', 'ACTIVE')
AND attempts < :max_attempts
ORDER BY
AND is_manual = FALSE
ORDER BY
CASE WHEN status = 'awaiting_ai_synthesis' THEN 1 ELSE 2 END,
priority_score DESC
FOR UPDATE SKIP LOCKED
LIMIT 1
)
RETURNING id, make, marketing_name, vehicle_class, power_kw, engine_capacity,
RETURNING id, make, marketing_name, vehicle_class, power_kw, engine_capacity,
fuel_type, engine_code, euro_classification, cylinders, raw_search_context, attempts;
""")

4
backend/app/workers/vehicle/vehicle_robot_4_vin_auditor.py
View File

@@ -86,10 +86,10 @@ class VINAuditor:
async with AsyncSessionLocal() as db:
# ATOMI ZÁROLÁS (Asset táblán)
query = text("""
UPDATE data.assets
UPDATE vehicle.assets
SET status = 'audit_in_progress'
WHERE id = (
SELECT id FROM data.assets
SELECT id FROM vehicle.assets
WHERE is_verified = false
AND vin IS NOT NULL
AND status NOT IN ('audit_in_progress', 'audit_failed')