átlagos kiegészítséek jó sok
This commit is contained in:
Roo
@@ -1,169 +1,153 @@
|
||||
# /opt/docker/dev/service_finder/backend/app/scripts/sync_engine.py
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Universal Schema Synchronizer
|
||||
|
||||
Dynamically imports all SQLAlchemy models from app.models, compares them with the live database,
|
||||
and creates missing tables/columns without dropping anything.
|
||||
|
||||
Safety First:
|
||||
- NEVER drops tables or columns.
|
||||
- Prints planned SQL before execution.
|
||||
- Requires confirmation for destructive operations (none in this script).
|
||||
"""
|
||||
|
||||
# docker exec -it sf_api python -m app.scripts.sync_engine
|
||||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy import inspect, text
|
||||
from sqlalchemy.schema import CreateTable, AddConstraint
|
||||
from sqlalchemy.sql.ddl import CreateColumn
|
||||
from sqlalchemy.schema import CreateTable
|
||||
|
||||
# Add backend to path
|
||||
# Path beállítása
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from app.database import Base
|
||||
from app.core.config import settings
|
||||
|
||||
def dynamic_import_models():
|
||||
"""
|
||||
Dynamically import all .py files in app.models directory to ensure Base.metadata is populated.
|
||||
"""
|
||||
"""Modellek betöltése a Metadata feltöltéséhez."""
|
||||
models_dir = Path(__file__).parent.parent / "models"
|
||||
imported = []
|
||||
|
||||
for py_file in models_dir.glob("*.py"):
|
||||
if py_file.name == "__init__.py":
|
||||
continue
|
||||
module_name = f"app.models.{py_file.stem}"
|
||||
# Rekurzív bejárás az alkönyvtárakkal együtt
|
||||
for py_file in models_dir.rglob("*.py"):
|
||||
if py_file.name == "__init__.py": continue
|
||||
# Számítsuk ki a modulnevet a models könyvtárhoz képest
|
||||
relative_path = py_file.relative_to(models_dir)
|
||||
# Konvertáljuk path-t modulná: pl. identity/identity.py -> identity.identity
|
||||
module_stem = str(relative_path).replace('/', '.').replace('\\', '.')[:-3] # eltávolítjuk a .py-t
|
||||
module_name = f"app.models.{module_stem}"
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
imported.append(module_name)
|
||||
print(f"✅ Imported {module_name}")
|
||||
importlib.import_module(module_name)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not import {module_name}: {e}")
|
||||
|
||||
# Also ensure the __init__ is loaded (it imports many models manually)
|
||||
import app.models
|
||||
print(f"📦 Total tables in Base.metadata: {len(Base.metadata.tables)}")
|
||||
return imported
|
||||
# Csak debug célra
|
||||
print(f"Failed to import {module_name}: {e}")
|
||||
pass
|
||||
|
||||
async def compare_and_repair():
|
||||
"""
|
||||
Compare SQLAlchemy metadata with live database and create missing tables/columns.
|
||||
"""
|
||||
print("🔗 Connecting to database...")
|
||||
async def perform_detailed_audit():
|
||||
engine = create_async_engine(str(settings.SQLALCHEMY_DATABASE_URI))
|
||||
|
||||
def get_diff_and_repair(connection):
|
||||
# Audit számlálók
|
||||
stats = {"ok": 0, "fixed": 0, "extra": 0, "missing": 0}
|
||||
|
||||
def audit_logic(connection):
|
||||
inspector = inspect(connection)
|
||||
metadata = Base.metadata
|
||||
db_schemas = inspector.get_schema_names()
|
||||
model_schemas = sorted({t.schema for t in metadata.sorted_tables if t.schema})
|
||||
|
||||
print("\n" + "="*80)
|
||||
print(f"{'🔍 RÉSZLETES SCHEMA AUDIT JELENTÉS':^80}")
|
||||
print("="*80)
|
||||
|
||||
# --- A IRÁNY: KÓD -> ADATBÁZIS (Minden ellenőrzése) ---
|
||||
print(f"\n[A IRÁNY: Kód (SQLAlchemy) -> Adatbázis (PostgreSQL)]")
|
||||
print("-" * 50)
|
||||
|
||||
# Get all schemas from models
|
||||
expected_schemas = sorted({t.schema for t in Base.metadata.sorted_tables if t.schema})
|
||||
print(f"📋 Expected schemas: {expected_schemas}")
|
||||
|
||||
# Ensure enum types exist in marketplace schema
|
||||
if 'marketplace' in expected_schemas:
|
||||
print("\n🔧 Ensuring enum types in marketplace schema...")
|
||||
# moderation_status enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'moderation_status' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.moderation_status AS ENUM ('pending', 'approved', 'rejected');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
# source_type enum
|
||||
connection.execute(text("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'source_type' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'marketplace')) THEN
|
||||
CREATE TYPE marketplace.source_type AS ENUM ('manual', 'ocr', 'import');
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
print("✅ Enum types ensured.")
|
||||
|
||||
for schema in expected_schemas:
|
||||
print(f"\n--- 🔍 Checking schema '{schema}' ---")
|
||||
|
||||
# Check if schema exists
|
||||
db_schemas = inspector.get_schema_names()
|
||||
for schema in model_schemas:
|
||||
# 1. Séma ellenőrzése
|
||||
if schema not in db_schemas:
|
||||
print(f"❌ Schema '{schema}' missing. Creating...")
|
||||
print(f"❌ HIÁNYZIK: Séma [{schema}] -> Létrehozás...")
|
||||
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
|
||||
print(f"✅ Schema '{schema}' created.")
|
||||
|
||||
# Get tables in this schema from models
|
||||
model_tables = [t for t in Base.metadata.sorted_tables if t.schema == schema]
|
||||
stats["fixed"] += 1
|
||||
else:
|
||||
print(f"✅ RENDBEN: Séma [{schema}] létezik.")
|
||||
stats["ok"] += 1
|
||||
|
||||
db_tables = inspector.get_table_names(schema=schema)
|
||||
|
||||
model_tables = [t for t in metadata.sorted_tables if t.schema == schema]
|
||||
|
||||
for table in model_tables:
|
||||
full_name = f"{schema}.{table.name}"
|
||||
|
||||
# 2. Tábla ellenőrzése
|
||||
if table.name not in db_tables:
|
||||
print(f"❌ Missing table: {schema}.{table.name}")
|
||||
# Generate CREATE TABLE statement
|
||||
create_stmt = CreateTable(table)
|
||||
# Print SQL for debugging
|
||||
sql_str = str(create_stmt.compile(bind=engine))
|
||||
print(f" SQL: {sql_str}")
|
||||
connection.execute(create_stmt)
|
||||
print(f"✅ Table {schema}.{table.name} created.")
|
||||
print(f" ❌ HIÁNYZIK: Tábla [{full_name}] -> Létrehozás...")
|
||||
connection.execute(CreateTable(table))
|
||||
stats["fixed"] += 1
|
||||
continue
|
||||
else:
|
||||
# Check columns
|
||||
db_columns = {c['name']: c for c in inspector.get_columns(table.name, schema=schema)}
|
||||
model_columns = table.columns
|
||||
|
||||
missing_cols = []
|
||||
for col in model_columns:
|
||||
if col.name not in db_columns:
|
||||
missing_cols.append(col)
|
||||
|
||||
if missing_cols:
|
||||
print(f"⚠️ Table {schema}.{table.name} missing columns: {[c.name for c in missing_cols]}")
|
||||
for col in missing_cols:
|
||||
# Generate ADD COLUMN statement
|
||||
col_type = col.type.compile(dialect=engine.dialect)
|
||||
sql = f'ALTER TABLE "{schema}"."{table.name}" ADD COLUMN "{col.name}" {col_type}'
|
||||
if col.nullable is False:
|
||||
sql += " NOT NULL"
|
||||
if col.default is not None:
|
||||
# Handle default values (simplistic)
|
||||
sql += f" DEFAULT {col.default.arg}"
|
||||
print(f" SQL: {sql}")
|
||||
connection.execute(text(sql))
|
||||
print(f"✅ Column {col.name} added.")
|
||||
print(f" ✅ RENDBEN: Tábla [{full_name}] létezik.")
|
||||
stats["ok"] += 1
|
||||
|
||||
# 3. Oszlopok ellenőrzése
|
||||
db_cols = {c['name']: c for c in inspector.get_columns(table.name, schema=schema)}
|
||||
for col in table.columns:
|
||||
col_path = f"{full_name}.{col.name}"
|
||||
if col.name not in db_cols:
|
||||
print(f" ❌ HIÁNYZIK: Oszlop [{col_path}] -> Hozzáadás...")
|
||||
col_type = col.type.compile(dialect=connection.dialect)
|
||||
default_sql = ""
|
||||
if col.server_default is not None:
|
||||
arg = col.server_default.arg
|
||||
val = arg.text if hasattr(arg, 'text') else str(arg)
|
||||
default_sql = f" DEFAULT {val}"
|
||||
null_sql = " NOT NULL" if not col.nullable else ""
|
||||
connection.execute(text(f'ALTER TABLE "{schema}"."{table.name}" ADD COLUMN "{col.name}" {col_type}{default_sql}{null_sql}'))
|
||||
stats["fixed"] += 1
|
||||
else:
|
||||
print(f"✅ Table {schema}.{table.name} is up‑to‑date.")
|
||||
print(f" ✅ RENDBEN: Oszlop [{col_path}]")
|
||||
stats["ok"] += 1
|
||||
|
||||
# --- B IRÁNY: ADATBÁZIS -> KÓD (Árnyék adatok keresése) ---
|
||||
print(f"\n[B IRÁNY: Adatbázis -> Kód (Extra elemek keresése)]")
|
||||
print("-" * 50)
|
||||
|
||||
print("\n--- ✅ Schema synchronization complete. ---")
|
||||
|
||||
for schema in model_schemas:
|
||||
if schema not in db_schemas: continue
|
||||
|
||||
db_tables = inspector.get_table_names(schema=schema)
|
||||
model_table_names = {t.name for t in metadata.sorted_tables if t.schema == schema}
|
||||
|
||||
for db_table in db_tables:
|
||||
# Ignore deprecated tables (ending with _deprecated)
|
||||
if db_table.endswith("_deprecated"):
|
||||
continue
|
||||
full_db_name = f"{schema}.{db_table}"
|
||||
if db_table not in model_table_names:
|
||||
print(f" ⚠️ EXTRA TÁBLA: [{full_db_name}] (Nincs a kódban!)")
|
||||
stats["extra"] += 1
|
||||
else:
|
||||
# Extra oszlopok a táblán belül
|
||||
db_cols = inspector.get_columns(db_table, schema=schema)
|
||||
model_col_names = {c.name for c in metadata.tables[full_db_name].columns}
|
||||
|
||||
for db_col in db_cols:
|
||||
col_name = db_col['name']
|
||||
if col_name not in model_col_names:
|
||||
print(f" ⚠️ EXTRA OSZLOP: [{full_db_name}.{col_name}]")
|
||||
stats["extra"] += 1
|
||||
|
||||
# --- ÖSSZESÍTŐ ---
|
||||
print("\n" + "="*80)
|
||||
print(f"{'📊 AUDIT ÖSSZESÍTŐ':^80}")
|
||||
print("="*80)
|
||||
print(f" ✅ Megfelelt (OK): {stats['ok']:>4} elem")
|
||||
print(f" ❌ Javítva/Pótolva (Fixed): {stats['fixed']:>4} elem")
|
||||
print(f" ⚠️ Extra (Shadow Data): {stats['extra']:>4} elem")
|
||||
print("-" * 80)
|
||||
if stats["fixed"] == 0 and stats["extra"] == 0:
|
||||
print(f"{'✨ A RENDSZER TÖKÉLETESEN SZINKRONBAN VAN!':^80}")
|
||||
else:
|
||||
print(f"{'ℹ️ A rendszer üzemkész, de nézd át az extra (Shadow) elemeket!':^80}")
|
||||
print("="*80 + "\n")
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(get_diff_and_repair)
|
||||
|
||||
await conn.run_sync(audit_logic)
|
||||
await engine.dispose()
|
||||
|
||||
async def main():
|
||||
print("🚀 Universal Schema Synchronizer")
|
||||
print("=" * 50)
|
||||
|
||||
# Step 1: Dynamic import
|
||||
print("\n📥 Step 1: Dynamically importing all models...")
|
||||
dynamic_import_models()
|
||||
|
||||
# Step 2: Compare and repair
|
||||
print("\n🔧 Step 2: Comparing with database and repairing...")
|
||||
await compare_and_repair()
|
||||
|
||||
# Step 3: Final verification
|
||||
print("\n📊 Step 3: Final verification...")
|
||||
# Run compare_schema.py logic to confirm everything is green
|
||||
from app.tests_internal.diagnostics.compare_schema import compare
|
||||
await compare()
|
||||
|
||||
print("\n✨ Synchronization finished successfully!")
|
||||
await perform_detailed_audit()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user