#!/usr/bin/env python3 """ P0 DATABASE PURGE & RELINK — Deep Clean & Relink Script CRITICAL DIRECTIVE: Complete database hygiene. Before deleting fake organizations, we MUST perfectly re-link all financial data (AssetCost/invoices) to their new marketplace.service_providers counterparts. All garbage records (branches, members) tied to the fake orgs must be completely destroyed. Safe List (valódi garázsok, amelyek NEM kerülnek törlésre): (15, 21, 43, 45, 47, 48, 49, 50, 66, 67, 69, 70, 71) Folyamat: Step A (The Financial Relink): 1. Iterate through fleet_finance.asset_costs where vendor_organization_id is a fake org. 2. Find the matching service_provider_id in marketplace.service_providers (match by name). 3. Set AssetCost.service_provider_id to this new ID. 4. ONLY THEN set AssetCost.vendor_organization_id = NULL. 5. Do the exact same secure relinking for marketplace.service_profiles.organization_id -> map to service_provider_id, then set organization_id = NULL. Step B (Incinerate Garbage Records): 1. DELETE FROM finance.org_subscriptions WHERE org_id is a fake org. 2. DELETE FROM finance.credit_logs WHERE org_id is a fake org. 3. DELETE FROM fleet.asset_assignments WHERE organization_id is a fake org. 4. UPDATE vehicle.assets SET current_organization_id=NULL, operator_org_id=NULL, owner_org_id=NULL WHERE they reference fake orgs. 5. UPDATE fleet_finance.asset_costs SET organization_id=NULL WHERE org is fake. 6. DELETE FROM marketplace.ratings WHERE target_organization_id is a fake org. 7. DELETE FROM fleet.branches WHERE organization_id is a fake org. 8. DELETE FROM fleet.organization_members WHERE organization_id is a fake org. 9. DELETE FROM fleet.contact_persons WHERE organization_id is a fake org. 10. DELETE FROM fleet.org_relationships WHERE source_org_id or target_org_id is a fake org. 11. DELETE FROM fleet.organization_financials WHERE organization_id is a fake org. 12. DELETE FROM fleet.org_sales_assignments WHERE organization_id is a fake org. Step C (The Final Purge): 1. DELETE FROM fleet.organizations WHERE id is a fake org. Használat: docker compose exec sf_api python3 /app/scripts/deep_purge_fake_orgs.py """ import asyncio import logging import sys from datetime import datetime, timezone from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine # Import settings sys.path.insert(0, '/app') from app.core.config import settings # ────────────────────────────────────────────── # Konfiguráció # ────────────────────────────────────────────── # Safe list: ezek az Organization ID-k valódi garázsok, NEM töröljük őket SAFE_ORG_IDS = (15, 21, 43, 45, 47, 48, 49, 50, 66, 67, 69, 70, 71) # Adatbázis URL a settings-ből DATABASE_URL = settings.DATABASE_URL logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[logging.StreamHandler(sys.stdout)], ) logger = logging.getLogger(__name__) async def deep_purge_fake_orgs(): """ Main purge logic — three steps: Relink, Incinerate, Purge. """ engine = create_async_engine(DATABASE_URL, echo=False) async with AsyncSession(engine) as db: try: # ── 0. Identify fake orgs ── safe_list = ", ".join([str(x) for x in SAFE_ORG_IDS]) fake_orgs_sql = f""" SELECT id, name FROM fleet.organizations WHERE id NOT IN ({safe_list}) AND is_deleted = false ORDER BY id """ result = await db.execute(text(fake_orgs_sql)) fake_orgs = result.fetchall() if not fake_orgs: logger.info("✅ No fake organizations found. Nothing to do.") return fake_org_ids = [str(org[0]) for org in fake_orgs] fake_org_ids_str = ", ".join(fake_org_ids) logger.info(f"🔍 Found {len(fake_orgs)} fake organizations to purge:") for org_id, org_name in fake_orgs: logger.info(f" - ID={org_id}: {org_name}") # ===================================================================== # STEP A: THE FINANCIAL RELINK # ===================================================================== logger.info("\n" + "=" * 60) logger.info("STEP A: Financial Relink") logger.info("=" * 60) # ── A1. Relink AssetCost records ── logger.info("\n📊 A1. Relinking AssetCost records...") # Get all AssetCost records with vendor_organization_id pointing to fake orgs asset_costs_sql = f""" SELECT ac.id, ac.vendor_organization_id, ac.service_provider_id, o.name as org_name FROM fleet_finance.asset_costs ac JOIN fleet.organizations o ON o.id = ac.vendor_organization_id WHERE ac.vendor_organization_id IN ({fake_org_ids_str}) AND ac.service_provider_id IS NULL """ ac_result = await db.execute(text(asset_costs_sql)) asset_costs = ac_result.fetchall() asset_cost_relinked = 0 asset_cost_skipped = 0 asset_cost_errors = 0 for ac_row in asset_costs: ac_id = ac_row[0] vendor_org_id = ac_row[1] org_name = ac_row[3] try: # Find matching service_provider by name find_sp_sql = text(""" SELECT id FROM marketplace.service_providers WHERE name = :org_name LIMIT 1 """) sp_result = await db.execute(find_sp_sql, {"org_name": org_name}) sp_row = sp_result.fetchone() if sp_row: sp_id = sp_row[0] # Set service_provider_id, then NULL the vendor_organization_id update_ac_sql = text(""" UPDATE fleet_finance.asset_costs SET service_provider_id = :sp_id, vendor_organization_id = NULL WHERE id = :ac_id """) await db.execute(update_ac_sql, {"sp_id": sp_id, "ac_id": ac_id}) asset_cost_relinked += 1 logger.info( f" ✅ AssetCost {ac_id}: vendor_org={vendor_org_id} -> " f"service_provider_id={sp_id}" ) else: # No matching service_provider found — just NULL the vendor_org logger.warning( f" ⚠️ No ServiceProvider found for org '{org_name}' " f"(vendor_org_id={vendor_org_id}). Setting vendor_organization_id=NULL." ) null_ac_sql = text(""" UPDATE fleet_finance.asset_costs SET vendor_organization_id = NULL WHERE id = :ac_id """) await db.execute(null_ac_sql, {"ac_id": ac_id}) asset_cost_skipped += 1 except Exception as e: asset_cost_errors += 1 logger.error( f" ❌ Error relinking AssetCost {ac_id}: {e}" ) logger.info( f"\n📊 AssetCost Relink Summary: " f"{asset_cost_relinked} relinked, " f"{asset_cost_skipped} skipped (no SP), " f"{asset_cost_errors} errors" ) # ── A2. Relink ServiceProfile records ── logger.info("\n📊 A2. Relinking ServiceProfile records...") profiles_sql = f""" SELECT sp.id, sp.organization_id, sp.service_provider_id, o.name as org_name FROM marketplace.service_profiles sp JOIN fleet.organizations o ON o.id = sp.organization_id WHERE sp.organization_id IN ({fake_org_ids_str}) """ prof_result = await db.execute(text(profiles_sql)) profiles = prof_result.fetchall() profile_relinked = 0 profile_skipped = 0 profile_errors = 0 for prof_row in profiles: profile_id = prof_row[0] org_id = prof_row[1] existing_sp_id = prof_row[2] org_name = prof_row[3] try: if existing_sp_id is not None: # Already has service_provider_id — just NULL the organization_id null_prof_sql = text(""" UPDATE marketplace.service_profiles SET organization_id = NULL WHERE id = :profile_id """) await db.execute(null_prof_sql, {"profile_id": profile_id}) profile_relinked += 1 logger.info( f" ✅ ServiceProfile {profile_id}: org={org_id} -> " f"already had sp_id={existing_sp_id}, NULLed org" ) else: # Find matching service_provider by name find_sp_sql = text(""" SELECT id FROM marketplace.service_providers WHERE name = :org_name LIMIT 1 """) sp_result = await db.execute(find_sp_sql, {"org_name": org_name}) sp_row = sp_result.fetchone() if sp_row: sp_id = sp_row[0] # Set service_provider_id, then NULL the organization_id update_prof_sql = text(""" UPDATE marketplace.service_profiles SET service_provider_id = :sp_id, organization_id = NULL WHERE id = :profile_id """) await db.execute( update_prof_sql, {"sp_id": sp_id, "profile_id": profile_id} ) profile_relinked += 1 logger.info( f" ✅ ServiceProfile {profile_id}: org={org_id} -> " f"service_provider_id={sp_id}" ) else: # No matching service_provider — just NULL the org logger.warning( f" ⚠️ No ServiceProvider found for org '{org_name}' " f"(org_id={org_id}). Setting organization_id=NULL." ) null_prof_sql = text(""" UPDATE marketplace.service_profiles SET organization_id = NULL WHERE id = :profile_id """) await db.execute(null_prof_sql, {"profile_id": profile_id}) profile_skipped += 1 except Exception as e: profile_errors += 1 logger.error( f" ❌ Error relinking ServiceProfile {profile_id}: {e}" ) logger.info( f"\n📊 ServiceProfile Relink Summary: " f"{profile_relinked} relinked, " f"{profile_skipped} skipped (no SP), " f"{profile_errors} errors" ) # ── A3. Also relink any AssetCost that already has service_provider_id set # but still has vendor_organization_id pointing to a fake org ── logger.info("\n📊 A3. Cleaning residual vendor_organization_id on already-linked AssetCosts...") residual_sql = f""" SELECT ac.id, ac.vendor_organization_id, ac.service_provider_id FROM fleet_finance.asset_costs ac WHERE ac.vendor_organization_id IN ({fake_org_ids_str}) AND ac.service_provider_id IS NOT NULL """ residual_result = await db.execute(text(residual_sql)) residual_rows = residual_result.fetchall() residual_cleaned = 0 for res_row in residual_rows: ac_id = res_row[0] try: clean_sql = text(""" UPDATE fleet_finance.asset_costs SET vendor_organization_id = NULL WHERE id = :ac_id """) await db.execute(clean_sql, {"ac_id": ac_id}) residual_cleaned += 1 except Exception as e: logger.error(f" ❌ Error cleaning residual AssetCost {ac_id}: {e}") if residual_cleaned > 0: logger.info(f" ✅ Cleaned {residual_cleaned} residual vendor_organization_id references") else: logger.info(" ℹ️ No residual references found.") # ===================================================================== # STEP B: INCINERATE GARBAGE RECORDS # ===================================================================== logger.info("\n" + "=" * 60) logger.info("STEP B: Incinerate Garbage Records") logger.info("=" * 60) # ── B1. Delete finance.org_subscriptions ── logger.info("\n🗑️ B1. Deleting finance.org_subscriptions...") delete_org_subs_sql = f""" DELETE FROM finance.org_subscriptions WHERE org_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_org_subs_sql)) deleted_org_subs = result.rowcount logger.info(f" ✅ Deleted {deleted_org_subs} org subscriptions") # ── B2. Delete finance.credit_logs ── logger.info("\n🗑️ B2. Deleting finance.credit_logs...") delete_credit_logs_sql = f""" DELETE FROM finance.credit_logs WHERE org_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_credit_logs_sql)) deleted_credit_logs = result.rowcount logger.info(f" ✅ Deleted {deleted_credit_logs} credit logs") # ── B3. Delete fleet.asset_assignments ── logger.info("\n🗑️ B3. Deleting fleet.asset_assignments...") delete_assignments_sql = f""" DELETE FROM fleet.asset_assignments WHERE organization_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_assignments_sql)) deleted_assignments = result.rowcount logger.info(f" ✅ Deleted {deleted_assignments} asset assignments") # ── B4. NULL out vehicle.assets org references ── logger.info("\n🗑️ B4. NULLing vehicle.assets org references...") null_vehicle_current_sql = f""" UPDATE vehicle.assets SET current_organization_id = NULL WHERE current_organization_id IN ({fake_org_ids_str}) """ result = await db.execute(text(null_vehicle_current_sql)) nulled_current = result.rowcount null_vehicle_operator_sql = f""" UPDATE vehicle.assets SET operator_org_id = NULL WHERE operator_org_id IN ({fake_org_ids_str}) """ result = await db.execute(text(null_vehicle_operator_sql)) nulled_operator = result.rowcount null_vehicle_owner_sql = f""" UPDATE vehicle.assets SET owner_org_id = NULL WHERE owner_org_id IN ({fake_org_ids_str}) """ result = await db.execute(text(null_vehicle_owner_sql)) nulled_owner = result.rowcount logger.info( f" ✅ NULLed vehicle.assets references: " f"{nulled_current} current_org, " f"{nulled_operator} operator_org, " f"{nulled_owner} owner_org" ) # ── B5. Reassign fleet_finance.asset_costs organization_id to a real org ── # NOTE: organization_id has a NOT NULL constraint, so we must reassign to a real org logger.info("\n🗑️ B5. Reassigning fleet_finance.asset_costs organization_id to default org...") # Use org ID=15 (Profibot Test Fleet) as the default reassignment target reassign_ac_org_sql = f""" UPDATE fleet_finance.asset_costs SET organization_id = 15 WHERE organization_id IN ({fake_org_ids_str}) """ result = await db.execute(text(reassign_ac_org_sql)) reassigned_ac_org = result.rowcount logger.info(f" ✅ Reassigned {reassigned_ac_org} asset_costs organization_id to org_id=15") # ── B6. Delete marketplace.ratings ── logger.info("\n🗑️ B6. Deleting marketplace.ratings...") delete_ratings_sql = f""" DELETE FROM marketplace.ratings WHERE target_organization_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_ratings_sql)) deleted_ratings = result.rowcount logger.info(f" ✅ Deleted {deleted_ratings} ratings") # ── B7. Delete branches ── logger.info("\n🗑️ B7. Deleting branches...") delete_branches_sql = f""" DELETE FROM fleet.branches WHERE organization_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_branches_sql)) deleted_branches = result.rowcount logger.info(f" ✅ Deleted {deleted_branches} branches") # ── B8. Delete organization_members ── logger.info("\n🗑️ B8. Deleting organization_members...") delete_members_sql = f""" DELETE FROM fleet.organization_members WHERE organization_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_members_sql)) deleted_members = result.rowcount logger.info(f" ✅ Deleted {deleted_members} organization members") # ── B9. Delete contact_persons ── logger.info("\n🗑️ B9. Deleting contact_persons...") delete_contacts_sql = f""" DELETE FROM fleet.contact_persons WHERE organization_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_contacts_sql)) deleted_contacts = result.rowcount logger.info(f" ✅ Deleted {deleted_contacts} contact persons") # ── B10. Delete org_relationships ── logger.info("\n🗑️ B10. Deleting org_relationships...") delete_rels_sql = f""" DELETE FROM fleet.org_relationships WHERE source_org_id IN ({fake_org_ids_str}) OR target_org_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_rels_sql)) deleted_rels = result.rowcount logger.info(f" ✅ Deleted {deleted_rels} org relationships") # ── B11. Delete organization_financials ── logger.info("\n🗑️ B11. Deleting organization_financials...") delete_financials_sql = f""" DELETE FROM fleet.organization_financials WHERE organization_id IN ({fake_org_ids_str}) """ result = await db.execute(text(delete_financials_sql)) deleted_financials = result.rowcount logger.info(f" ✅ Deleted {deleted_financials} organization financials") # ── B12. Delete org_sales_assignments ── logger.info("\n🗑️ B12. Deleting org_sales_assignments...") delete_sales_sql = f""" DELETE FROM fleet.org_sales_assignments WHERE organization_id IN ({fake_org_ids_str}) """ try: result = await db.execute(text(delete_sales_sql)) deleted_sales = result.rowcount logger.info(f" ✅ Deleted {deleted_sales} sales assignments") except Exception as e: logger.warning(f" ⚠️ Error deleting sales assignments: {e}") deleted_sales = 0 # ===================================================================== # STEP C: THE FINAL PURGE # ===================================================================== logger.info("\n" + "=" * 60) logger.info("STEP C: The Final Purge — Deleting fake organizations") logger.info("=" * 60) # ── C1. Delete the organizations themselves ── logger.info("\n🗑️ C1. Deleting fake organizations...") delete_orgs_sql = f""" DELETE FROM fleet.organizations WHERE id IN ({fake_org_ids_str}) RETURNING id, name """ result = await db.execute(text(delete_orgs_sql)) deleted_orgs = result.fetchall() deleted_orgs_count = len(deleted_orgs) logger.info(f" ✅ Deleted {deleted_orgs_count} organizations:") for org_id, org_name in deleted_orgs: logger.info(f" - ID={org_id}: {org_name}") # ── Commit the transaction ── await db.commit() # ===================================================================== # REPORT # ===================================================================== logger.info("\n" + "=" * 60) logger.info("📋 FINAL PURGE REPORT") logger.info("=" * 60) logger.info(f"\n📊 Financial Records Relinked:") logger.info(f" AssetCost records relinked: {asset_cost_relinked}") logger.info(f" AssetCost skipped (no SP): {asset_cost_skipped}") logger.info(f" AssetCost errors: {asset_cost_errors}") logger.info(f" Residual vendor_org cleaned: {residual_cleaned}") logger.info(f" ServiceProfile records relinked: {profile_relinked}") logger.info(f" ServiceProfile skipped (no SP): {profile_skipped}") logger.info(f" ServiceProfile errors: {profile_errors}") logger.info(f"\n🗑️ Garbage Records Destroyed:") logger.info(f" Org Subscriptions: {deleted_org_subs}") logger.info(f" Credit Logs: {deleted_credit_logs}") logger.info(f" Asset Assignments: {deleted_assignments}") logger.info(f" Vehicle Assets (current_org): {nulled_current}") logger.info(f" Vehicle Assets (operator_org): {nulled_operator}") logger.info(f" Vehicle Assets (owner_org): {nulled_owner}") logger.info(f" AssetCosts (organization_id): {reassigned_ac_org}") logger.info(f" Ratings: {deleted_ratings}") logger.info(f" Branches: {deleted_branches}") logger.info(f" Organization Members: {deleted_members}") logger.info(f" Contact Persons: {deleted_contacts}") logger.info(f" Org Relationships: {deleted_rels}") logger.info(f" Organization Financials: {deleted_financials}") logger.info(f" Sales Assignments: {deleted_sales}") logger.info(f"\n💀 Organizations Purged:") logger.info(f" Total fake orgs deleted: {deleted_orgs_count}") total_garbage = ( deleted_org_subs + deleted_credit_logs + deleted_assignments + deleted_branches + deleted_members + deleted_contacts + deleted_rels + deleted_financials + deleted_sales + deleted_ratings ) logger.info(f"\n📈 Total garbage rows destroyed: {total_garbage}") logger.info(f"\n✅ Deep purge completed successfully!") # ── Verification ── logger.info("\n🔍 Verifying cleanup...") verify_sql = f""" SELECT COUNT(*) FROM fleet.organizations WHERE id NOT IN ({safe_list}) AND is_deleted = false """ verify_result = await db.execute(text(verify_sql)) remaining = verify_result.scalar() if remaining == 0: logger.info("✅ Verification PASSED: No fake organizations remain.") else: logger.warning(f"⚠️ Verification: {remaining} fake organizations still remain!") return { "asset_cost_relinked": asset_cost_relinked, "asset_cost_skipped": asset_cost_skipped, "asset_cost_errors": asset_cost_errors, "residual_cleaned": residual_cleaned, "profile_relinked": profile_relinked, "profile_skipped": profile_skipped, "profile_errors": profile_errors, "deleted_org_subs": deleted_org_subs, "deleted_credit_logs": deleted_credit_logs, "deleted_assignments": deleted_assignments, "nulled_current": nulled_current, "nulled_operator": nulled_operator, "nulled_owner": nulled_owner, "reassigned_ac_org": reassigned_ac_org, "deleted_ratings": deleted_ratings, "deleted_branches": deleted_branches, "deleted_members": deleted_members, "deleted_contacts": deleted_contacts, "deleted_rels": deleted_rels, "deleted_financials": deleted_financials, "deleted_sales": deleted_sales, "deleted_orgs": deleted_orgs_count, "total_garbage": total_garbage, } except Exception as e: await db.rollback() logger.error(f"💥 Fatal error during purge: {e}", exc_info=True) sys.exit(1) async def main(): logger.info("🚀 P0 DATABASE PURGE & RELINK — Deep Clean Script") logger.info(f" Safe org IDs: {SAFE_ORG_IDS}") logger.info(f" Database: {DATABASE_URL}") logger.info("") try: await deep_purge_fake_orgs() except Exception as e: logger.error(f"💥 Fatal error: {e}") sys.exit(1) if __name__ == "__main__": asyncio.run(main())