Files
service-finder/backend/scripts/deep_purge_fake_orgs.py

599 lines
28 KiB
Python
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
P0 DATABASE PURGE & RELINK — Deep Clean & Relink Script
CRITICAL DIRECTIVE: Complete database hygiene. Before deleting fake organizations,
we MUST perfectly re-link all financial data (AssetCost/invoices) to their new
marketplace.service_providers counterparts. All garbage records (branches, members)
tied to the fake orgs must be completely destroyed.
Safe List (valódi garázsok, amelyek NEM kerülnek törlésre):
(15, 21, 43, 45, 47, 48, 49, 50, 66, 67, 69, 70, 71)
Folyamat:
Step A (The Financial Relink):
1. Iterate through fleet_finance.asset_costs where vendor_organization_id is a fake org.
2. Find the matching service_provider_id in marketplace.service_providers (match by name).
3. Set AssetCost.service_provider_id to this new ID.
4. ONLY THEN set AssetCost.vendor_organization_id = NULL.
5. Do the exact same secure relinking for marketplace.service_profiles.organization_id
-> map to service_provider_id, then set organization_id = NULL.
Step B (Incinerate Garbage Records):
1. DELETE FROM finance.org_subscriptions WHERE org_id is a fake org.
2. DELETE FROM finance.credit_logs WHERE org_id is a fake org.
3. DELETE FROM fleet.asset_assignments WHERE organization_id is a fake org.
4. UPDATE vehicle.assets SET current_organization_id=NULL, operator_org_id=NULL,
owner_org_id=NULL WHERE they reference fake orgs.
5. UPDATE fleet_finance.asset_costs SET organization_id=NULL WHERE org is fake.
6. DELETE FROM marketplace.ratings WHERE target_organization_id is a fake org.
7. DELETE FROM fleet.branches WHERE organization_id is a fake org.
8. DELETE FROM fleet.organization_members WHERE organization_id is a fake org.
9. DELETE FROM fleet.contact_persons WHERE organization_id is a fake org.
10. DELETE FROM fleet.org_relationships WHERE source_org_id or target_org_id is a fake org.
11. DELETE FROM fleet.organization_financials WHERE organization_id is a fake org.
12. DELETE FROM fleet.org_sales_assignments WHERE organization_id is a fake org.
Step C (The Final Purge):
1. DELETE FROM fleet.organizations WHERE id is a fake org.
Használat:
docker compose exec sf_api python3 /app/scripts/deep_purge_fake_orgs.py
"""
import asyncio
import logging
import sys
from datetime import datetime, timezone
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
# Import settings
sys.path.insert(0, '/app')
from app.core.config import settings
# ──────────────────────────────────────────────
# Konfiguráció
# ──────────────────────────────────────────────
# Safe list: ezek az Organization ID-k valódi garázsok, NEM töröljük őket
SAFE_ORG_IDS = (15, 21, 43, 45, 47, 48, 49, 50, 66, 67, 69, 70, 71)
# Adatbázis URL a settings-ből
DATABASE_URL = settings.DATABASE_URL
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[logging.StreamHandler(sys.stdout)],
)
logger = logging.getLogger(__name__)
async def deep_purge_fake_orgs():
"""
Main purge logic — three steps: Relink, Incinerate, Purge.
"""
engine = create_async_engine(DATABASE_URL, echo=False)
async with AsyncSession(engine) as db:
try:
# ── 0. Identify fake orgs ──
safe_list = ", ".join([str(x) for x in SAFE_ORG_IDS])
fake_orgs_sql = f"""
SELECT id, name
FROM fleet.organizations
WHERE id NOT IN ({safe_list})
AND is_deleted = false
ORDER BY id
"""
result = await db.execute(text(fake_orgs_sql))
fake_orgs = result.fetchall()
if not fake_orgs:
logger.info("✅ No fake organizations found. Nothing to do.")
return
fake_org_ids = [str(org[0]) for org in fake_orgs]
fake_org_ids_str = ", ".join(fake_org_ids)
logger.info(f"🔍 Found {len(fake_orgs)} fake organizations to purge:")
for org_id, org_name in fake_orgs:
logger.info(f" - ID={org_id}: {org_name}")
# =====================================================================
# STEP A: THE FINANCIAL RELINK
# =====================================================================
logger.info("\n" + "=" * 60)
logger.info("STEP A: Financial Relink")
logger.info("=" * 60)
# ── A1. Relink AssetCost records ──
logger.info("\n📊 A1. Relinking AssetCost records...")
# Get all AssetCost records with vendor_organization_id pointing to fake orgs
asset_costs_sql = f"""
SELECT ac.id, ac.vendor_organization_id, ac.service_provider_id,
o.name as org_name
FROM fleet_finance.asset_costs ac
JOIN fleet.organizations o ON o.id = ac.vendor_organization_id
WHERE ac.vendor_organization_id IN ({fake_org_ids_str})
AND ac.service_provider_id IS NULL
"""
ac_result = await db.execute(text(asset_costs_sql))
asset_costs = ac_result.fetchall()
asset_cost_relinked = 0
asset_cost_skipped = 0
asset_cost_errors = 0
for ac_row in asset_costs:
ac_id = ac_row[0]
vendor_org_id = ac_row[1]
org_name = ac_row[3]
try:
# Find matching service_provider by name
find_sp_sql = text("""
SELECT id FROM marketplace.service_providers
WHERE name = :org_name
LIMIT 1
""")
sp_result = await db.execute(find_sp_sql, {"org_name": org_name})
sp_row = sp_result.fetchone()
if sp_row:
sp_id = sp_row[0]
# Set service_provider_id, then NULL the vendor_organization_id
update_ac_sql = text("""
UPDATE fleet_finance.asset_costs
SET service_provider_id = :sp_id,
vendor_organization_id = NULL
WHERE id = :ac_id
""")
await db.execute(update_ac_sql, {"sp_id": sp_id, "ac_id": ac_id})
asset_cost_relinked += 1
logger.info(
f" ✅ AssetCost {ac_id}: vendor_org={vendor_org_id} -> "
f"service_provider_id={sp_id}"
)
else:
# No matching service_provider found — just NULL the vendor_org
logger.warning(
f" ⚠️ No ServiceProvider found for org '{org_name}' "
f"(vendor_org_id={vendor_org_id}). Setting vendor_organization_id=NULL."
)
null_ac_sql = text("""
UPDATE fleet_finance.asset_costs
SET vendor_organization_id = NULL
WHERE id = :ac_id
""")
await db.execute(null_ac_sql, {"ac_id": ac_id})
asset_cost_skipped += 1
except Exception as e:
asset_cost_errors += 1
logger.error(
f" ❌ Error relinking AssetCost {ac_id}: {e}"
)
logger.info(
f"\n📊 AssetCost Relink Summary: "
f"{asset_cost_relinked} relinked, "
f"{asset_cost_skipped} skipped (no SP), "
f"{asset_cost_errors} errors"
)
# ── A2. Relink ServiceProfile records ──
logger.info("\n📊 A2. Relinking ServiceProfile records...")
profiles_sql = f"""
SELECT sp.id, sp.organization_id, sp.service_provider_id,
o.name as org_name
FROM marketplace.service_profiles sp
JOIN fleet.organizations o ON o.id = sp.organization_id
WHERE sp.organization_id IN ({fake_org_ids_str})
"""
prof_result = await db.execute(text(profiles_sql))
profiles = prof_result.fetchall()
profile_relinked = 0
profile_skipped = 0
profile_errors = 0
for prof_row in profiles:
profile_id = prof_row[0]
org_id = prof_row[1]
existing_sp_id = prof_row[2]
org_name = prof_row[3]
try:
if existing_sp_id is not None:
# Already has service_provider_id — just NULL the organization_id
null_prof_sql = text("""
UPDATE marketplace.service_profiles
SET organization_id = NULL
WHERE id = :profile_id
""")
await db.execute(null_prof_sql, {"profile_id": profile_id})
profile_relinked += 1
logger.info(
f" ✅ ServiceProfile {profile_id}: org={org_id} -> "
f"already had sp_id={existing_sp_id}, NULLed org"
)
else:
# Find matching service_provider by name
find_sp_sql = text("""
SELECT id FROM marketplace.service_providers
WHERE name = :org_name
LIMIT 1
""")
sp_result = await db.execute(find_sp_sql, {"org_name": org_name})
sp_row = sp_result.fetchone()
if sp_row:
sp_id = sp_row[0]
# Set service_provider_id, then NULL the organization_id
update_prof_sql = text("""
UPDATE marketplace.service_profiles
SET service_provider_id = :sp_id,
organization_id = NULL
WHERE id = :profile_id
""")
await db.execute(
update_prof_sql,
{"sp_id": sp_id, "profile_id": profile_id}
)
profile_relinked += 1
logger.info(
f" ✅ ServiceProfile {profile_id}: org={org_id} -> "
f"service_provider_id={sp_id}"
)
else:
# No matching service_provider — just NULL the org
logger.warning(
f" ⚠️ No ServiceProvider found for org '{org_name}' "
f"(org_id={org_id}). Setting organization_id=NULL."
)
null_prof_sql = text("""
UPDATE marketplace.service_profiles
SET organization_id = NULL
WHERE id = :profile_id
""")
await db.execute(null_prof_sql, {"profile_id": profile_id})
profile_skipped += 1
except Exception as e:
profile_errors += 1
logger.error(
f" ❌ Error relinking ServiceProfile {profile_id}: {e}"
)
logger.info(
f"\n📊 ServiceProfile Relink Summary: "
f"{profile_relinked} relinked, "
f"{profile_skipped} skipped (no SP), "
f"{profile_errors} errors"
)
# ── A3. Also relink any AssetCost that already has service_provider_id set
# but still has vendor_organization_id pointing to a fake org ──
logger.info("\n📊 A3. Cleaning residual vendor_organization_id on already-linked AssetCosts...")
residual_sql = f"""
SELECT ac.id, ac.vendor_organization_id, ac.service_provider_id
FROM fleet_finance.asset_costs ac
WHERE ac.vendor_organization_id IN ({fake_org_ids_str})
AND ac.service_provider_id IS NOT NULL
"""
residual_result = await db.execute(text(residual_sql))
residual_rows = residual_result.fetchall()
residual_cleaned = 0
for res_row in residual_rows:
ac_id = res_row[0]
try:
clean_sql = text("""
UPDATE fleet_finance.asset_costs
SET vendor_organization_id = NULL
WHERE id = :ac_id
""")
await db.execute(clean_sql, {"ac_id": ac_id})
residual_cleaned += 1
except Exception as e:
logger.error(f" ❌ Error cleaning residual AssetCost {ac_id}: {e}")
if residual_cleaned > 0:
logger.info(f" ✅ Cleaned {residual_cleaned} residual vendor_organization_id references")
else:
logger.info(" No residual references found.")
# =====================================================================
# STEP B: INCINERATE GARBAGE RECORDS
# =====================================================================
logger.info("\n" + "=" * 60)
logger.info("STEP B: Incinerate Garbage Records")
logger.info("=" * 60)
# ── B1. Delete finance.org_subscriptions ──
logger.info("\n🗑️ B1. Deleting finance.org_subscriptions...")
delete_org_subs_sql = f"""
DELETE FROM finance.org_subscriptions
WHERE org_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_org_subs_sql))
deleted_org_subs = result.rowcount
logger.info(f" ✅ Deleted {deleted_org_subs} org subscriptions")
# ── B2. Delete finance.credit_logs ──
logger.info("\n🗑️ B2. Deleting finance.credit_logs...")
delete_credit_logs_sql = f"""
DELETE FROM finance.credit_logs
WHERE org_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_credit_logs_sql))
deleted_credit_logs = result.rowcount
logger.info(f" ✅ Deleted {deleted_credit_logs} credit logs")
# ── B3. Delete fleet.asset_assignments ──
logger.info("\n🗑️ B3. Deleting fleet.asset_assignments...")
delete_assignments_sql = f"""
DELETE FROM fleet.asset_assignments
WHERE organization_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_assignments_sql))
deleted_assignments = result.rowcount
logger.info(f" ✅ Deleted {deleted_assignments} asset assignments")
# ── B4. NULL out vehicle.assets org references ──
logger.info("\n🗑️ B4. NULLing vehicle.assets org references...")
null_vehicle_current_sql = f"""
UPDATE vehicle.assets
SET current_organization_id = NULL
WHERE current_organization_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(null_vehicle_current_sql))
nulled_current = result.rowcount
null_vehicle_operator_sql = f"""
UPDATE vehicle.assets
SET operator_org_id = NULL
WHERE operator_org_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(null_vehicle_operator_sql))
nulled_operator = result.rowcount
null_vehicle_owner_sql = f"""
UPDATE vehicle.assets
SET owner_org_id = NULL
WHERE owner_org_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(null_vehicle_owner_sql))
nulled_owner = result.rowcount
logger.info(
f" ✅ NULLed vehicle.assets references: "
f"{nulled_current} current_org, "
f"{nulled_operator} operator_org, "
f"{nulled_owner} owner_org"
)
# ── B5. Reassign fleet_finance.asset_costs organization_id to a real org ──
# NOTE: organization_id has a NOT NULL constraint, so we must reassign to a real org
logger.info("\n🗑️ B5. Reassigning fleet_finance.asset_costs organization_id to default org...")
# Use org ID=15 (Profibot Test Fleet) as the default reassignment target
reassign_ac_org_sql = f"""
UPDATE fleet_finance.asset_costs
SET organization_id = 15
WHERE organization_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(reassign_ac_org_sql))
reassigned_ac_org = result.rowcount
logger.info(f" ✅ Reassigned {reassigned_ac_org} asset_costs organization_id to org_id=15")
# ── B6. Delete marketplace.ratings ──
logger.info("\n🗑️ B6. Deleting marketplace.ratings...")
delete_ratings_sql = f"""
DELETE FROM marketplace.ratings
WHERE target_organization_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_ratings_sql))
deleted_ratings = result.rowcount
logger.info(f" ✅ Deleted {deleted_ratings} ratings")
# ── B7. Delete branches ──
logger.info("\n🗑️ B7. Deleting branches...")
delete_branches_sql = f"""
DELETE FROM fleet.branches
WHERE organization_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_branches_sql))
deleted_branches = result.rowcount
logger.info(f" ✅ Deleted {deleted_branches} branches")
# ── B8. Delete organization_members ──
logger.info("\n🗑️ B8. Deleting organization_members...")
delete_members_sql = f"""
DELETE FROM fleet.organization_members
WHERE organization_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_members_sql))
deleted_members = result.rowcount
logger.info(f" ✅ Deleted {deleted_members} organization members")
# ── B9. Delete contact_persons ──
logger.info("\n🗑️ B9. Deleting contact_persons...")
delete_contacts_sql = f"""
DELETE FROM fleet.contact_persons
WHERE organization_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_contacts_sql))
deleted_contacts = result.rowcount
logger.info(f" ✅ Deleted {deleted_contacts} contact persons")
# ── B10. Delete org_relationships ──
logger.info("\n🗑️ B10. Deleting org_relationships...")
delete_rels_sql = f"""
DELETE FROM fleet.org_relationships
WHERE source_org_id IN ({fake_org_ids_str})
OR target_org_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_rels_sql))
deleted_rels = result.rowcount
logger.info(f" ✅ Deleted {deleted_rels} org relationships")
# ── B11. Delete organization_financials ──
logger.info("\n🗑️ B11. Deleting organization_financials...")
delete_financials_sql = f"""
DELETE FROM fleet.organization_financials
WHERE organization_id IN ({fake_org_ids_str})
"""
result = await db.execute(text(delete_financials_sql))
deleted_financials = result.rowcount
logger.info(f" ✅ Deleted {deleted_financials} organization financials")
# ── B12. Delete org_sales_assignments ──
logger.info("\n🗑️ B12. Deleting org_sales_assignments...")
delete_sales_sql = f"""
DELETE FROM fleet.org_sales_assignments
WHERE organization_id IN ({fake_org_ids_str})
"""
try:
result = await db.execute(text(delete_sales_sql))
deleted_sales = result.rowcount
logger.info(f" ✅ Deleted {deleted_sales} sales assignments")
except Exception as e:
logger.warning(f" ⚠️ Error deleting sales assignments: {e}")
deleted_sales = 0
# =====================================================================
# STEP C: THE FINAL PURGE
# =====================================================================
logger.info("\n" + "=" * 60)
logger.info("STEP C: The Final Purge — Deleting fake organizations")
logger.info("=" * 60)
# ── C1. Delete the organizations themselves ──
logger.info("\n🗑️ C1. Deleting fake organizations...")
delete_orgs_sql = f"""
DELETE FROM fleet.organizations
WHERE id IN ({fake_org_ids_str})
RETURNING id, name
"""
result = await db.execute(text(delete_orgs_sql))
deleted_orgs = result.fetchall()
deleted_orgs_count = len(deleted_orgs)
logger.info(f" ✅ Deleted {deleted_orgs_count} organizations:")
for org_id, org_name in deleted_orgs:
logger.info(f" - ID={org_id}: {org_name}")
# ── Commit the transaction ──
await db.commit()
# =====================================================================
# REPORT
# =====================================================================
logger.info("\n" + "=" * 60)
logger.info("📋 FINAL PURGE REPORT")
logger.info("=" * 60)
logger.info(f"\n📊 Financial Records Relinked:")
logger.info(f" AssetCost records relinked: {asset_cost_relinked}")
logger.info(f" AssetCost skipped (no SP): {asset_cost_skipped}")
logger.info(f" AssetCost errors: {asset_cost_errors}")
logger.info(f" Residual vendor_org cleaned: {residual_cleaned}")
logger.info(f" ServiceProfile records relinked: {profile_relinked}")
logger.info(f" ServiceProfile skipped (no SP): {profile_skipped}")
logger.info(f" ServiceProfile errors: {profile_errors}")
logger.info(f"\n🗑️ Garbage Records Destroyed:")
logger.info(f" Org Subscriptions: {deleted_org_subs}")
logger.info(f" Credit Logs: {deleted_credit_logs}")
logger.info(f" Asset Assignments: {deleted_assignments}")
logger.info(f" Vehicle Assets (current_org): {nulled_current}")
logger.info(f" Vehicle Assets (operator_org): {nulled_operator}")
logger.info(f" Vehicle Assets (owner_org): {nulled_owner}")
logger.info(f" AssetCosts (organization_id): {reassigned_ac_org}")
logger.info(f" Ratings: {deleted_ratings}")
logger.info(f" Branches: {deleted_branches}")
logger.info(f" Organization Members: {deleted_members}")
logger.info(f" Contact Persons: {deleted_contacts}")
logger.info(f" Org Relationships: {deleted_rels}")
logger.info(f" Organization Financials: {deleted_financials}")
logger.info(f" Sales Assignments: {deleted_sales}")
logger.info(f"\n💀 Organizations Purged:")
logger.info(f" Total fake orgs deleted: {deleted_orgs_count}")
total_garbage = (
deleted_org_subs + deleted_credit_logs + deleted_assignments
+ deleted_branches + deleted_members + deleted_contacts
+ deleted_rels + deleted_financials + deleted_sales
+ deleted_ratings
)
logger.info(f"\n📈 Total garbage rows destroyed: {total_garbage}")
logger.info(f"\n✅ Deep purge completed successfully!")
# ── Verification ──
logger.info("\n🔍 Verifying cleanup...")
verify_sql = f"""
SELECT COUNT(*) FROM fleet.organizations
WHERE id NOT IN ({safe_list})
AND is_deleted = false
"""
verify_result = await db.execute(text(verify_sql))
remaining = verify_result.scalar()
if remaining == 0:
logger.info("✅ Verification PASSED: No fake organizations remain.")
else:
logger.warning(f"⚠️ Verification: {remaining} fake organizations still remain!")
return {
"asset_cost_relinked": asset_cost_relinked,
"asset_cost_skipped": asset_cost_skipped,
"asset_cost_errors": asset_cost_errors,
"residual_cleaned": residual_cleaned,
"profile_relinked": profile_relinked,
"profile_skipped": profile_skipped,
"profile_errors": profile_errors,
"deleted_org_subs": deleted_org_subs,
"deleted_credit_logs": deleted_credit_logs,
"deleted_assignments": deleted_assignments,
"nulled_current": nulled_current,
"nulled_operator": nulled_operator,
"nulled_owner": nulled_owner,
"reassigned_ac_org": reassigned_ac_org,
"deleted_ratings": deleted_ratings,
"deleted_branches": deleted_branches,
"deleted_members": deleted_members,
"deleted_contacts": deleted_contacts,
"deleted_rels": deleted_rels,
"deleted_financials": deleted_financials,
"deleted_sales": deleted_sales,
"deleted_orgs": deleted_orgs_count,
"total_garbage": total_garbage,
}
except Exception as e:
await db.rollback()
logger.error(f"💥 Fatal error during purge: {e}", exc_info=True)
sys.exit(1)
async def main():
logger.info("🚀 P0 DATABASE PURGE & RELINK — Deep Clean Script")
logger.info(f" Safe org IDs: {SAFE_ORG_IDS}")
logger.info(f" Database: {DATABASE_URL}")
logger.info("")
try:
await deep_purge_fake_orgs()
except Exception as e:
logger.error(f"💥 Fatal error: {e}")
sys.exit(1)
if __name__ == "__main__":
asyncio.run(main())