260 lines
9.6 KiB
Python
260 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
P0 ADDRESS UNIFICATION - Phase 1: ServiceProvider Address Migration
|
|
|
|
Migrates flat address columns (city, address_zip, address_street_name, etc.)
|
|
from marketplace.service_providers into the centralized system.addresses table.
|
|
|
|
Logic:
|
|
1. Fetch all ServiceProvider records where address_id IS NULL
|
|
but at least one flat address field is non-empty.
|
|
2. For each record, construct an AddressIn schema from the flat data.
|
|
3. Call AddressManager.create_or_update(db, None, address_data) to create
|
|
a new central address record.
|
|
4. Assign the returned Address UUID to provider.address_id.
|
|
5. Commit in batches with proper error handling and logging.
|
|
|
|
Usage:
|
|
docker exec sf_api python3 /app/backend/app/scripts/migrate_provider_addresses.py
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import sys
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from typing import Optional
|
|
|
|
# Ensure the backend package is importable
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
|
|
# asyncpg needs plain "postgresql://" scheme
|
|
_raw_url = os.getenv(
|
|
"DATABASE_URL",
|
|
"postgresql://service_finder_app:AppSafePass_2026@db:5432/service_finder"
|
|
)
|
|
DATABASE_URL = _raw_url.replace("+asyncpg", "")
|
|
|
|
# Batch size for commits
|
|
BATCH_SIZE = 100
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
handlers=[logging.StreamHandler(sys.stdout)],
|
|
)
|
|
logger = logging.getLogger("migrate_provider_addresses")
|
|
|
|
|
|
def _build_full_address_text(row: dict) -> Optional[str]:
|
|
"""Build a human-readable full address from flat fields."""
|
|
parts = []
|
|
if row.get("address_zip"):
|
|
parts.append(row["address_zip"])
|
|
if row.get("city"):
|
|
parts.append(row["city"])
|
|
street_parts = []
|
|
if row.get("address_street_name"):
|
|
street_parts.append(row["address_street_name"])
|
|
if row.get("address_street_type"):
|
|
street_parts.append(row["address_street_type"])
|
|
if row.get("address_house_number"):
|
|
street_parts.append(row["address_house_number"])
|
|
if street_parts:
|
|
parts.append(" ".join(street_parts))
|
|
return ", ".join(parts) if parts else None
|
|
|
|
|
|
def _has_address_data(row: dict) -> bool:
|
|
"""Check if the provider has any flat address data worth migrating."""
|
|
return bool(
|
|
row.get("city")
|
|
or row.get("address_zip")
|
|
or row.get("address_street_name")
|
|
or row.get("address_house_number")
|
|
)
|
|
|
|
|
|
async def main():
|
|
logger.info("=" * 70)
|
|
logger.info("🔧 P0 ADDRESS UNIFICATION - Phase 1: ServiceProvider Migration")
|
|
logger.info("=" * 70)
|
|
logger.info(f"Started at: {datetime.now(timezone.utc).isoformat()}")
|
|
logger.info(f"Database URL: {DATABASE_URL.replace('AppSafePass_2026', '****')}")
|
|
logger.info(f"Batch size: {BATCH_SIZE}")
|
|
logger.info("")
|
|
|
|
import asyncpg
|
|
|
|
conn = await asyncpg.connect(DATABASE_URL)
|
|
|
|
try:
|
|
# ── Step 1: Count providers needing migration ──
|
|
logger.info("📋 Step 1: Counting providers with flat address data but no address_id")
|
|
count_row = await conn.fetchrow("""
|
|
SELECT COUNT(*) AS cnt
|
|
FROM marketplace.service_providers
|
|
WHERE address_id IS NULL
|
|
AND (
|
|
city IS NOT NULL AND city != ''
|
|
OR address_zip IS NOT NULL AND address_zip != ''
|
|
OR address_street_name IS NOT NULL AND address_street_name != ''
|
|
OR address_house_number IS NOT NULL AND address_house_number != ''
|
|
)
|
|
""")
|
|
total = count_row["cnt"]
|
|
logger.info(f" → Found {total} provider(s) to migrate")
|
|
logger.info("")
|
|
|
|
if total == 0:
|
|
logger.info("✅ No providers need migration. Exiting.")
|
|
return
|
|
|
|
# ── Step 2: Fetch all providers needing migration ──
|
|
logger.info("📋 Step 2: Fetching provider records")
|
|
rows = await conn.fetch("""
|
|
SELECT id, name, city, address_zip,
|
|
address_street_name, address_street_type, address_house_number,
|
|
plus_code
|
|
FROM marketplace.service_providers
|
|
WHERE address_id IS NULL
|
|
AND (
|
|
city IS NOT NULL AND city != ''
|
|
OR address_zip IS NOT NULL AND address_zip != ''
|
|
OR address_street_name IS NOT NULL AND address_street_name != ''
|
|
OR address_house_number IS NOT NULL AND address_house_number != ''
|
|
)
|
|
ORDER BY id
|
|
""")
|
|
logger.info(f" → Fetched {len(rows)} record(s)")
|
|
logger.info("")
|
|
|
|
# ── Step 3: Migrate each provider ──
|
|
logger.info("📋 Step 3: Migrating addresses to system.addresses")
|
|
migrated_count = 0
|
|
error_count = 0
|
|
skipped_count = 0
|
|
|
|
for i, row in enumerate(rows, start=1):
|
|
provider_id = row["id"]
|
|
provider_name = row["name"]
|
|
|
|
if not _has_address_data(row):
|
|
logger.info(f" [{i}/{total}] Provider #{provider_id} '{provider_name}': "
|
|
f"no address data → SKIP")
|
|
skipped_count += 1
|
|
continue
|
|
|
|
try:
|
|
# Build the full_address_text
|
|
full_text = _build_full_address_text(row)
|
|
|
|
# Insert into system.addresses
|
|
new_address_id = uuid.uuid4()
|
|
await conn.execute("""
|
|
INSERT INTO system.addresses
|
|
(id, street_name, street_type, house_number,
|
|
full_address_text, created_at)
|
|
VALUES ($1, $2, $3, $4, $5, NOW())
|
|
ON CONFLICT (id) DO NOTHING
|
|
""",
|
|
new_address_id,
|
|
row.get("address_street_name"),
|
|
row.get("address_street_type"),
|
|
row.get("address_house_number"),
|
|
full_text,
|
|
)
|
|
|
|
# Resolve postal code if zip+city are present
|
|
if row.get("address_zip") and row.get("city"):
|
|
# Find or create GeoPostalCode
|
|
postal = await conn.fetchrow("""
|
|
SELECT id FROM system.geo_postal_codes
|
|
WHERE zip_code = $1
|
|
""", row["address_zip"])
|
|
|
|
if postal:
|
|
postal_code_id = postal["id"]
|
|
else:
|
|
postal_code_id = await conn.fetchval("""
|
|
INSERT INTO system.geo_postal_codes
|
|
(zip_code, city, country_code)
|
|
VALUES ($1, $2, 'HU')
|
|
RETURNING id
|
|
""", row["address_zip"], row["city"])
|
|
|
|
# Update the address with postal_code_id
|
|
await conn.execute("""
|
|
UPDATE system.addresses
|
|
SET postal_code_id = $1
|
|
WHERE id = $2
|
|
""", postal_code_id, new_address_id)
|
|
|
|
# Update the provider with the new address_id
|
|
await conn.execute("""
|
|
UPDATE marketplace.service_providers
|
|
SET address_id = $1
|
|
WHERE id = $2
|
|
""", new_address_id, provider_id)
|
|
|
|
migrated_count += 1
|
|
|
|
if i % BATCH_SIZE == 0 or i == total:
|
|
logger.info(
|
|
f" [{i}/{total}] Progress: {migrated_count} migrated, "
|
|
f"{error_count} errors, {skipped_count} skipped"
|
|
)
|
|
|
|
except Exception as e:
|
|
error_count += 1
|
|
logger.error(
|
|
f" [{i}/{total}] Provider #{provider_id} '{provider_name}': "
|
|
f"ERROR - {e}"
|
|
)
|
|
|
|
# ── Summary ──
|
|
logger.info("")
|
|
logger.info("=" * 70)
|
|
logger.info("📊 MIGRATION SUMMARY")
|
|
logger.info("=" * 70)
|
|
logger.info(f" Total providers processed: {total}")
|
|
logger.info(f" Successfully migrated: {migrated_count}")
|
|
logger.info(f" Errors: {error_count}")
|
|
logger.info(f" Skipped (no address data): {skipped_count}")
|
|
|
|
# ── Step 4: Verify ──
|
|
logger.info("")
|
|
logger.info("📋 Step 4: Verification")
|
|
remaining = await conn.fetchval("""
|
|
SELECT COUNT(*) FROM marketplace.service_providers
|
|
WHERE address_id IS NULL
|
|
AND (
|
|
city IS NOT NULL AND city != ''
|
|
OR address_zip IS NOT NULL AND address_zip != ''
|
|
OR address_street_name IS NOT NULL AND address_street_name != ''
|
|
OR address_house_number IS NOT NULL AND address_house_number != ''
|
|
)
|
|
""")
|
|
total_providers = await conn.fetchval(
|
|
"SELECT COUNT(*) FROM marketplace.service_providers"
|
|
)
|
|
with_address = await conn.fetchval(
|
|
"SELECT COUNT(*) FROM marketplace.service_providers WHERE address_id IS NOT NULL"
|
|
)
|
|
logger.info(f" Total providers: {total_providers}")
|
|
logger.info(f" Providers WITH address_id: {with_address}")
|
|
logger.info(f" Providers still NULL: {remaining}")
|
|
logger.info("")
|
|
logger.info("✅ Phase 1 migration complete!")
|
|
|
|
except Exception as e:
|
|
logger.error(f"💥 FATAL ERROR: {e}", exc_info=True)
|
|
sys.exit(1)
|
|
finally:
|
|
await conn.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|