Merging schema_extension #9

Merged
jonas merged 15 commits from schema_extension into main 2025-10-20 07:19:26 +00:00
3 changed files with 74 additions and 11 deletions
Showing only changes of commit 27ed8dcd1f - Show all commits

View File

@@ -97,7 +97,7 @@ class HashedCustomer(Base):
hashed_country_code = Column(String(64)) hashed_country_code = Column(String(64))
hashed_gender = Column(String(64)) hashed_gender = Column(String(64))
hashed_birth_date = Column(String(64)) hashed_birth_date = Column(String(64))
created_at = Column(DateTime) created_at = Column(DateTime(timezone=True))
customer = relationship("Customer", backref="hashed_version") customer = relationship("Customer", backref="hashed_version")
@@ -114,7 +114,7 @@ class Reservation(Base):
num_children = Column(Integer) num_children = Column(Integer)
children_ages = Column(String) # comma-separated children_ages = Column(String) # comma-separated
offer = Column(String) offer = Column(String)
created_at = Column(DateTime) created_at = Column(DateTime(timezone=True))
# Add all UTM fields and user comment for XML # Add all UTM fields and user comment for XML
utm_source = Column(String) utm_source = Column(String)
utm_medium = Column(String) utm_medium = Column(String)
@@ -142,4 +142,4 @@ class AckedRequest(Base):
unique_id = Column( unique_id = Column(
String, index=True String, index=True
) # Should match Reservation.form_id or another unique field ) # Should match Reservation.form_id or another unique field
timestamp = Column(DateTime) timestamp = Column(DateTime(timezone=True))

View File

@@ -1,10 +1,16 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Fix PostgreSQL sequence values after migration from SQLite. """Fix PostgreSQL sequences and migrate datetime columns after SQLite migration.
This script resets all ID sequence values to match the current maximum ID This script performs two operations:
in each table. This is necessary because the migration script inserts records 1. Migrates DateTime columns to TIMESTAMP WITH TIME ZONE for timezone-aware support
2. Resets all ID sequence values to match the current maximum ID in each table
The sequence reset is necessary because the migration script inserts records
with explicit IDs, which doesn't automatically advance PostgreSQL sequences. with explicit IDs, which doesn't automatically advance PostgreSQL sequences.
The datetime migration ensures proper handling of timezone-aware datetimes,
which is required by the application code.
Usage: Usage:
# Using default config.yaml # Using default config.yaml
uv run python -m alpine_bits_python.util.fix_postgres_sequences uv run python -m alpine_bits_python.util.fix_postgres_sequences
@@ -42,14 +48,40 @@ from alpine_bits_python.logging_config import get_logger, setup_logging
_LOGGER = get_logger(__name__) _LOGGER = get_logger(__name__)
async def migrate_datetime_columns(session: AsyncSession) -> None:
"""Migrate DateTime columns to TIMESTAMP WITH TIME ZONE.
This updates the columns to properly handle timezone-aware datetimes.
"""
_LOGGER.info("\nMigrating DateTime columns to timezone-aware...")
datetime_columns = [
("hashed_customers", "created_at"),
("reservations", "created_at"),
("acked_requests", "timestamp"),
]
for table_name, column_name in datetime_columns:
_LOGGER.info(f" {table_name}.{column_name}: Converting to TIMESTAMPTZ")
await session.execute(
text(
f"ALTER TABLE {table_name} "
f"ALTER COLUMN {column_name} TYPE TIMESTAMP WITH TIME ZONE"
)
)
await session.commit()
_LOGGER.info("✓ DateTime columns migrated to timezone-aware")
async def fix_sequences(database_url: str) -> None: async def fix_sequences(database_url: str) -> None:
"""Fix PostgreSQL sequences to match current max IDs. """Fix PostgreSQL sequences to match current max IDs and migrate datetime columns.
Args: Args:
database_url: PostgreSQL database URL database_url: PostgreSQL database URL
""" """
_LOGGER.info("=" * 70) _LOGGER.info("=" * 70)
_LOGGER.info("PostgreSQL Sequence Fix") _LOGGER.info("PostgreSQL Migration & Sequence Fix")
_LOGGER.info("=" * 70) _LOGGER.info("=" * 70)
_LOGGER.info("Database: %s", database_url.split("@")[-1] if "@" in database_url else database_url) _LOGGER.info("Database: %s", database_url.split("@")[-1] if "@" in database_url else database_url)
_LOGGER.info("=" * 70) _LOGGER.info("=" * 70)
@@ -59,6 +91,11 @@ async def fix_sequences(database_url: str) -> None:
SessionMaker = async_sessionmaker(engine, expire_on_commit=False) SessionMaker = async_sessionmaker(engine, expire_on_commit=False)
try: try:
# Migrate datetime columns first
async with SessionMaker() as session:
await migrate_datetime_columns(session)
# Then fix sequences
async with SessionMaker() as session: async with SessionMaker() as session:
# List of tables and their sequence names # List of tables and their sequence names
tables = [ tables = [
@@ -105,9 +142,12 @@ async def fix_sequences(database_url: str) -> None:
await session.commit() await session.commit()
_LOGGER.info("\n" + "=" * 70) _LOGGER.info("\n" + "=" * 70)
_LOGGER.info("Sequences fixed successfully!") _LOGGER.info("Migration completed successfully!")
_LOGGER.info("=" * 70) _LOGGER.info("=" * 70)
_LOGGER.info("\nYou can now insert new records without ID conflicts.") _LOGGER.info("\nChanges applied:")
_LOGGER.info(" 1. DateTime columns are now timezone-aware (TIMESTAMPTZ)")
_LOGGER.info(" 2. Sequences are reset to match current max IDs")
_LOGGER.info("\nYou can now insert new records without conflicts.")
except Exception as e: except Exception as e:
_LOGGER.exception("Failed to fix sequences: %s", e) _LOGGER.exception("Failed to fix sequences: %s", e)

View File

@@ -345,8 +345,31 @@ async def migrate_data(
_LOGGER.info("✓ Migrated %d acked requests", len(acked_requests)) _LOGGER.info("✓ Migrated %d acked requests", len(acked_requests))
# Migrate datetime columns to timezone-aware
_LOGGER.info("\n[5/6] Converting DateTime columns to timezone-aware...")
async with target_engine.begin() as conn:
await conn.execute(
text(
"ALTER TABLE hashed_customers "
"ALTER COLUMN created_at TYPE TIMESTAMP WITH TIME ZONE"
)
)
await conn.execute(
text(
"ALTER TABLE reservations "
"ALTER COLUMN created_at TYPE TIMESTAMP WITH TIME ZONE"
)
)
await conn.execute(
text(
"ALTER TABLE acked_requests "
"ALTER COLUMN timestamp TYPE TIMESTAMP WITH TIME ZONE"
)
)
_LOGGER.info("✓ DateTime columns converted to timezone-aware")
# Reset PostgreSQL sequences # Reset PostgreSQL sequences
_LOGGER.info("\n[5/5] Resetting PostgreSQL sequences...") _LOGGER.info("\n[6/6] Resetting PostgreSQL sequences...")
async with TargetSession() as target_session: async with TargetSession() as target_session:
await reset_sequences(target_session) await reset_sequences(target_session)
_LOGGER.info("✓ Sequences reset to match current max IDs") _LOGGER.info("✓ Sequences reset to match current max IDs")