jonas/alpinebits_python
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

Migration successfull. Does not cause any problems and new foreign keys work

Browse Source
This commit is contained in:
Jonas Linter
2025-12-02 11:27:07 +01:00
parent e24866d8a0
commit ee80c57bcb
2 changed files with 257 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
alembic/versions/2025_12_02_1106-b50c0f45030a_id_columns_changed_to_integer_foreign_.py Normal file
View File

@@ -0,0 +1,167 @@
"""Id columns changed to integer, foreign_keys added
Revision ID: b50c0f45030a
Revises: b2cfe2d3aabc
Create Date: 2025-12-02 11:06:25.850790
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'b50c0f45030a'
down_revision: Union[str, Sequence[str], None] = 'b2cfe2d3aabc'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
# Drop composite FK constraint first (references guest_id columns)
op.drop_constraint(
'conversions_hotel_id_guest_id_fkey', 'conversions', type_='foreignkey'
)
# Now convert the guest_id columns
op.alter_column('conversion_guests', 'guest_id',
existing_type=sa.VARCHAR(),
type_=sa.Integer(),
existing_nullable=False,
postgresql_using='guest_id::integer')
op.alter_column('conversion_guests', 'is_regular',
existing_type=sa.BOOLEAN(),
nullable=True)
op.drop_constraint(op.f('conversion_guests_hashed_customer_id_fkey'), 'conversion_guests', type_='foreignkey')
op.create_foreign_key(op.f('fk_conversion_guests_hashed_customer_id_hashed_customers'), 'conversion_guests', 'hashed_customers', ['hashed_customer_id'], ['id'])
# Create FK with NOT VALID to skip checking existing data
# (hotels table will be populated from config when app starts)
op.create_foreign_key(
op.f('fk_conversion_guests_hotel_id_hotels'),
'conversion_guests',
'hotels',
['hotel_id'],
['hotel_id'],
ondelete='CASCADE',
postgresql_not_valid=True
)
op.alter_column('conversions', 'hotel_id',
existing_type=sa.VARCHAR(),
nullable=False)
op.alter_column('conversions', 'pms_reservation_id',
existing_type=sa.VARCHAR(),
type_=sa.Integer(),
nullable=False,
postgresql_using='pms_reservation_id::integer')
op.alter_column('conversions', 'guest_id',
existing_type=sa.VARCHAR(),
type_=sa.Integer(),
existing_nullable=True,
postgresql_using='guest_id::integer')
op.alter_column('conversions', 'directly_attributable',
existing_type=sa.BOOLEAN(),
nullable=True)
op.alter_column('conversions', 'guest_matched',
existing_type=sa.BOOLEAN(),
nullable=True)
# Re-create composite FK constraint after column type changes
op.create_foreign_key(
'conversions_hotel_id_guest_id_fkey',
'conversions',
'conversion_guests',
['hotel_id', 'guest_id'],
['hotel_id', 'guest_id'],
ondelete='SET NULL'
)
op.create_unique_constraint('uq_conversion_hotel_reservation', 'conversions', ['hotel_id', 'pms_reservation_id'])
# Create FK with NOT VALID for same reason as above
op.create_foreign_key(
op.f('fk_conversions_hotel_id_hotels'),
'conversions',
'hotels',
['hotel_id'],
['hotel_id'],
ondelete='CASCADE',
postgresql_not_valid=True
)
op.drop_constraint(op.f('customers_contact_id_key'), 'customers', type_='unique')
op.create_unique_constraint(op.f('uq_customers_contact_id'), 'customers', ['contact_id'])
op.drop_constraint(op.f('hashed_customers_contact_id_key'), 'hashed_customers', type_='unique')
op.drop_constraint(op.f('hashed_customers_customer_id_key'), 'hashed_customers', type_='unique')
op.create_unique_constraint(op.f('uq_hashed_customers_contact_id'), 'hashed_customers', ['contact_id'])
op.create_unique_constraint(op.f('uq_hashed_customers_customer_id'), 'hashed_customers', ['customer_id'])
op.drop_index(op.f('ix_reservations_hashed_customer_id'), table_name='reservations')
op.drop_constraint(op.f('reservations_md5_unique_id_key'), 'reservations', type_='unique')
op.drop_constraint(op.f('reservations_unique_id_key'), 'reservations', type_='unique')
op.create_unique_constraint(op.f('uq_reservations_md5_unique_id'), 'reservations', ['md5_unique_id'])
op.create_unique_constraint(op.f('uq_reservations_unique_id'), 'reservations', ['unique_id'])
op.drop_index(op.f('idx_room_availability_inventory_date'), table_name='room_availability')
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f('idx_room_availability_inventory_date'), 'room_availability', ['inventory_id', 'date'], unique=False)
op.drop_constraint(op.f('uq_reservations_unique_id'), 'reservations', type_='unique')
op.drop_constraint(op.f('uq_reservations_md5_unique_id'), 'reservations', type_='unique')
op.create_unique_constraint(op.f('reservations_unique_id_key'), 'reservations', ['unique_id'], postgresql_nulls_not_distinct=False)
op.create_unique_constraint(op.f('reservations_md5_unique_id_key'), 'reservations', ['md5_unique_id'], postgresql_nulls_not_distinct=False)
op.create_index(op.f('ix_reservations_hashed_customer_id'), 'reservations', ['hashed_customer_id'], unique=False)
op.drop_constraint(op.f('uq_hashed_customers_customer_id'), 'hashed_customers', type_='unique')
op.drop_constraint(op.f('uq_hashed_customers_contact_id'), 'hashed_customers', type_='unique')
op.create_unique_constraint(op.f('hashed_customers_customer_id_key'), 'hashed_customers', ['customer_id'], postgresql_nulls_not_distinct=False)
op.create_unique_constraint(op.f('hashed_customers_contact_id_key'), 'hashed_customers', ['contact_id'], postgresql_nulls_not_distinct=False)
op.drop_constraint(op.f('uq_customers_contact_id'), 'customers', type_='unique')
op.create_unique_constraint(op.f('customers_contact_id_key'), 'customers', ['contact_id'], postgresql_nulls_not_distinct=False)
op.drop_constraint(op.f('fk_conversions_hotel_id_hotels'), 'conversions', type_='foreignkey')
op.drop_constraint('uq_conversion_hotel_reservation', 'conversions', type_='unique')
# Drop composite FK constraint before changing column types back
op.drop_constraint(
'conversions_hotel_id_guest_id_fkey', 'conversions', type_='foreignkey'
)
op.alter_column('conversions', 'guest_matched',
existing_type=sa.BOOLEAN(),
nullable=False)
op.alter_column('conversions', 'directly_attributable',
existing_type=sa.BOOLEAN(),
nullable=False)
op.alter_column('conversions', 'guest_id',
existing_type=sa.Integer(),
type_=sa.VARCHAR(),
existing_nullable=True)
op.alter_column('conversions', 'pms_reservation_id',
existing_type=sa.Integer(),
type_=sa.VARCHAR(),
nullable=True)
op.alter_column('conversions', 'hotel_id',
existing_type=sa.VARCHAR(),
nullable=True)
op.drop_constraint(op.f('fk_conversion_guests_hotel_id_hotels'), 'conversion_guests', type_='foreignkey')
op.drop_constraint(op.f('fk_conversion_guests_hashed_customer_id_hashed_customers'), 'conversion_guests', type_='foreignkey')
op.create_foreign_key(op.f('conversion_guests_hashed_customer_id_fkey'), 'conversion_guests', 'hashed_customers', ['hashed_customer_id'], ['id'], ondelete='SET NULL')
op.alter_column('conversion_guests', 'is_regular',
existing_type=sa.BOOLEAN(),
nullable=False)
op.alter_column('conversion_guests', 'guest_id',
existing_type=sa.Integer(),
type_=sa.VARCHAR(),
existing_nullable=False)
# Re-create composite FK constraint after reverting column types
op.create_foreign_key(
'conversions_hotel_id_guest_id_fkey',
'conversions',
'conversion_guests',
['hotel_id', 'guest_id'],
['hotel_id', 'guest_id'],
ondelete='SET NULL'
)
# ### end Alembic commands ###

122
src/alpine_bits_python/db.py
View File

@@ -4,8 +4,6 @@ import os
from collections.abc import AsyncGenerator, Callable from collections.abc import AsyncGenerator, Callable
from typing import TypeVar from typing import TypeVar
from .const import WebhookStatus
from sqlalchemy import ( from sqlalchemy import (
JSON, JSON,
Boolean, Boolean,
@@ -17,6 +15,7 @@ from sqlalchemy import (
ForeignKeyConstraint, ForeignKeyConstraint,
Index, Index,
Integer, Integer,
MetaData,
String, String,
UniqueConstraint, UniqueConstraint,
func, func,
@@ -30,6 +29,7 @@ from sqlalchemy.ext.asyncio import (
) )
from sqlalchemy.orm import backref, declarative_base, relationship from sqlalchemy.orm import backref, declarative_base, relationship
from .const import WebhookStatus
from .logging_config import get_logger from .logging_config import get_logger
_LOGGER = get_logger(__name__) _LOGGER = get_logger(__name__)
@@ -58,7 +58,16 @@ class Base:
# __table_args__ = {"schema": _SCHEMA} # __table_args__ = {"schema": _SCHEMA}
Base = declarative_base(cls=Base) # Define naming convention for constraints
metadata = MetaData(naming_convention={
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s"
})
Base = declarative_base(cls=Base, metadata=metadata)
# Type variable for async functions # Type variable for async functions
T = TypeVar("T") T = TypeVar("T")
@@ -353,7 +362,10 @@ class HashedCustomer(Base):
__tablename__ = "hashed_customers" __tablename__ = "hashed_customers"
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
customer_id = Column( customer_id = Column(
Integer, ForeignKey("customers.id", ondelete="SET NULL"), unique=True, nullable=True Integer,
ForeignKey("customers.id", ondelete="SET NULL"),
unique=True,
nullable=True,
) )
contact_id = Column(String, unique=True) # Keep unhashed for reference contact_id = Column(String, unique=True) # Keep unhashed for reference
hashed_email = Column(String(64)) # SHA256 produces 64 hex chars hashed_email = Column(String(64)) # SHA256 produces 64 hex chars
@@ -367,7 +379,9 @@ class HashedCustomer(Base):
hashed_birth_date = Column(String(64)) hashed_birth_date = Column(String(64))
created_at = Column(DateTime(timezone=True)) created_at = Column(DateTime(timezone=True))
customer = relationship("Customer", backref=backref("hashed_version", uselist=False, lazy="joined")) customer = relationship(
"Customer", backref=backref("hashed_version", uselist=False, lazy="joined")
)
class ConversionGuest(Base): class ConversionGuest(Base):
@@ -383,7 +397,13 @@ class ConversionGuest(Base):
__tablename__ = "conversion_guests" __tablename__ = "conversion_guests"
# Natural keys from PMS - composite primary key # Natural keys from PMS - composite primary key
hotel_id = Column(String(50), ForeignKey("hotels.hotel_id", ondelete="CASCADE"), nullable=False, primary_key=True, index=True) hotel_id = Column(
String(50),
ForeignKey("hotels.hotel_id", ondelete="CASCADE"),
nullable=False,
primary_key=True,
index=True,
)
guest_id = Column(Integer, nullable=False, primary_key=True, index=True) guest_id = Column(Integer, nullable=False, primary_key=True, index=True)
# Unhashed guest information (for reference/transition period) # Unhashed guest information (for reference/transition period)
@@ -401,10 +421,14 @@ class ConversionGuest(Base):
hashed_birth_date = Column(String(64)) hashed_birth_date = Column(String(64))
# Matched customer reference (nullable, filled after matching) # Matched customer reference (nullable, filled after matching)
hashed_customer_id = Column(Integer, ForeignKey("hashed_customers.id"), nullable=True, index=True) hashed_customer_id = Column(
Integer, ForeignKey("hashed_customers.id"), nullable=True, index=True
)
# Guest classification # Guest classification
is_regular = Column(Boolean, default=False) # True if guest has many prior stays before appearing in our reservations is_regular = Column(
Boolean, default=False
) # True if guest has many prior stays before appearing in our reservations
# Metadata # Metadata
first_seen = Column(DateTime(timezone=True)) first_seen = Column(DateTime(timezone=True))
@@ -428,7 +452,7 @@ class ConversionGuest(Base):
def create_from_conversion_data( def create_from_conversion_data(
cls, cls,
hotel_id: str, hotel_id: str,
guest_id: str | None, guest_id: int | None,
guest_first_name: str | None, guest_first_name: str | None,
guest_last_name: str | None, guest_last_name: str | None,
guest_email: str | None, guest_email: str | None,
@@ -483,7 +507,9 @@ class ConversionGuest(Base):
self.hashed_country_code = self._normalize_and_hash(guest_country_code) self.hashed_country_code = self._normalize_and_hash(guest_country_code)
if guest_birth_date: if guest_birth_date:
self.guest_birth_date = guest_birth_date self.guest_birth_date = guest_birth_date
self.hashed_birth_date = self._normalize_and_hash(guest_birth_date.isoformat()) self.hashed_birth_date = self._normalize_and_hash(
guest_birth_date.isoformat()
)
self.last_seen = now self.last_seen = now
@@ -491,7 +517,9 @@ class Reservation(Base):
__tablename__ = "reservations" __tablename__ = "reservations"
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
customer_id = Column(Integer, ForeignKey("customers.id", ondelete="SET NULL")) customer_id = Column(Integer, ForeignKey("customers.id", ondelete="SET NULL"))
hashed_customer_id = Column(Integer, ForeignKey("hashed_customers.id", ondelete="CASCADE")) hashed_customer_id = Column(
Integer, ForeignKey("hashed_customers.id", ondelete="CASCADE")
)
unique_id = Column(String, unique=True) unique_id = Column(String, unique=True)
md5_unique_id = Column(String(32), unique=True) # max length 32 guaranteed md5_unique_id = Column(String(32), unique=True) # max length 32 guaranteed
start_date = Column(Date) start_date = Column(Date)
@@ -578,9 +606,18 @@ class Conversion(Base):
) )
# Reservation metadata from XML # Reservation metadata from XML
hotel_id = Column(String(50), ForeignKey("hotels.hotel_id", ondelete="CASCADE"), nullable=False, index=True) # hotelID attribute hotel_id = Column(
pms_reservation_id = Column(Integer, nullable=False, index=True) # id attribute from reservation String(50),
guest_id = Column(Integer, nullable=True, index=True) # PMS guest ID, FK to conversion_guests ForeignKey("hotels.hotel_id", ondelete="CASCADE"),
nullable=False,
index=True,
) # hotelID attribute
pms_reservation_id = Column(
Integer, nullable=False, index=True
) # id attribute from reservation
guest_id = Column(
Integer, nullable=True, index=True
) # PMS guest ID, FK to conversion_guests
reservation_number = Column(String) # number attribute reservation_number = Column(String) # number attribute
reservation_date = Column(Date) # date attribute (when reservation was made) reservation_date = Column(Date) # date attribute (when reservation was made)
@@ -588,9 +625,6 @@ class Conversion(Base):
reservation_type = Column(String) # type attribute (e.g., "reservation") reservation_type = Column(String) # type attribute (e.g., "reservation")
booking_channel = Column(String) # bookingChannel attribute booking_channel = Column(String) # bookingChannel attribute
# Advertising/tracking data - used for matching to existing reservations # Advertising/tracking data - used for matching to existing reservations
advertising_medium = Column( advertising_medium = Column(
String, index=True String, index=True
@@ -603,7 +637,9 @@ class Conversion(Base):
) # advertisingCampagne (contains fbclid/gclid) ) # advertisingCampagne (contains fbclid/gclid)
# Attribution flags - track how this conversion was matched # Attribution flags - track how this conversion was matched
directly_attributable = Column(Boolean, default=False) # Matched by ID (high confidence) directly_attributable = Column(
Boolean, default=False
) # Matched by ID (high confidence)
guest_matched = Column(Boolean, default=False) # Matched by guest details only guest_matched = Column(Boolean, default=False) # Matched by guest details only
# Metadata # Metadata
@@ -617,7 +653,9 @@ class Conversion(Base):
["conversion_guests.hotel_id", "conversion_guests.guest_id"], ["conversion_guests.hotel_id", "conversion_guests.guest_id"],
ondelete="SET NULL", ondelete="SET NULL",
), ),
UniqueConstraint("hotel_id", "pms_reservation_id", name="uq_conversion_hotel_reservation"), UniqueConstraint(
"hotel_id", "pms_reservation_id", name="uq_conversion_hotel_reservation"
),
) )
# Relationships # Relationships
@@ -690,7 +728,10 @@ class HotelInventory(Base):
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
hotel_id = Column( hotel_id = Column(
String(50), ForeignKey("hotels.hotel_id", ondelete="CASCADE"), nullable=False, index=True String(50),
ForeignKey("hotels.hotel_id", ondelete="CASCADE"),
nullable=False,
index=True,
) )
inv_type_code = Column(String(8), nullable=False, index=True) inv_type_code = Column(String(8), nullable=False, index=True)
inv_code = Column(String(16), nullable=True, index=True) inv_code = Column(String(16), nullable=True, index=True)
@@ -726,7 +767,10 @@ class RoomAvailability(Base):
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
inventory_id = Column( inventory_id = Column(
Integer, ForeignKey("hotel_inventory.id", ondelete="CASCADE"), nullable=False, index=True Integer,
ForeignKey("hotel_inventory.id", ondelete="CASCADE"),
nullable=False,
index=True,
) )
date = Column(Date, nullable=False, index=True) date = Column(Date, nullable=False, index=True)
count_type_2 = Column(Integer, nullable=True) count_type_2 = Column(Integer, nullable=True)
@@ -739,7 +783,9 @@ class RoomAvailability(Base):
inventory_item = relationship("HotelInventory", back_populates="availability") inventory_item = relationship("HotelInventory", back_populates="availability")
__table_args__ = ( __table_args__ = (
UniqueConstraint("inventory_id", "date", name="uq_room_availability_unique_key"), UniqueConstraint(
"inventory_id", "date", name="uq_room_availability_unique_key"
),
) )
@@ -787,7 +833,9 @@ class WebhookEndpoint(Base):
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
# Hotel association # Hotel association
hotel_id = Column(String(50), ForeignKey("hotels.hotel_id"), nullable=False, index=True) hotel_id = Column(
String(50), ForeignKey("hotels.hotel_id"), nullable=False, index=True
)
# Webhook configuration # Webhook configuration
webhook_secret = Column(String(64), unique=True, nullable=False, index=True) webhook_secret = Column(String(64), unique=True, nullable=False, index=True)
@@ -803,7 +851,7 @@ class WebhookEndpoint(Base):
webhook_requests = relationship("WebhookRequest", back_populates="webhook_endpoint") webhook_requests = relationship("WebhookRequest", back_populates="webhook_endpoint")
__table_args__ = ( __table_args__ = (
Index('idx_webhook_endpoint_hotel_type', 'hotel_id', 'webhook_type'), Index("idx_webhook_endpoint_hotel_type", "hotel_id", "webhook_type"),
) )
@@ -816,11 +864,17 @@ class WebhookRequest(Base):
# Request identification # Request identification
payload_hash = Column(String(64), unique=True, nullable=False, index=True) # SHA256 payload_hash = Column(String(64), unique=True, nullable=False, index=True) # SHA256
webhook_endpoint_id = Column(Integer, ForeignKey("webhook_endpoints.id"), nullable=True, index=True) webhook_endpoint_id = Column(
hotel_id = Column(String(50), ForeignKey("hotels.hotel_id"), nullable=True, index=True) Integer, ForeignKey("webhook_endpoints.id"), nullable=True, index=True
)
hotel_id = Column(
String(50), ForeignKey("hotels.hotel_id"), nullable=True, index=True
)
# Processing tracking # Processing tracking
status = Column(String(20), nullable=False, default=WebhookStatus.PENDING.value, index=True) status = Column(
String(20), nullable=False, default=WebhookStatus.PENDING.value, index=True
)
# Status values: 'pending', 'processing', 'completed', 'failed' set by Enum WebhookStatus # Status values: 'pending', 'processing', 'completed', 'failed' set by Enum WebhookStatus
processing_started_at = Column(DateTime(timezone=True), nullable=True) processing_started_at = Column(DateTime(timezone=True), nullable=True)
@@ -841,16 +895,20 @@ class WebhookRequest(Base):
# Result tracking # Result tracking
created_customer_id = Column(Integer, ForeignKey("customers.id"), nullable=True) created_customer_id = Column(Integer, ForeignKey("customers.id"), nullable=True)
created_reservation_id = Column(Integer, ForeignKey("reservations.id"), nullable=True) created_reservation_id = Column(
Integer, ForeignKey("reservations.id"), nullable=True
)
# Relationships # Relationships
webhook_endpoint = relationship("WebhookEndpoint", back_populates="webhook_requests") webhook_endpoint = relationship(
"WebhookEndpoint", back_populates="webhook_requests"
)
hotel = relationship("Hotel") hotel = relationship("Hotel")
customer = relationship("Customer") customer = relationship("Customer")
reservation = relationship("Reservation") reservation = relationship("Reservation")
__table_args__ = ( __table_args__ = (
Index('idx_webhook_status_created', 'status', 'created_at'), Index("idx_webhook_status_created", "status", "created_at"),
Index('idx_webhook_hotel_created', 'hotel_id', 'created_at'), Index("idx_webhook_hotel_created", "hotel_id", "created_at"),
Index('idx_webhook_purge_candidate', 'status', 'purged_at', 'created_at'), Index("idx_webhook_purge_candidate", "status", "purged_at", "created_at"),
) )