merge_db_fixes_to_main #16

Merged
jonas merged 40 commits from merge_db_fixes_to_main into main 2025-12-09 11:37:21 +00:00
7 changed files with 75 additions and 72 deletions
Showing only changes of commit ad29a0a2f6 - Show all commits

View File

@@ -0,0 +1,63 @@
"""removed hashed_customer completly
Revision ID: 3147e421bc47
Revises: 0fbeb40dbb2c
Create Date: 2025-12-03 11:42:05.722690
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = '3147e421bc47'
down_revision: Union[str, Sequence[str], None] = '0fbeb40dbb2c'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_conversion_guests_hashed_customer_id'), table_name='conversion_guests')
op.drop_constraint(op.f('fk_conversion_guests_hashed_customer_id_hashed_customers'), 'conversion_guests', type_='foreignkey')
op.drop_column('conversion_guests', 'hashed_customer_id')
op.drop_index(op.f('ix_conversions_hashed_customer_id'), table_name='conversions')
op.drop_constraint(op.f('conversions_hashed_customer_id_fkey'), 'conversions', type_='foreignkey')
op.drop_column('conversions', 'hashed_customer_id')
op.drop_table('hashed_customers')
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('conversions', sa.Column('hashed_customer_id', sa.INTEGER(), autoincrement=False, nullable=True))
op.create_foreign_key(op.f('conversions_hashed_customer_id_fkey'), 'conversions', 'hashed_customers', ['hashed_customer_id'], ['id'])
op.create_index(op.f('ix_conversions_hashed_customer_id'), 'conversions', ['hashed_customer_id'], unique=False)
op.add_column('conversion_guests', sa.Column('hashed_customer_id', sa.INTEGER(), autoincrement=False, nullable=True))
op.create_foreign_key(op.f('fk_conversion_guests_hashed_customer_id_hashed_customers'), 'conversion_guests', 'hashed_customers', ['hashed_customer_id'], ['id'])
op.create_index(op.f('ix_conversion_guests_hashed_customer_id'), 'conversion_guests', ['hashed_customer_id'], unique=False)
op.create_table('hashed_customers',
sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column('customer_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('contact_id', sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column('hashed_email', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('hashed_phone', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('hashed_given_name', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('hashed_surname', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('hashed_city', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('hashed_postal_code', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('hashed_country_code', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('hashed_gender', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('hashed_birth_date', sa.VARCHAR(length=64), autoincrement=False, nullable=True),
sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
sa.ForeignKeyConstraint(['customer_id'], ['customers.id'], name=op.f('hashed_customers_customer_id_fkey'), ondelete='SET NULL'),
sa.PrimaryKeyConstraint('id', name=op.f('hashed_customers_pkey')),
sa.UniqueConstraint('contact_id', name=op.f('uq_hashed_customers_contact_id'), postgresql_include=[], postgresql_nulls_not_distinct=False),
sa.UniqueConstraint('customer_id', name=op.f('uq_hashed_customers_customer_id'), postgresql_include=[], postgresql_nulls_not_distinct=False)
)
# ### end Alembic commands ###

View File

@@ -629,10 +629,8 @@ class ConversionService:
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
query = select(Reservation).options( query = select(Reservation).options(
selectinload(Reservation.customer).selectinload( selectinload(Reservation.customer),
Customer.hashed_version
),
selectinload(Reservation.hashed_customer),
) )
result = await session.execute(query) result = await session.execute(query)
reservations = result.scalars().all() reservations = result.scalars().all()
@@ -645,13 +643,11 @@ class ConversionService:
if hotel_code not in self._reservation_cache: if hotel_code not in self._reservation_cache:
self._reservation_cache[hotel_code] = [] self._reservation_cache[hotel_code] = []
# Cache the hashed customer - prefer direct relationship, fall back to customer relationship # Cache the hashed customer - prefer direct relationship, fall back to customer relationship
hashed_customer = None customer = None
if reservation.hashed_customer: if reservation.customer:
hashed_customer = reservation.hashed_customer customer = reservation.customer
elif reservation.customer and reservation.customer.hashed_version:
hashed_customer = reservation.customer.hashed_version
self._reservation_cache[hotel_code].append( self._reservation_cache[hotel_code].append(
(reservation, hashed_customer) (reservation, customer)
) )
self._cache_initialized = True self._cache_initialized = True
@@ -1431,7 +1427,6 @@ class ConversionService:
conversion.reservation_id = matched_reservation.id conversion.reservation_id = matched_reservation.id
conversion.customer_id = matched_hashed_customer.id conversion.customer_id = matched_hashed_customer.id
conversion.hashed_customer_id = matched_hashed_customer.id
conversion.directly_attributable = True conversion.directly_attributable = True
conversion.guest_matched = True conversion.guest_matched = True
conversion.updated_at = datetime.now() conversion.updated_at = datetime.now()
@@ -1447,7 +1442,6 @@ class ConversionService:
elif matched_hashed_customer and conversion.customer_id is None: elif matched_hashed_customer and conversion.customer_id is None:
# Only count new customer matches (conversions that didn't have a customer before) # Only count new customer matches (conversions that didn't have a customer before)
conversion.customer_id = matched_hashed_customer.id conversion.customer_id = matched_hashed_customer.id
conversion.hashed_customer_id = matched_hashed_customer.id
conversion.directly_attributable = False conversion.directly_attributable = False
conversion.guest_matched = True conversion.guest_matched = True
conversion.updated_at = datetime.now() conversion.updated_at = datetime.now()
@@ -1742,8 +1736,6 @@ class ConversionService:
if matched_reservation: if matched_reservation:
matched_customer = matched_reservation.customer matched_customer = matched_reservation.customer
if matched_customer and matched_customer.hashed_version:
matched_hashed_customer = matched_customer.hashed_version
_LOGGER.info( _LOGGER.info(
"Phase 3a: Matched conversion by advertising ID (pms_id=%s, reservation_id=%d)", "Phase 3a: Matched conversion by advertising ID (pms_id=%s, reservation_id=%d)",
@@ -1752,14 +1744,12 @@ class ConversionService:
) )
# Update the conversion with matched entities if found # Update the conversion with matched entities if found
if matched_reservation or matched_customer or matched_hashed_customer: if matched_reservation or matched_customer:
conversion.reservation_id = ( conversion.reservation_id = (
matched_reservation.id if matched_reservation else None matched_reservation.id if matched_reservation else None
) )
conversion.customer_id = matched_customer.id if matched_customer else None conversion.customer_id = matched_customer.id if matched_customer else None
conversion.hashed_customer_id = (
matched_hashed_customer.id if matched_hashed_customer else None
)
# ID-based matches are always directly attributable # ID-based matches are always directly attributable
conversion.directly_attributable = True conversion.directly_attributable = True

View File

@@ -6,7 +6,7 @@ from pydantic import ValidationError
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from .db import Customer, HashedCustomer from .db import Customer
from .logging_config import get_logger from .logging_config import get_logger
from .schemas import CustomerData from .schemas import CustomerData

View File

@@ -362,36 +362,6 @@ class Customer(Base):
self.hashed_birth_date = self._normalize_and_hash(self.birth_date) self.hashed_birth_date = self._normalize_and_hash(self.birth_date)
class HashedCustomer(Base):
"""Hashed customer data for Meta Conversion API.
Stores SHA256 hashed versions of customer PII according to Meta's requirements.
This allows sending conversion events without exposing raw customer data.
"""
__tablename__ = "hashed_customers"
id = Column(Integer, primary_key=True)
customer_id = Column(
Integer,
ForeignKey("customers.id", ondelete="SET NULL"),
unique=True,
nullable=True,
)
contact_id = Column(String, unique=True) # Keep unhashed for reference
hashed_email = Column(String(64)) # SHA256 produces 64 hex chars
hashed_phone = Column(String(64))
hashed_given_name = Column(String(64))
hashed_surname = Column(String(64))
hashed_city = Column(String(64))
hashed_postal_code = Column(String(64))
hashed_country_code = Column(String(64))
hashed_gender = Column(String(64))
hashed_birth_date = Column(String(64))
created_at = Column(DateTime(timezone=True))
customer = relationship(
"Customer", backref=backref("hashed_version", uselist=False, lazy="joined")
)
class ConversionGuest(Base): class ConversionGuest(Base):
@@ -430,10 +400,7 @@ class ConversionGuest(Base):
hashed_country_code = Column(String(64)) hashed_country_code = Column(String(64))
hashed_birth_date = Column(String(64)) hashed_birth_date = Column(String(64))
# Matched customer reference (nullable, filled after matching)
hashed_customer_id = Column(
Integer, ForeignKey("hashed_customers.id"), nullable=True, index=True
)
# Guest classification # Guest classification
is_regular = Column( is_regular = Column(
@@ -452,7 +419,6 @@ class ConversionGuest(Base):
primaryjoin="and_(ConversionGuest.hotel_id == foreign(Conversion.hotel_id), " primaryjoin="and_(ConversionGuest.hotel_id == foreign(Conversion.hotel_id), "
"ConversionGuest.guest_id == foreign(Conversion.guest_id))", "ConversionGuest.guest_id == foreign(Conversion.guest_id))",
) )
hashed_customer = relationship("HashedCustomer", backref="conversion_guests")
@staticmethod @staticmethod
def _normalize_and_hash(value): def _normalize_and_hash(value):
@@ -613,9 +579,6 @@ class Conversion(Base):
Integer, ForeignKey("reservations.id"), nullable=True, index=True Integer, ForeignKey("reservations.id"), nullable=True, index=True
) )
customer_id = Column(Integer, ForeignKey("customers.id"), nullable=True, index=True) customer_id = Column(Integer, ForeignKey("customers.id"), nullable=True, index=True)
hashed_customer_id = Column(
Integer, ForeignKey("hashed_customers.id"), nullable=True, index=True
)
# Reservation metadata from XML # Reservation metadata from XML
hotel_id = Column( hotel_id = Column(
@@ -670,7 +633,6 @@ class Conversion(Base):
# Relationships # Relationships
reservation = relationship("Reservation", backref="conversions") reservation = relationship("Reservation", backref="conversions")
customer = relationship("Customer", backref="conversions") customer = relationship("Customer", backref="conversions")
hashed_customer = relationship("HashedCustomer", backref="conversions")
guest = relationship( guest = relationship(
"ConversionGuest", "ConversionGuest",
back_populates="conversions", back_populates="conversions",

View File

@@ -7,7 +7,7 @@ from typing import Optional
from sqlalchemy import and_, select from sqlalchemy import and_, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from .db import AckedRequest, Customer, HashedCustomer, Reservation from .db import AckedRequest, Customer, Reservation
from .schemas import ReservationData from .schemas import ReservationData
@@ -64,17 +64,6 @@ class ReservationService:
reservation_data, customer_id reservation_data, customer_id
) )
# Automatically populate hashed_customer_id from the customer
# Since hashed_customer is always created when a customer is created,
# we can get it by querying for the hashed_customer with matching customer_id
hashed_customer_result = await self.session.execute(
select(HashedCustomer).where(
HashedCustomer.customer_id == customer_id
)
)
hashed_customer = hashed_customer_result.scalar_one_or_none()
if hashed_customer:
reservation.hashed_customer_id = hashed_customer.id
self.session.add(reservation) self.session.add(reservation)

View File

@@ -51,7 +51,6 @@ from alpine_bits_python.db import (
AckedRequest, AckedRequest,
Base, Base,
Customer, Customer,
HashedCustomer,
Reservation, Reservation,
get_database_url, get_database_url,
) )

View File

@@ -203,7 +203,7 @@ async def process_wix_form_submission(
"name_title": None, "name_title": None,
} }
# This automatically creates/updates both Customer and HashedCustomer # This automatically creates/updates Customer
db_customer = await customer_service.get_or_create_customer(customer_data) db_customer = await customer_service.get_or_create_customer(customer_data)
# Determine hotel_code and hotel_name # Determine hotel_code and hotel_name