4 Commits

Author SHA1 Message Date
Jonas Linter
de61d67508 Update guest IDs in reservation tests for consistency 2025-12-02 15:27:14 +01:00
Jonas Linter
473becfe5b Fixed up the damm tests 2025-12-02 15:24:30 +01:00
Jonas Linter
0f3805bed4 New pydantic model for ConversionGuest 2025-12-02 13:18:43 +01:00
Jonas Linter
b1c867ca93 Migration successfull. Does not cause any problems and new foreign keys work 2025-12-02 11:27:07 +01:00
8 changed files with 711 additions and 250 deletions

View File

@@ -0,0 +1,167 @@
"""Id columns changed to integer, foreign_keys added
Revision ID: b50c0f45030a
Revises: b2cfe2d3aabc
Create Date: 2025-12-02 11:06:25.850790
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'b50c0f45030a'
down_revision: Union[str, Sequence[str], None] = 'b2cfe2d3aabc'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
# Drop composite FK constraint first (references guest_id columns)
op.drop_constraint(
'conversions_hotel_id_guest_id_fkey', 'conversions', type_='foreignkey'
)
# Now convert the guest_id columns
op.alter_column('conversion_guests', 'guest_id',
existing_type=sa.VARCHAR(),
type_=sa.Integer(),
existing_nullable=False,
postgresql_using='guest_id::integer')
op.alter_column('conversion_guests', 'is_regular',
existing_type=sa.BOOLEAN(),
nullable=True)
op.drop_constraint(op.f('conversion_guests_hashed_customer_id_fkey'), 'conversion_guests', type_='foreignkey')
op.create_foreign_key(op.f('fk_conversion_guests_hashed_customer_id_hashed_customers'), 'conversion_guests', 'hashed_customers', ['hashed_customer_id'], ['id'])
# Create FK with NOT VALID to skip checking existing data
# (hotels table will be populated from config when app starts)
op.create_foreign_key(
op.f('fk_conversion_guests_hotel_id_hotels'),
'conversion_guests',
'hotels',
['hotel_id'],
['hotel_id'],
ondelete='CASCADE',
postgresql_not_valid=True
)
op.alter_column('conversions', 'hotel_id',
existing_type=sa.VARCHAR(),
nullable=False)
op.alter_column('conversions', 'pms_reservation_id',
existing_type=sa.VARCHAR(),
type_=sa.Integer(),
nullable=False,
postgresql_using='pms_reservation_id::integer')
op.alter_column('conversions', 'guest_id',
existing_type=sa.VARCHAR(),
type_=sa.Integer(),
existing_nullable=True,
postgresql_using='guest_id::integer')
op.alter_column('conversions', 'directly_attributable',
existing_type=sa.BOOLEAN(),
nullable=True)
op.alter_column('conversions', 'guest_matched',
existing_type=sa.BOOLEAN(),
nullable=True)
# Re-create composite FK constraint after column type changes
op.create_foreign_key(
'conversions_hotel_id_guest_id_fkey',
'conversions',
'conversion_guests',
['hotel_id', 'guest_id'],
['hotel_id', 'guest_id'],
ondelete='SET NULL'
)
op.create_unique_constraint('uq_conversion_hotel_reservation', 'conversions', ['hotel_id', 'pms_reservation_id'])
# Create FK with NOT VALID for same reason as above
op.create_foreign_key(
op.f('fk_conversions_hotel_id_hotels'),
'conversions',
'hotels',
['hotel_id'],
['hotel_id'],
ondelete='CASCADE',
postgresql_not_valid=True
)
op.drop_constraint(op.f('customers_contact_id_key'), 'customers', type_='unique')
op.create_unique_constraint(op.f('uq_customers_contact_id'), 'customers', ['contact_id'])
op.drop_constraint(op.f('hashed_customers_contact_id_key'), 'hashed_customers', type_='unique')
op.drop_constraint(op.f('hashed_customers_customer_id_key'), 'hashed_customers', type_='unique')
op.create_unique_constraint(op.f('uq_hashed_customers_contact_id'), 'hashed_customers', ['contact_id'])
op.create_unique_constraint(op.f('uq_hashed_customers_customer_id'), 'hashed_customers', ['customer_id'])
op.drop_index(op.f('ix_reservations_hashed_customer_id'), table_name='reservations')
op.drop_constraint(op.f('reservations_md5_unique_id_key'), 'reservations', type_='unique')
op.drop_constraint(op.f('reservations_unique_id_key'), 'reservations', type_='unique')
op.create_unique_constraint(op.f('uq_reservations_md5_unique_id'), 'reservations', ['md5_unique_id'])
op.create_unique_constraint(op.f('uq_reservations_unique_id'), 'reservations', ['unique_id'])
op.drop_index(op.f('idx_room_availability_inventory_date'), table_name='room_availability')
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f('idx_room_availability_inventory_date'), 'room_availability', ['inventory_id', 'date'], unique=False)
op.drop_constraint(op.f('uq_reservations_unique_id'), 'reservations', type_='unique')
op.drop_constraint(op.f('uq_reservations_md5_unique_id'), 'reservations', type_='unique')
op.create_unique_constraint(op.f('reservations_unique_id_key'), 'reservations', ['unique_id'], postgresql_nulls_not_distinct=False)
op.create_unique_constraint(op.f('reservations_md5_unique_id_key'), 'reservations', ['md5_unique_id'], postgresql_nulls_not_distinct=False)
op.create_index(op.f('ix_reservations_hashed_customer_id'), 'reservations', ['hashed_customer_id'], unique=False)
op.drop_constraint(op.f('uq_hashed_customers_customer_id'), 'hashed_customers', type_='unique')
op.drop_constraint(op.f('uq_hashed_customers_contact_id'), 'hashed_customers', type_='unique')
op.create_unique_constraint(op.f('hashed_customers_customer_id_key'), 'hashed_customers', ['customer_id'], postgresql_nulls_not_distinct=False)
op.create_unique_constraint(op.f('hashed_customers_contact_id_key'), 'hashed_customers', ['contact_id'], postgresql_nulls_not_distinct=False)
op.drop_constraint(op.f('uq_customers_contact_id'), 'customers', type_='unique')
op.create_unique_constraint(op.f('customers_contact_id_key'), 'customers', ['contact_id'], postgresql_nulls_not_distinct=False)
op.drop_constraint(op.f('fk_conversions_hotel_id_hotels'), 'conversions', type_='foreignkey')
op.drop_constraint('uq_conversion_hotel_reservation', 'conversions', type_='unique')
# Drop composite FK constraint before changing column types back
op.drop_constraint(
'conversions_hotel_id_guest_id_fkey', 'conversions', type_='foreignkey'
)
op.alter_column('conversions', 'guest_matched',
existing_type=sa.BOOLEAN(),
nullable=False)
op.alter_column('conversions', 'directly_attributable',
existing_type=sa.BOOLEAN(),
nullable=False)
op.alter_column('conversions', 'guest_id',
existing_type=sa.Integer(),
type_=sa.VARCHAR(),
existing_nullable=True)
op.alter_column('conversions', 'pms_reservation_id',
existing_type=sa.Integer(),
type_=sa.VARCHAR(),
nullable=True)
op.alter_column('conversions', 'hotel_id',
existing_type=sa.VARCHAR(),
nullable=True)
op.drop_constraint(op.f('fk_conversion_guests_hotel_id_hotels'), 'conversion_guests', type_='foreignkey')
op.drop_constraint(op.f('fk_conversion_guests_hashed_customer_id_hashed_customers'), 'conversion_guests', type_='foreignkey')
op.create_foreign_key(op.f('conversion_guests_hashed_customer_id_fkey'), 'conversion_guests', 'hashed_customers', ['hashed_customer_id'], ['id'], ondelete='SET NULL')
op.alter_column('conversion_guests', 'is_regular',
existing_type=sa.BOOLEAN(),
nullable=False)
op.alter_column('conversion_guests', 'guest_id',
existing_type=sa.Integer(),
type_=sa.VARCHAR(),
existing_nullable=False)
# Re-create composite FK constraint after reverting column types
op.create_foreign_key(
'conversions_hotel_id_guest_id_fkey',
'conversions',
'conversion_guests',
['hotel_id', 'guest_id'],
['hotel_id', 'guest_id'],
ondelete='SET NULL'
)
# ### end Alembic commands ###

View File

@@ -32,6 +32,7 @@ from sqlalchemy import and_, select, update
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlalchemy.orm import selectinload
from alpine_bits_python.hotel_service import HotelService
from alpine_bits_python.schemas import WebhookRequestData
from .alpinebits_server import (
@@ -46,14 +47,15 @@ from .const import HttpStatusCode, WebhookStatus
from .conversion_service import ConversionService
from .csv_import import CSVImporter
from .db import Customer as DBCustomer
from .db import Reservation as DBReservation
from .db import (
Hotel,
ResilientAsyncSession,
SessionMaker,
WebhookEndpoint,
WebhookRequest,
create_database_engine,
)
from .db import Reservation as DBReservation
from .db_setup import run_startup_tasks
from .email_monitoring import ReservationStatsCollector
from .email_service import create_email_service
@@ -890,8 +892,6 @@ async def handle_webhook_unified(
webhook_request.status = WebhookStatus.PROCESSING
webhook_request.processing_started_at = timestamp
else:
webhook_request_data = WebhookRequestData(
payload_hash=payload_hash,
webhook_endpoint_id=webhook_endpoint.id,
@@ -1134,6 +1134,7 @@ async def _process_conversion_xml_background(
filename: str,
session_maker: SessionMaker,
log_filename: Path,
hotel: Hotel,
):
"""Background task to process conversion XML.
@@ -1162,7 +1163,7 @@ async def _process_conversion_xml_background(
# Now process the conversion XML
_LOGGER.info("Starting database processing of %s", filename)
conversion_service = ConversionService(session_maker)
conversion_service = ConversionService(session_maker, hotel.hotel_id)
processing_stats = await conversion_service.process_conversion_xml(xml_content)
_LOGGER.info(
@@ -1250,6 +1251,10 @@ async def handle_xml_upload(
extension = Path(filename).suffix or ".xml"
log_filename = logs_dir / f"{base_filename}_{username}_{timestamp}{extension}"
hotel_service = HotelService(db_session)
hotel = await hotel_service.get_hotel_by_username(username)
_LOGGER.info(
"XML file queued for processing: %s by user %s (original: %s)",
log_filename,
@@ -1266,6 +1271,7 @@ async def handle_xml_upload(
filename,
session_maker,
log_filename,
hotel,
)
response_headers = {

View File

@@ -1,13 +1,12 @@
"""Service for handling conversion data from hotel PMS XML files."""
import asyncio
import hashlib
import xml.etree.ElementTree as ET
from datetime import UTC, datetime
from decimal import Decimal
from typing import Any
from sqlalchemy import insert, or_, select
from sqlalchemy import or_, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
@@ -22,6 +21,7 @@ from .db import (
SessionMaker,
)
from .logging_config import get_logger
from .schemas import ConversionData, ConversionGuestData
_LOGGER = get_logger(__name__)
@@ -37,7 +37,11 @@ class ConversionService:
2. Concurrent mode: SessionMaker passed in, creates independent sessions per task
"""
def __init__(self, session: AsyncSession | SessionMaker | None = None, hotel_id: str | None = None):
def __init__(
self,
session: AsyncSession | SessionMaker | None = None,
hotel_id: str | None = None,
):
"""Initialize the ConversionService.
Args:
@@ -81,17 +85,19 @@ class ConversionService:
async def _extract_unique_guests_from_xml(
self, reservations: list
) -> dict[tuple[str, str | None], dict]:
) -> dict[tuple[str, int], ConversionGuestData]:
"""Extract and deduplicate all guest data from XML reservations.
Phase 0: Single pass through XML to collect all unique guests.
Uses (hotel_id, guest_id) as the key for deduplication.
Validates each guest using Pydantic before storing.
Args:
reservations: List of XML reservation elements
Returns:
Dictionary mapping (hotel_id, guest_id) to guest data dict
Dictionary mapping (hotel_id, guest_id) to validated ConversionGuestData
"""
guest_data_by_key = {}
now = datetime.now(UTC)
@@ -101,7 +107,10 @@ class ConversionService:
guest_elem = reservation_elem.find("guest")
if guest_elem is None:
_LOGGER.debug("No guest element found, skipping reservation %s (will be created with guest_id=None in Phase 2)", reservation_elem.get("id"))
_LOGGER.debug(
"No guest element found, skipping reservation %s (will be created with guest_id=None in Phase 2)",
reservation_elem.get("id"),
)
continue
guest_id = guest_elem.get("id")
@@ -111,33 +120,50 @@ class ConversionService:
guest_country_code = guest_elem.get("countryCode")
guest_birth_date_str = guest_elem.get("dateOfBirth")
guest_birth_date = None
if guest_birth_date_str:
try:
guest_birth_date = datetime.strptime(guest_birth_date_str, "%Y-%m-%d").date()
guest_birth_date = datetime.strptime(
guest_birth_date_str, "%Y-%m-%d"
).date()
except ValueError:
_LOGGER.warning("Invalid birth date format: %s", guest_birth_date_str)
_LOGGER.warning(
"Invalid birth date format: %s", guest_birth_date_str
)
key = (hotel_id, guest_id)
# Validate guest data with Pydantic during extraction
try:
validated_guest = ConversionGuestData(
hotel_id=hotel_id,
guest_id=guest_id, # Will be validated and converted to int
guest_first_name=guest_first_name,
guest_last_name=guest_last_name,
guest_email=guest_email,
guest_country_code=guest_country_code,
guest_birth_date=guest_birth_date,
first_seen=now,
last_seen=now,
)
# Store guest data by key (will keep the last occurrence from XML)
guest_data_by_key[key] = {
"hotel_id": hotel_id,
"guest_id": guest_id,
"guest_first_name": guest_first_name,
"guest_last_name": guest_last_name,
"guest_email": guest_email,
"guest_country_code": guest_country_code,
"guest_birth_date": guest_birth_date,
"now": now,
}
# Use validated guest_id (now an int) for the key
key = (hotel_id, validated_guest.guest_id)
# Store validated guest data (will keep the last occurrence from XML)
guest_data_by_key[key] = validated_guest
except ValueError:
_LOGGER.exception(
"Failed to validate guest data for reservation %s",
reservation_elem.get("id"),
)
continue
return guest_data_by_key
async def _bulk_upsert_guests(
self, session: AsyncSession, guest_data_by_key: dict[tuple[str, str | None], dict]
self,
session: AsyncSession,
guest_data_by_key: dict[tuple[str, int], ConversionGuestData],
) -> None:
"""Bulk upsert all unique guests to database using PostgreSQL ON CONFLICT.
@@ -147,7 +173,9 @@ class ConversionService:
Args:
session: AsyncSession to use
guest_data_by_key: Dictionary mapping (hotel_id, guest_id) to guest data
guest_data_by_key: Dictionary mapping (hotel_id, guest_id) to
validated ConversionGuestData
"""
if not guest_data_by_key:
return
@@ -160,28 +188,12 @@ class ConversionService:
batch_end = min(batch_start + batch_size, len(items))
batch_items = items[batch_start:batch_end]
# Prepare list of values for this batch
# Prepare list of values for this batch (already validated)
values_list = []
for (hotel_id, guest_id), guest_data in batch_items:
now = guest_data["now"]
values_list.append({
"hotel_id": guest_data["hotel_id"],
"guest_id": guest_data["guest_id"],
"guest_first_name": guest_data["guest_first_name"],
"guest_last_name": guest_data["guest_last_name"],
"guest_email": guest_data["guest_email"],
"guest_country_code": guest_data["guest_country_code"],
"guest_birth_date": guest_data["guest_birth_date"],
"hashed_first_name": ConversionGuest._normalize_and_hash(guest_data["guest_first_name"]),
"hashed_last_name": ConversionGuest._normalize_and_hash(guest_data["guest_last_name"]),
"hashed_email": ConversionGuest._normalize_and_hash(guest_data["guest_email"]),
"hashed_country_code": ConversionGuest._normalize_and_hash(guest_data["guest_country_code"]),
"hashed_birth_date": ConversionGuest._normalize_and_hash(
guest_data["guest_birth_date"].isoformat() if guest_data["guest_birth_date"] else None
),
"first_seen": now,
"last_seen": now,
})
for (hotel_id, guest_id), validated_guest in batch_items:
# Convert validated Pydantic model to dict for insertion
# (all validations and hash calculations are already done)
values_list.append(validated_guest.model_dump())
# Use PostgreSQL ON CONFLICT DO UPDATE for atomic upsert
stmt = pg_insert(ConversionGuest).values(values_list)
@@ -252,9 +264,13 @@ class ConversionService:
stats["deleted_reservations"] += 1
pms_reservation_id_str = deleted_res.get("ID")
try:
pms_reservation_id = int(pms_reservation_id_str) if pms_reservation_id_str else None
pms_reservation_id = (
int(pms_reservation_id_str) if pms_reservation_id_str else None
)
if pms_reservation_id is None:
_LOGGER.warning("Deleted reservation missing ID attribute, skipping")
_LOGGER.warning(
"Deleted reservation missing ID attribute, skipping"
)
continue
await self._handle_deleted_reservation(pms_reservation_id, session)
await session.commit()
@@ -295,7 +311,10 @@ class ConversionService:
# Phase 1: Bulk upsert all unique guests to database
if guest_data_by_key:
_LOGGER.debug("Phase 1: Bulk upserting %d unique guests to database", len(guest_data_by_key))
_LOGGER.debug(
"Phase 1: Bulk upserting %d unique guests to database",
len(guest_data_by_key),
)
if self.session_maker:
session = await self.session_maker.create_session()
else:
@@ -304,7 +323,9 @@ class ConversionService:
try:
await self._bulk_upsert_guests(session, guest_data_by_key)
await session.commit()
_LOGGER.info("Phase 1: Successfully upserted %d guests", len(guest_data_by_key))
_LOGGER.info(
"Phase 1: Successfully upserted %d guests", len(guest_data_by_key)
)
except Exception as e:
await session.rollback()
_LOGGER.exception("Phase 1: Error during bulk guest upsert: %s", e)
@@ -318,9 +339,13 @@ class ConversionService:
# Returns list of successfully created pms_reservation_ids
_LOGGER.debug("Phase 2: Creating/updating conversions")
if self.supports_concurrent:
pms_reservation_ids = await self._process_reservations_concurrent(reservations, stats)
pms_reservation_ids = await self._process_reservations_concurrent(
reservations, stats
)
else:
pms_reservation_ids = await self._process_reservations_sequential(reservations, stats)
pms_reservation_ids = await self._process_reservations_sequential(
reservations, stats
)
_LOGGER.debug(
"Phase 3: Found %d successfully created conversions out of %d total reservations",
@@ -335,9 +360,13 @@ class ConversionService:
if pms_reservation_ids:
_LOGGER.debug("Phase 3: Matching conversions to reservations/customers")
if self.supports_concurrent:
await self._match_conversions_from_db_concurrent(pms_reservation_ids, stats)
await self._match_conversions_from_db_concurrent(
pms_reservation_ids, stats
)
else:
await self._match_conversions_from_db_sequential(pms_reservation_ids, stats)
await self._match_conversions_from_db_sequential(
pms_reservation_ids, stats
)
return stats
@@ -375,9 +404,12 @@ class ConversionService:
try:
# Load all reservations with their hashed customers in one query
from sqlalchemy.orm import selectinload
query = select(Reservation).options(
selectinload(Reservation.customer).selectinload(Customer.hashed_version),
selectinload(Reservation.hashed_customer)
selectinload(Reservation.customer).selectinload(
Customer.hashed_version
),
selectinload(Reservation.hashed_customer),
)
result = await session.execute(query)
reservations = result.scalars().all()
@@ -420,6 +452,7 @@ class ConversionService:
Returns:
List of pms_reservation_ids that were successfully created/updated
"""
semaphore = asyncio.Semaphore(1) # Process one at a time
results = []
@@ -460,6 +493,7 @@ class ConversionService:
Returns:
List of pms_reservation_ids that were successfully created/updated
"""
if not self.session_maker:
_LOGGER.error(
@@ -518,7 +552,7 @@ class ConversionService:
pms_reservation_id if successfully created/updated, None if error occurred
"""
pms_reservation_id = reservation_elem.get("id")
pms_reservation_id = int(reservation_elem.get("id"))
async with semaphore:
# In concurrent mode, create a new session for this task
@@ -568,16 +602,22 @@ class ConversionService:
"""
if not self.hotel_id:
_LOGGER.error("Cannot delete reservation: hotel_id not set in ConversionService")
_LOGGER.error(
"Cannot delete reservation: hotel_id not set in ConversionService"
)
return
_LOGGER.info("Processing deleted reservation: Hotel %s, PMS ID %s", self.hotel_id, pms_reservation_id)
_LOGGER.info(
"Processing deleted reservation: Hotel %s, PMS ID %s",
self.hotel_id,
pms_reservation_id,
)
# Delete conversion records for this hotel + pms_reservation_id
result = await session.execute(
select(Conversion).where(
Conversion.hotel_id == self.hotel_id,
Conversion.pms_reservation_id == pms_reservation_id
Conversion.pms_reservation_id == pms_reservation_id,
)
)
conversions = result.scalars().all()
@@ -612,9 +652,15 @@ class ConversionService:
"daily_sales_count": 0,
}
# Extract reservation metadata
hotel_id = reservation_elem.get("hotelID")
pms_reservation_id = reservation_elem.get("id")
try:
# Extract reservation metadata
pms_reservation_id = int(reservation_elem.get("id"))
except ValueError as e:
_LOGGER.error("Invalid reservation metadata in reservation element: %s", e)
return stats
reservation_number = reservation_elem.get("number")
reservation_date_str = reservation_elem.get("date")
creation_time_str = reservation_elem.get("creationTime")
@@ -684,7 +730,7 @@ class ConversionService:
existing_result = await session.execute(
select(Conversion).where(
Conversion.hotel_id == hotel_id,
Conversion.pms_reservation_id == pms_reservation_id
Conversion.pms_reservation_id == pms_reservation_id,
)
)
existing_conversion = existing_result.scalar_one_or_none()
@@ -711,11 +757,8 @@ class ConversionService:
else:
# Create new conversion entry (without matching - will be done later)
# Note: Guest information (first_name, last_name, email, etc) is stored in ConversionGuest table
conversion = Conversion(
conversion_data = ConversionData(
# Links to existing entities (nullable, will be filled in after matching)
reservation_id=None,
customer_id=None,
hashed_customer_id=None,
# Reservation metadata
hotel_id=hotel_id,
guest_id=guest_id, # Links to ConversionGuest
@@ -730,9 +773,8 @@ class ConversionService:
advertising_partner=advertising_partner,
advertising_campagne=advertising_campagne,
# Metadata
created_at=datetime.now(),
updated_at=datetime.now(),
)
conversion = Conversion(**conversion_data.model_dump())
session.add(conversion)
_LOGGER.debug(
"Created conversion (pms_id=%s)",
@@ -744,9 +786,7 @@ class ConversionService:
# Fetch ALL existing rooms for this conversion (not just the ones in current XML)
existing_rooms_result = await session.execute(
select(ConversionRoom).where(
ConversionRoom.conversion_id == conversion.id
)
select(ConversionRoom).where(ConversionRoom.conversion_id == conversion.id)
)
existing_rooms = {
room.pms_hotel_reservation_id: room
@@ -841,7 +881,6 @@ class ConversionService:
# Check if room reservation already exists using batch-loaded data
existing_room_reservation = existing_rooms.get(pms_hotel_reservation_id)
if existing_room_reservation:
# Update existing room reservation with all fields
existing_room_reservation.arrival_date = arrival_date
@@ -910,7 +949,6 @@ class ConversionService:
return stats
async def _match_by_advertising(
self,
advertising_campagne: str,
@@ -1026,9 +1064,7 @@ class ConversionService:
session = self.session
# Query all hashed customers that match the guest details
query = select(HashedCustomer).options(
selectinload(HashedCustomer.customer)
)
query = select(HashedCustomer).options(selectinload(HashedCustomer.customer))
# Build filter conditions
conditions = []
@@ -1165,6 +1201,7 @@ class ConversionService:
Returns:
Dictionary mapping guest_id to matched HashedCustomer (or None if no match)
"""
# Find all conversions that either:
# - Have no match at all (reservation_id IS NULL AND customer_id IS NULL), OR
@@ -1246,6 +1283,7 @@ class ConversionService:
guest_to_hashed_customer: Mapping from guest_id to matched HashedCustomer
session: AsyncSession for database queries
stats: Shared stats dictionary to update
"""
for guest_id, matched_hashed_customer in guest_to_hashed_customer.items():
if not matched_hashed_customer or not matched_hashed_customer.customer_id:
@@ -1259,7 +1297,10 @@ class ConversionService:
(Conversion.guest_id == guest_id)
& (Conversion.reservation_id.is_(None))
)
.options(selectinload(Conversion.conversion_rooms), selectinload(Conversion.guest))
.options(
selectinload(Conversion.conversion_rooms),
selectinload(Conversion.guest),
)
)
conversions = result.scalars().all()
@@ -1275,7 +1316,10 @@ class ConversionService:
# Try to link each conversion to a reservation for this customer
for conversion in conversions:
matched_reservation, is_attributable = await self._check_if_attributable(
(
matched_reservation,
is_attributable,
) = await self._check_if_attributable(
matched_hashed_customer.customer_id, conversion, session
)
@@ -1329,6 +1373,7 @@ class ConversionService:
Args:
session: AsyncSession for database queries
"""
# Get all ConversionGuests that have ANY customer link
# This includes:
@@ -1345,7 +1390,9 @@ class ConversionService:
_LOGGER.debug("Phase 3d: No matched guests to check for regularity")
return
_LOGGER.debug("Phase 3d: Checking regularity for %d matched guests", len(matched_guests))
_LOGGER.debug(
"Phase 3d: Checking regularity for %d matched guests", len(matched_guests)
)
for conversion_guest in matched_guests:
if not conversion_guest.hashed_customer_id:
@@ -1458,7 +1505,7 @@ class ConversionService:
async def _match_conversion_from_db_safe(
self,
pms_reservation_id: str,
pms_reservation_id: int,
semaphore: asyncio.Semaphore,
stats: dict[str, int],
) -> None:
@@ -1528,12 +1575,15 @@ class ConversionService:
pms_reservation_id: PMS reservation ID to match
session: AsyncSession to use
stats: Shared stats dictionary to update (optional)
"""
if session is None:
session = self.session
if not self.hotel_id:
_LOGGER.error("Cannot match conversion: hotel_id not set in ConversionService")
_LOGGER.error(
"Cannot match conversion: hotel_id not set in ConversionService"
)
return
# Get the conversion from the database with related data
@@ -1541,9 +1591,12 @@ class ConversionService:
select(Conversion)
.where(
Conversion.hotel_id == self.hotel_id,
Conversion.pms_reservation_id == pms_reservation_id
Conversion.pms_reservation_id == pms_reservation_id,
)
.options(
selectinload(Conversion.guest),
selectinload(Conversion.conversion_rooms),
)
.options(selectinload(Conversion.guest), selectinload(Conversion.conversion_rooms))
)
conversion = result.scalar_one_or_none()
@@ -1601,9 +1654,7 @@ class ConversionService:
conversion.reservation_id = (
matched_reservation.id if matched_reservation else None
)
conversion.customer_id = (
matched_customer.id if matched_customer else None
)
conversion.customer_id = matched_customer.id if matched_customer else None
conversion.hashed_customer_id = (
matched_hashed_customer.id if matched_hashed_customer else None
)
@@ -1647,6 +1698,7 @@ class ConversionService:
guest_id: The guest ID to evaluate
customer_id: The matched customer ID
session: AsyncSession for database queries
"""
# Get the ConversionGuest record
guest_result = await session.execute(
@@ -1670,7 +1722,9 @@ class ConversionService:
.order_by(Conversion.reservation_date.asc())
.limit(1)
)
earliest_paying_conversion = earliest_paying_conversion_result.scalar_one_or_none()
earliest_paying_conversion = (
earliest_paying_conversion_result.scalar_one_or_none()
)
if not earliest_paying_conversion:
# No paying conversions found for this guest
@@ -1695,7 +1749,10 @@ class ConversionService:
# (meaning they were already a customer before we sent them a reservation)
# Compare against the reservation's creation date (when WE created/sent it), not check-in date
# Convert created_at to date for comparison with reservation_date (both are dates)
is_regular = earliest_paying_conversion.reservation_date < earliest_reservation.created_at.date()
is_regular = (
earliest_paying_conversion.reservation_date
< earliest_reservation.created_at.date()
)
conversion_guest.is_regular = is_regular
if is_regular:
@@ -1735,6 +1792,7 @@ class ConversionService:
Tuple of (matched_reservation, is_attributable) where:
- matched_reservation: The Reservation that matches (if any)
- is_attributable: True if the reservation's dates match this conversion
"""
# Check if conversion_room dates exist (criterion for attributability)
if not conversion.conversion_rooms:
@@ -1763,12 +1821,12 @@ class ConversionService:
and reservation.end_date
):
# Check if dates match or mostly match (within 7 day tolerance)
arrival_match = abs(
(room.arrival_date - reservation.start_date).days
) <= 7
departure_match = abs(
(room.departure_date - reservation.end_date).days
) <= 7
arrival_match = (
abs((room.arrival_date - reservation.start_date).days) <= 7
)
departure_match = (
abs((room.departure_date - reservation.end_date).days) <= 7
)
if arrival_match and departure_match:
_LOGGER.info(

View File

@@ -4,8 +4,6 @@ import os
from collections.abc import AsyncGenerator, Callable
from typing import TypeVar
from .const import WebhookStatus
from sqlalchemy import (
JSON,
Boolean,
@@ -17,6 +15,7 @@ from sqlalchemy import (
ForeignKeyConstraint,
Index,
Integer,
MetaData,
String,
UniqueConstraint,
func,
@@ -30,6 +29,7 @@ from sqlalchemy.ext.asyncio import (
)
from sqlalchemy.orm import backref, declarative_base, relationship
from .const import WebhookStatus
from .logging_config import get_logger
_LOGGER = get_logger(__name__)
@@ -58,7 +58,16 @@ class Base:
# __table_args__ = {"schema": _SCHEMA}
Base = declarative_base(cls=Base)
# Define naming convention for constraints
metadata = MetaData(naming_convention={
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s"
})
Base = declarative_base(cls=Base, metadata=metadata)
# Type variable for async functions
T = TypeVar("T")
@@ -353,7 +362,10 @@ class HashedCustomer(Base):
__tablename__ = "hashed_customers"
id = Column(Integer, primary_key=True)
customer_id = Column(
Integer, ForeignKey("customers.id", ondelete="SET NULL"), unique=True, nullable=True
Integer,
ForeignKey("customers.id", ondelete="SET NULL"),
unique=True,
nullable=True,
)
contact_id = Column(String, unique=True) # Keep unhashed for reference
hashed_email = Column(String(64)) # SHA256 produces 64 hex chars
@@ -367,7 +379,9 @@ class HashedCustomer(Base):
hashed_birth_date = Column(String(64))
created_at = Column(DateTime(timezone=True))
customer = relationship("Customer", backref=backref("hashed_version", uselist=False, lazy="joined"))
customer = relationship(
"Customer", backref=backref("hashed_version", uselist=False, lazy="joined")
)
class ConversionGuest(Base):
@@ -383,7 +397,13 @@ class ConversionGuest(Base):
__tablename__ = "conversion_guests"
# Natural keys from PMS - composite primary key
hotel_id = Column(String(50), ForeignKey("hotels.hotel_id", ondelete="CASCADE"), nullable=False, primary_key=True, index=True)
hotel_id = Column(
String(50),
ForeignKey("hotels.hotel_id", ondelete="CASCADE"),
nullable=False,
primary_key=True,
index=True,
)
guest_id = Column(Integer, nullable=False, primary_key=True, index=True)
# Unhashed guest information (for reference/transition period)
@@ -401,10 +421,14 @@ class ConversionGuest(Base):
hashed_birth_date = Column(String(64))
# Matched customer reference (nullable, filled after matching)
hashed_customer_id = Column(Integer, ForeignKey("hashed_customers.id"), nullable=True, index=True)
hashed_customer_id = Column(
Integer, ForeignKey("hashed_customers.id"), nullable=True, index=True
)
# Guest classification
is_regular = Column(Boolean, default=False) # True if guest has many prior stays before appearing in our reservations
is_regular = Column(
Boolean, default=False
) # True if guest has many prior stays before appearing in our reservations
# Metadata
first_seen = Column(DateTime(timezone=True))
@@ -428,7 +452,7 @@ class ConversionGuest(Base):
def create_from_conversion_data(
cls,
hotel_id: str,
guest_id: str | None,
guest_id: int | None,
guest_first_name: str | None,
guest_last_name: str | None,
guest_email: str | None,
@@ -483,7 +507,9 @@ class ConversionGuest(Base):
self.hashed_country_code = self._normalize_and_hash(guest_country_code)
if guest_birth_date:
self.guest_birth_date = guest_birth_date
self.hashed_birth_date = self._normalize_and_hash(guest_birth_date.isoformat())
self.hashed_birth_date = self._normalize_and_hash(
guest_birth_date.isoformat()
)
self.last_seen = now
@@ -491,7 +517,9 @@ class Reservation(Base):
__tablename__ = "reservations"
id = Column(Integer, primary_key=True)
customer_id = Column(Integer, ForeignKey("customers.id", ondelete="SET NULL"))
hashed_customer_id = Column(Integer, ForeignKey("hashed_customers.id", ondelete="CASCADE"))
hashed_customer_id = Column(
Integer, ForeignKey("hashed_customers.id", ondelete="CASCADE")
)
unique_id = Column(String, unique=True)
md5_unique_id = Column(String(32), unique=True) # max length 32 guaranteed
start_date = Column(Date)
@@ -578,9 +606,18 @@ class Conversion(Base):
)
# Reservation metadata from XML
hotel_id = Column(String(50), ForeignKey("hotels.hotel_id", ondelete="CASCADE"), nullable=False, index=True) # hotelID attribute
pms_reservation_id = Column(Integer, nullable=False, index=True) # id attribute from reservation
guest_id = Column(Integer, nullable=True, index=True) # PMS guest ID, FK to conversion_guests
hotel_id = Column(
String(50),
ForeignKey("hotels.hotel_id", ondelete="CASCADE"),
nullable=False,
index=True,
) # hotelID attribute
pms_reservation_id = Column(
Integer, nullable=False, index=True
) # id attribute from reservation
guest_id = Column(
Integer, nullable=True, index=True
) # PMS guest ID, FK to conversion_guests
reservation_number = Column(String) # number attribute
reservation_date = Column(Date) # date attribute (when reservation was made)
@@ -588,9 +625,6 @@ class Conversion(Base):
reservation_type = Column(String) # type attribute (e.g., "reservation")
booking_channel = Column(String) # bookingChannel attribute
# Advertising/tracking data - used for matching to existing reservations
advertising_medium = Column(
String, index=True
@@ -603,7 +637,9 @@ class Conversion(Base):
) # advertisingCampagne (contains fbclid/gclid)
# Attribution flags - track how this conversion was matched
directly_attributable = Column(Boolean, default=False) # Matched by ID (high confidence)
directly_attributable = Column(
Boolean, default=False
) # Matched by ID (high confidence)
guest_matched = Column(Boolean, default=False) # Matched by guest details only
# Metadata
@@ -617,7 +653,9 @@ class Conversion(Base):
["conversion_guests.hotel_id", "conversion_guests.guest_id"],
ondelete="SET NULL",
),
UniqueConstraint("hotel_id", "pms_reservation_id", name="uq_conversion_hotel_reservation"),
UniqueConstraint(
"hotel_id", "pms_reservation_id", name="uq_conversion_hotel_reservation"
),
)
# Relationships
@@ -690,7 +728,10 @@ class HotelInventory(Base):
id = Column(Integer, primary_key=True)
hotel_id = Column(
String(50), ForeignKey("hotels.hotel_id", ondelete="CASCADE"), nullable=False, index=True
String(50),
ForeignKey("hotels.hotel_id", ondelete="CASCADE"),
nullable=False,
index=True,
)
inv_type_code = Column(String(8), nullable=False, index=True)
inv_code = Column(String(16), nullable=True, index=True)
@@ -726,7 +767,10 @@ class RoomAvailability(Base):
id = Column(Integer, primary_key=True)
inventory_id = Column(
Integer, ForeignKey("hotel_inventory.id", ondelete="CASCADE"), nullable=False, index=True
Integer,
ForeignKey("hotel_inventory.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
date = Column(Date, nullable=False, index=True)
count_type_2 = Column(Integer, nullable=True)
@@ -739,7 +783,9 @@ class RoomAvailability(Base):
inventory_item = relationship("HotelInventory", back_populates="availability")
__table_args__ = (
UniqueConstraint("inventory_id", "date", name="uq_room_availability_unique_key"),
UniqueConstraint(
"inventory_id", "date", name="uq_room_availability_unique_key"
),
)
@@ -787,7 +833,9 @@ class WebhookEndpoint(Base):
id = Column(Integer, primary_key=True)
# Hotel association
hotel_id = Column(String(50), ForeignKey("hotels.hotel_id"), nullable=False, index=True)
hotel_id = Column(
String(50), ForeignKey("hotels.hotel_id"), nullable=False, index=True
)
# Webhook configuration
webhook_secret = Column(String(64), unique=True, nullable=False, index=True)
@@ -803,7 +851,7 @@ class WebhookEndpoint(Base):
webhook_requests = relationship("WebhookRequest", back_populates="webhook_endpoint")
__table_args__ = (
Index('idx_webhook_endpoint_hotel_type', 'hotel_id', 'webhook_type'),
Index("idx_webhook_endpoint_hotel_type", "hotel_id", "webhook_type"),
)
@@ -816,11 +864,17 @@ class WebhookRequest(Base):
# Request identification
payload_hash = Column(String(64), unique=True, nullable=False, index=True) # SHA256
webhook_endpoint_id = Column(Integer, ForeignKey("webhook_endpoints.id"), nullable=True, index=True)
hotel_id = Column(String(50), ForeignKey("hotels.hotel_id"), nullable=True, index=True)
webhook_endpoint_id = Column(
Integer, ForeignKey("webhook_endpoints.id"), nullable=True, index=True
)
hotel_id = Column(
String(50), ForeignKey("hotels.hotel_id"), nullable=True, index=True
)
# Processing tracking
status = Column(String(20), nullable=False, default=WebhookStatus.PENDING.value, index=True)
status = Column(
String(20), nullable=False, default=WebhookStatus.PENDING.value, index=True
)
# Status values: 'pending', 'processing', 'completed', 'failed' set by Enum WebhookStatus
processing_started_at = Column(DateTime(timezone=True), nullable=True)
@@ -841,16 +895,20 @@ class WebhookRequest(Base):
# Result tracking
created_customer_id = Column(Integer, ForeignKey("customers.id"), nullable=True)
created_reservation_id = Column(Integer, ForeignKey("reservations.id"), nullable=True)
created_reservation_id = Column(
Integer, ForeignKey("reservations.id"), nullable=True
)
# Relationships
webhook_endpoint = relationship("WebhookEndpoint", back_populates="webhook_requests")
webhook_endpoint = relationship(
"WebhookEndpoint", back_populates="webhook_requests"
)
hotel = relationship("Hotel")
customer = relationship("Customer")
reservation = relationship("Reservation")
__table_args__ = (
Index('idx_webhook_status_created', 'status', 'created_at'),
Index('idx_webhook_hotel_created', 'hotel_id', 'created_at'),
Index('idx_webhook_purge_candidate', 'status', 'purged_at', 'created_at'),
Index("idx_webhook_status_created", "status", "created_at"),
Index("idx_webhook_hotel_created", "hotel_id", "created_at"),
Index("idx_webhook_purge_candidate", "status", "purged_at", "created_at"),
)

View File

@@ -11,7 +11,7 @@ from XML generation (xsdata) follows clean architecture principles.
import hashlib
import json
from datetime import date, datetime
from datetime import UTC, date, datetime
from enum import Enum
from typing import Any
@@ -20,6 +20,35 @@ from pydantic import BaseModel, EmailStr, Field, field_validator, model_validato
from .const import WebhookStatus
# Generalized integer validator for reuse across models
def convert_to_int(field_name: str, v: Any) -> int:
"""Convert a value to integer, handling string inputs.
Args:
field_name: Name of the field being validated (for error messages)
v: Value to convert (can be int, str, or None)
Returns:
Integer value
Raises:
ValueError: If value is None or cannot be converted to int
"""
if v is None:
msg = f"{field_name} cannot be None"
raise ValueError(msg)
if isinstance(v, int):
return v
if isinstance(v, str):
try:
return int(v)
except ValueError as e:
msg = f"{field_name} must be a valid integer, got: {v}"
raise ValueError(msg) from e
msg = f"{field_name} must be int or str, got: {type(v)}"
raise ValueError(msg)
# Country name to ISO 3166-1 alpha-2 code mapping
COUNTRY_NAME_TO_CODE = {
# English names
@@ -195,6 +224,7 @@ class CustomerData(BaseModel):
Returns:
2-letter ISO country code (uppercase) or None if input is None/empty
"""
if not v:
return None
@@ -367,8 +397,7 @@ class WebhookRequestData(BaseModel):
# Required fields
payload_json: dict[str, Any] | None = Field(
...,
description="Webhook payload (required for creation, nullable after purge)"
..., description="Webhook payload (required for creation, nullable after purge)"
)
# Auto-calculated from payload_json
@@ -376,7 +405,7 @@ class WebhookRequestData(BaseModel):
None,
min_length=64,
max_length=64,
description="SHA256 hash of canonical JSON payload (auto-calculated)"
description="SHA256 hash of canonical JSON payload (auto-calculated)",
)
# Optional foreign keys
@@ -455,35 +484,133 @@ class WebhookRequestData(BaseModel):
# Example usage in a service layer
class ReservationService:
"""Example service showing how to use Pydantic models with SQLAlchemy."""
class ConversionGuestData(BaseModel):
"""Validated conversion guest data from PMS XML.
def __init__(self, db_session):
self.db_session = db_session
async def create_reservation(
self, reservation_data: ReservationData, customer_data: CustomerData
):
"""Create a reservation with validated data.
The data has already been validated by Pydantic before reaching here.
Handles validation and hashing for guest records extracted from
hotel PMS conversion XML files.
"""
from alpine_bits_python.db import Customer, Reservation
# Convert validated Pydantic model to SQLAlchemy model
db_customer = Customer(**customer_data.model_dump(exclude_none=True))
self.db_session.add(db_customer)
await self.db_session.flush() # Get the customer ID
hotel_id: str = Field(..., min_length=1, max_length=50)
guest_id: int = Field(..., gt=0)
guest_first_name: str | None = Field(None, max_length=100)
guest_last_name: str | None = Field(None, max_length=100)
guest_email: str | None = Field(None, max_length=200)
guest_country_code: str | None = Field(None, max_length=10)
guest_birth_date: date | None = None
# Create reservation linked to customer
db_reservation = Reservation(
customer_id=db_customer.id,
**reservation_data.model_dump(
exclude={"children_ages"}
), # Handle separately
children_ages=",".join(map(str, reservation_data.children_ages)),
# Auto-calculated hashed fields
hashed_first_name: str | None = Field(None, max_length=64)
hashed_last_name: str | None = Field(None, max_length=64)
hashed_email: str | None = Field(None, max_length=64)
hashed_country_code: str | None = Field(None, max_length=64)
hashed_birth_date: str | None = Field(None, max_length=64)
# Timestamps
first_seen: datetime = Field(default_factory=lambda: datetime.now(UTC))
last_seen: datetime = Field(default_factory=lambda: datetime.now(UTC))
@staticmethod
def _normalize_and_hash(value: str | None) -> str | None:
"""Normalize and hash a value for privacy-preserving matching.
Uses the same logic as ConversionGuest._normalize_and_hash.
"""
if value is None or value == "":
return None
# Normalize: lowercase, strip whitespace
normalized = value.lower().strip()
if not normalized:
return None
# Hash with SHA256
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
@model_validator(mode="after")
def calculate_hashes(self) -> "ConversionGuestData":
"""Auto-calculate hashed fields from plain text fields."""
if self.hashed_first_name is None:
self.hashed_first_name = self._normalize_and_hash(self.guest_first_name)
if self.hashed_last_name is None:
self.hashed_last_name = self._normalize_and_hash(self.guest_last_name)
if self.hashed_email is None:
self.hashed_email = self._normalize_and_hash(self.guest_email)
if self.hashed_country_code is None:
self.hashed_country_code = self._normalize_and_hash(self.guest_country_code)
if self.hashed_birth_date is None and self.guest_birth_date is not None:
self.hashed_birth_date = self._normalize_and_hash(
self.guest_birth_date.isoformat()
)
self.db_session.add(db_reservation)
await self.db_session.commit()
return self
return db_reservation, db_customer
@field_validator("guest_id", mode="before")
@classmethod
def convert_guest_id_to_int(cls, v: Any) -> int:
"""Convert guest_id to integer (handles string input from XML)."""
return convert_to_int("guest_id", v)
model_config = {"from_attributes": True}
class ConversionData(BaseModel):
"""Validated conversion data from PMS XML.
Handles validation for conversion records extracted from
hotel PMS conversion XML files. This model ensures proper type conversion
and validation before creating a Conversion database entry.
"""
# Foreign key references (nullable - matched after creation)
reservation_id: int | None = Field(None, gt=0)
customer_id: int | None = Field(None, gt=0)
hashed_customer_id: int | None = Field(None, gt=0)
# Required reservation metadata from PMS
hotel_id: str = Field(..., min_length=1, max_length=50)
pms_reservation_id: int = Field(..., gt=0)
guest_id: int | None = Field(None, gt=0)
# Optional reservation metadata
reservation_number: str | None = Field(None, max_length=100)
reservation_date: date | None = None
creation_time: datetime | None = None
reservation_type: str | None = Field(None, max_length=50)
booking_channel: str | None = Field(None, max_length=100)
# Advertising/tracking data (used for matching)
advertising_medium: str | None = Field(None, max_length=200)
advertising_partner: str | None = Field(None, max_length=200)
advertising_campagne: str | None = Field(None, max_length=500)
# Attribution flags
directly_attributable: bool = Field(default=False)
guest_matched: bool = Field(default=False)
# Timestamps (auto-managed)
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
@field_validator(
"pms_reservation_id", "guest_id", "reservation_id", "customer_id",
"hashed_customer_id", mode="before"
)
@classmethod
def convert_int_fields(cls, v: Any) -> int | None:
"""Convert integer fields from string to int (handles XML input)."""
if v is None or v == "":
return None
# Get the field name from the validation context if available
# For now, use a generic name since we handle multiple fields
return convert_to_int("field", v)
@field_validator("hotel_id", "reservation_number", "reservation_type",
"booking_channel", "advertising_medium", "advertising_partner",
"advertising_campagne", mode="before")
@classmethod
def strip_string_fields(cls, v: str | None) -> str | None:
"""Strip whitespace from string fields."""
if v is None:
return None
stripped = str(v).strip()
return stripped if stripped else None
model_config = {"from_attributes": True}

View File

@@ -9,6 +9,43 @@ from typing import Optional
from xml.etree import ElementTree as ET
def validate_and_convert_id(field_name: str, value: str | int) -> str:
"""Validate that an ID field is convertible to integer and return as string.
This helper ensures ID fields (like reservation_id, guest_id) are valid integers,
which is important since the Pydantic models will convert them from strings to ints.
Args:
field_name: Name of the field for error messages
value: The ID value (can be string or int)
Returns:
String representation of the validated integer ID
Raises:
ValueError: If value cannot be converted to a valid positive integer
"""
def _raise_invalid_type_error():
"""Raise error for invalid ID type."""
msg = (
f"{field_name} must be convertible to a positive integer, "
f"got: {value!r} (type: {type(value).__name__})"
)
raise ValueError(msg)
try:
# Convert to int first to validate it's a valid integer
int_value = int(value)
if int_value <= 0:
msg = f"{field_name} must be a positive integer, got: {value}"
raise ValueError(msg)
# Return as string for XML attributes
return str(int_value)
except (ValueError, TypeError):
_raise_invalid_type_error()
class RoomReservationBuilder:
"""Builder for creating roomReservation XML elements with daily sales."""
@@ -133,7 +170,7 @@ class ReservationXMLBuilder:
def __init__(
self,
hotel_id: str,
reservation_id: str,
reservation_id: str | int,
reservation_number: str,
reservation_date: str,
creation_time: Optional[str] = None,
@@ -146,7 +183,7 @@ class ReservationXMLBuilder:
Args:
hotel_id: Hotel ID
reservation_id: Reservation ID
reservation_id: Reservation ID (must be convertible to positive integer)
reservation_number: Reservation number
reservation_date: Reservation date in YYYY-MM-DD format
creation_time: Creation timestamp (defaults to reservation_date + T00:00:00)
@@ -156,7 +193,7 @@ class ReservationXMLBuilder:
advertising_campagne: Advertising campaign
"""
self.hotel_id = hotel_id
self.reservation_id = reservation_id
self.reservation_id = validate_and_convert_id("reservation_id", reservation_id)
self.reservation_number = reservation_number
self.reservation_date = reservation_date
self.creation_time = creation_time or f"{reservation_date}T00:00:00"
@@ -170,7 +207,7 @@ class ReservationXMLBuilder:
def set_guest(
self,
guest_id: str,
guest_id: str | int,
first_name: str,
last_name: str,
email: str,
@@ -182,7 +219,7 @@ class ReservationXMLBuilder:
"""Set guest information for the reservation.
Args:
guest_id: Guest ID
guest_id: Guest ID (must be convertible to positive integer)
first_name: Guest first name
last_name: Guest last name
email: Guest email
@@ -194,8 +231,9 @@ class ReservationXMLBuilder:
Returns:
Self for method chaining
"""
validated_guest_id = validate_and_convert_id("guest_id", guest_id)
self.guest_data = {
"id": guest_id,
"id": validated_guest_id,
"firstName": first_name,
"lastName": last_name,
"email": email,

View File

@@ -29,7 +29,6 @@ from alpine_bits_python.db import (
ConversionGuest,
ConversionRoom,
Customer,
HashedCustomer,
Reservation,
)
@@ -130,9 +129,9 @@ class TestConversionServiceWithImportedData:
print(f"\nCSV Import Stats: {csv_stats}")
assert csv_stats["total_rows"] > 0, "CSV import should have processed rows"
assert (
csv_stats["created_reservations"] > 0
), "CSV import should create reservations"
assert csv_stats["created_reservations"] > 0, (
"CSV import should create reservations"
)
# Step 2: Load and process conversion XML
with xml_file.open(encoding="utf-8") as f:
@@ -166,30 +165,36 @@ class TestConversionServiceWithImportedData:
EXPECTED_MATCHED_TO_CUSTOMER = 0
print(f"\nBaseline Match Counts:")
print("\nBaseline Match Counts:")
print(f" Total reservations in XML: {EXPECTED_TOTAL_RESERVATIONS}")
print(f" Total daily sales records: {EXPECTED_TOTAL_DAILY_SALES}")
print(f" Total conversion room records: {EXPECTED_TOTAL_ROOMS}")
print(f" Matched to reservation: {EXPECTED_MATCHED_TO_RESERVATION}")
match_rate = (EXPECTED_MATCHED_TO_RESERVATION / EXPECTED_TOTAL_RESERVATIONS * 100) if EXPECTED_TOTAL_RESERVATIONS > 0 else 0
match_rate = (
(EXPECTED_MATCHED_TO_RESERVATION / EXPECTED_TOTAL_RESERVATIONS * 100)
if EXPECTED_TOTAL_RESERVATIONS > 0
else 0
)
print(f" Match rate: {match_rate:.1f}%")
print(f" Matched to customer: {EXPECTED_MATCHED_TO_CUSTOMER}")
print(f" Match rate (to customer): {(EXPECTED_MATCHED_TO_CUSTOMER / EXPECTED_TOTAL_RESERVATIONS * 100) if EXPECTED_TOTAL_RESERVATIONS > 0 else 0:.1f}%")
print(
f" Match rate (to customer): {(EXPECTED_MATCHED_TO_CUSTOMER / EXPECTED_TOTAL_RESERVATIONS * 100) if EXPECTED_TOTAL_RESERVATIONS > 0 else 0:.1f}%"
)
# Verify baseline stability on subsequent runs
assert (
stats["total_reservations"] == EXPECTED_TOTAL_RESERVATIONS
), f"Total reservations should be {EXPECTED_TOTAL_RESERVATIONS}, got {stats['total_reservations']}"
assert (
stats["total_daily_sales"] == EXPECTED_TOTAL_DAILY_SALES
), f"Total daily sales should be {EXPECTED_TOTAL_DAILY_SALES}, got {stats['total_daily_sales']}"
assert (
stats["matched_to_reservation"] == EXPECTED_MATCHED_TO_RESERVATION
), f"Matched reservations should be {EXPECTED_MATCHED_TO_RESERVATION}, got {stats['matched_to_reservation']}"
assert stats["total_reservations"] == EXPECTED_TOTAL_RESERVATIONS, (
f"Total reservations should be {EXPECTED_TOTAL_RESERVATIONS}, got {stats['total_reservations']}"
)
assert stats["total_daily_sales"] == EXPECTED_TOTAL_DAILY_SALES, (
f"Total daily sales should be {EXPECTED_TOTAL_DAILY_SALES}, got {stats['total_daily_sales']}"
)
assert stats["matched_to_reservation"] == EXPECTED_MATCHED_TO_RESERVATION, (
f"Matched reservations should be {EXPECTED_MATCHED_TO_RESERVATION}, got {stats['matched_to_reservation']}"
)
assert (
stats["matched_to_customer"] == EXPECTED_MATCHED_TO_CUSTOMER
), f"Matched customers should be {EXPECTED_MATCHED_TO_CUSTOMER}, got {stats['matched_to_customer']}"
assert stats["matched_to_customer"] == EXPECTED_MATCHED_TO_CUSTOMER, (
f"Matched customers should be {EXPECTED_MATCHED_TO_CUSTOMER}, got {stats['matched_to_customer']}"
)
@pytest.mark.asyncio
async def test_conversion_room_revenue_aggregation(
@@ -237,23 +242,25 @@ class TestConversionServiceWithImportedData:
# Note: Test data may not have revenue values in the XML
# The important thing is that we're capturing room-level data
print(f"\nRevenue Aggregation Stats:")
print("\nRevenue Aggregation Stats:")
print(f" Total conversion rooms: {len(all_rooms)}")
print(f" Rooms with revenue: {len(rooms_with_revenue)}")
if rooms_with_revenue:
# Verify revenue values are numeric and positive
for room in rooms_with_revenue:
assert isinstance(
room.total_revenue, (int, float)
), f"Revenue should be numeric, got {type(room.total_revenue)}"
assert (
room.total_revenue > 0
), f"Revenue should be positive, got {room.total_revenue}"
assert isinstance(room.total_revenue, (int, float)), (
f"Revenue should be numeric, got {type(room.total_revenue)}"
)
assert room.total_revenue > 0, (
f"Revenue should be positive, got {room.total_revenue}"
)
total_revenue = sum(room.total_revenue for room in rooms_with_revenue)
print(f" Total aggregated revenue: {total_revenue}")
print(f" Average revenue per room: {total_revenue / len(rooms_with_revenue)}")
print(
f" Average revenue per room: {total_revenue / len(rooms_with_revenue)}"
)
@pytest.mark.asyncio
async def test_conversion_matching_by_guest_details(
@@ -282,7 +289,9 @@ class TestConversionServiceWithImportedData:
dryrun=False,
)
assert csv_stats["created_reservations"] > 0, "Should have imported reservations"
assert csv_stats["created_reservations"] > 0, (
"Should have imported reservations"
)
# Process conversions
with xml_file.open(encoding="utf-8") as f:
@@ -307,14 +316,14 @@ class TestConversionServiceWithImportedData:
)
conversions_with_customers = result.scalars().all()
print(f"\nGuest Detail Matching:")
print("\nGuest Detail Matching:")
print(f" Total conversions: {len(all_conversions)}")
print(f" Conversions matched to customer: {len(conversions_with_customers)}")
print(f" Stats matched_to_customer: {stats['matched_to_customer']}")
# With this test data, matches may be 0 if guest names/emails don't align
# The important thing is that the matching logic runs without errors
print(f" Note: Matches depend on data alignment between CSV and XML files")
print(" Note: Matches depend on data alignment between CSV and XML files")
@pytest.mark.asyncio
async def test_conversion_service_error_handling(
@@ -354,7 +363,7 @@ class TestConversionServiceWithImportedData:
with room_number='201', second has status='request' with room_number='202'
4. The old room entries (101, 102) should no longer exist in the database
"""
from tests.helpers import ReservationXMLBuilder, MultiReservationXMLBuilder
from tests.helpers import MultiReservationXMLBuilder, ReservationXMLBuilder
# First batch: Process two reservations
multi_builder1 = MultiReservationXMLBuilder()
@@ -363,13 +372,13 @@ class TestConversionServiceWithImportedData:
res1_v1 = (
ReservationXMLBuilder(
hotel_id="39054_001",
reservation_id="res_001",
reservation_number="RES-001",
reservation_id="100",
reservation_number="100",
reservation_date="2025-11-14",
reservation_type="request",
)
.set_guest(
guest_id="guest_001",
guest_id="100",
first_name="Alice",
last_name="Johnson",
email="alice@example.com",
@@ -388,13 +397,13 @@ class TestConversionServiceWithImportedData:
res2_v1 = (
ReservationXMLBuilder(
hotel_id="39054_001",
reservation_id="res_002",
reservation_number="RES-002",
reservation_id="101",
reservation_number="101",
reservation_date="2025-11-15",
reservation_type="reservation",
)
.set_guest(
guest_id="guest_002",
guest_id="101",
first_name="Bob",
last_name="Smith",
email="bob@example.com",
@@ -437,13 +446,13 @@ class TestConversionServiceWithImportedData:
res1_v2 = (
ReservationXMLBuilder(
hotel_id="39054_001",
reservation_id="res_001", # Same ID
reservation_number="RES-001", # Same number
reservation_id="100", # Same ID
reservation_number="100", # Same number
reservation_date="2025-11-14",
reservation_type="reservation", # Changed from request
)
.set_guest(
guest_id="guest_001",
guest_id="100",
first_name="Alice",
last_name="Johnson",
email="alice@example.com",
@@ -462,13 +471,13 @@ class TestConversionServiceWithImportedData:
res2_v2 = (
ReservationXMLBuilder(
hotel_id="39054_001",
reservation_id="res_002", # Same ID
reservation_number="RES-002", # Same number
reservation_id="101", # Same ID
reservation_number="101", # Same number
reservation_date="2025-11-15",
reservation_type="request", # Changed from reservation
)
.set_guest(
guest_id="guest_002",
guest_id="101",
first_name="Bob",
last_name="Smith",
email="bob@example.com",
@@ -533,7 +542,6 @@ class TestConversionServiceWithImportedData:
)
class TestXMLBuilderUsage:
"""Demonstrate usage of XML builder helpers for creating test data."""
@@ -546,12 +554,12 @@ class TestXMLBuilderUsage:
xml_content = (
ReservationXMLBuilder(
hotel_id="39054_001",
reservation_id="test_123",
reservation_number="RES-123",
reservation_id="123",
reservation_number="123",
reservation_date="2025-11-14",
)
.set_guest(
guest_id="guest_001",
guest_id="157",
first_name="John",
last_name="Doe",
email="john@example.com",
@@ -563,7 +571,7 @@ class TestXMLBuilderUsage:
room_type="DZV",
room_number="101",
revenue_logis_per_day=150.0,
adults=2
adults=2,
)
.build_xml()
)
@@ -576,21 +584,19 @@ class TestXMLBuilderUsage:
assert stats["total_daily_sales"] == 5 # 4 nights + departure day
@pytest.mark.asyncio
async def test_using_xml_builder_for_multi_room_reservation(
self, test_db_session
):
async def test_using_xml_builder_for_multi_room_reservation(self, test_db_session):
"""Example: Create a reservation with multiple rooms."""
from tests.helpers import ReservationXMLBuilder
xml_content = (
ReservationXMLBuilder(
hotel_id="39054_001",
reservation_id="test_456",
reservation_number="RES-456",
reservation_id="456",
reservation_number="456",
reservation_date="2025-11-14",
)
.set_guest(
guest_id="guest_002",
guest_id="157",
first_name="Jane",
last_name="Smith",
email="jane@example.com",
@@ -620,7 +626,7 @@ class TestXMLBuilderUsage:
@pytest.mark.asyncio
async def test_using_multi_reservation_builder(self, test_db_session):
"""Example: Create multiple reservations in one XML document."""
from tests.helpers import ReservationXMLBuilder, MultiReservationXMLBuilder
from tests.helpers import MultiReservationXMLBuilder, ReservationXMLBuilder
multi_builder = MultiReservationXMLBuilder()
@@ -628,12 +634,12 @@ class TestXMLBuilderUsage:
res1 = (
ReservationXMLBuilder(
hotel_id="39054_001",
reservation_id="test_001",
reservation_number="RES-001",
reservation_id="175",
reservation_number="175",
reservation_date="2025-11-14",
)
.set_guest(
guest_id="guest_001",
guest_id="157",
first_name="Alice",
last_name="Johnson",
email="alice@example.com",
@@ -650,12 +656,12 @@ class TestXMLBuilderUsage:
res2 = (
ReservationXMLBuilder(
hotel_id="39054_001",
reservation_id="test_002",
reservation_id="2725",
reservation_number="RES-002",
reservation_date="2025-11-15",
)
.set_guest(
guest_id="guest_002",
guest_id="2525",
first_name="Bob",
last_name="Williams",
email="bob@example.com",
@@ -683,14 +689,12 @@ class TestHashedMatchingLogic:
"""Test the hashed matching logic used in ConversionService."""
@pytest.mark.asyncio
async def test_conversion_guest_hashed_fields_are_populated(
self, test_db_session
):
async def test_conversion_guest_hashed_fields_are_populated(self, test_db_session):
"""Test that ConversionGuest properly stores hashed versions of guest data."""
# Create a conversion guest
conversion_guest = ConversionGuest.create_from_conversion_data(
hotel_id="test_hotel",
guest_id="guest_123",
guest_id=123,
guest_first_name="Margaret",
guest_last_name="Brown",
guest_email="margaret@example.com",
@@ -721,7 +725,6 @@ class TestHashedMatchingLogic:
assert conversion_guest.hashed_last_name == expected_hashed_last
assert conversion_guest.hashed_email == expected_hashed_email
@pytest.mark.asyncio
async def test_conversion_records_created_before_matching(
self, test_db_session, test_config
@@ -749,11 +752,13 @@ class TestHashedMatchingLogic:
test_db_session.add(reservation)
await test_db_session.commit()
PMS_RESERVATION_ID = 157
# Create conversion XML with matching hashed data
xml_content = f"""<?xml version="1.0"?>
<root>
<reservation id="pms_123" hotelID="hotel_1" number="RES001" date="2025-01-15">
<guest id="guest_001" firstName="David" lastName="Miller" email="david@example.com"/>
<reservation id="{PMS_RESERVATION_ID}" hotelID="hotel_1" number="378" date="2025-01-15">
<guest id="123" firstName="David" lastName="Miller" email="david@example.com"/>
<roomReservations>
<roomReservation roomNumber="101" arrival="2025-01-15" departure="2025-01-17" status="confirmed">
<dailySales>
@@ -764,12 +769,14 @@ class TestHashedMatchingLogic:
</reservation>
</root>"""
service = ConversionService(test_db_session)
service = ConversionService(test_db_session, hotel_id="hotel_1")
stats = await service.process_conversion_xml(xml_content)
# Verify conversion was created
result = await test_db_session.execute(
select(Conversion).where(Conversion.pms_reservation_id == "pms_123")
select(Conversion).where(
Conversion.pms_reservation_id == PMS_RESERVATION_ID
)
)
conversion = result.scalar_one_or_none()
@@ -779,22 +786,23 @@ class TestHashedMatchingLogic:
# Verify conversion_guest was created with the correct data
from sqlalchemy.orm import selectinload
result_with_guest = await test_db_session.execute(
select(Conversion)
.where(Conversion.pms_reservation_id == "pms_123")
.where(Conversion.pms_reservation_id == PMS_RESERVATION_ID)
.options(selectinload(Conversion.guest))
)
conversion_with_guest = result_with_guest.scalar_one_or_none()
assert conversion_with_guest.guest is not None, "ConversionGuest relationship should exist"
assert conversion_with_guest.guest is not None, (
"ConversionGuest relationship should exist"
)
assert conversion_with_guest.guest.guest_first_name == "David"
assert conversion_with_guest.guest.guest_last_name == "Miller"
assert conversion_with_guest.guest.guest_email == "david@example.com"
# Verify conversion_room was created
room_result = await test_db_session.execute(
select(ConversionRoom).where(
ConversionRoom.conversion_id == conversion.id
)
select(ConversionRoom).where(ConversionRoom.conversion_id == conversion.id)
)
rooms = room_result.scalars().all()
assert len(rooms) > 0, "ConversionRoom should be created"
@@ -804,8 +812,6 @@ class TestHashedMatchingLogic:
assert stats["total_reservations"] == 1
assert stats["total_daily_sales"] == 1
@pytest.mark.asyncio
async def test_conversion_guest_composite_key_prevents_duplicates(
self, test_db_session
@@ -819,7 +825,7 @@ class TestHashedMatchingLogic:
Now the database itself enforces uniqueness at the PK level.
"""
hotel_id = "test_hotel"
guest_id = "guest_123"
guest_id = 123
# Create and commit first conversion guest
guest1 = ConversionGuest.create_from_conversion_data(
@@ -862,6 +868,7 @@ class TestHashedMatchingLogic:
# The composite PK constraint prevents the duplicate insert
from sqlalchemy.exc import IntegrityError
with pytest.raises(IntegrityError):
await test_db_session.commit()

View File

@@ -95,7 +95,7 @@ class TestReservationXMLBuilder:
reservation_date="2025-11-14",
)
builder.set_guest(
guest_id="guest_001",
guest_id="1001",
first_name="John",
last_name="Doe",
email="john@example.com",
@@ -138,7 +138,7 @@ class TestReservationXMLBuilder:
reservation_date="2025-11-14",
)
builder.set_guest(
guest_id="guest_001",
guest_id="1001",
first_name="John",
last_name="Doe",
email="john@example.com",
@@ -179,7 +179,7 @@ class TestReservationXMLBuilder:
advertising_campagne="EAIaIQobChMI...",
)
builder.set_guest(
guest_id="guest_001",
guest_id="1001",
first_name="John",
last_name="Doe",
email="john@example.com",
@@ -213,7 +213,7 @@ class TestMultiReservationXMLBuilder:
reservation_date="2025-11-14",
)
res1.set_guest(
guest_id="guest_001",
guest_id="1001",
first_name="John",
last_name="Doe",
email="john@example.com",
@@ -233,7 +233,7 @@ class TestMultiReservationXMLBuilder:
reservation_date="2025-11-15",
)
res2.set_guest(
guest_id="guest_002",
guest_id="1002",
first_name="Jane",
last_name="Smith",
email="jane@example.com",
@@ -268,7 +268,7 @@ class TestConvenienceFeatures:
reservation_date="2025-11-14",
)
.set_guest(
guest_id="guest_001",
guest_id="1001",
first_name="John",
last_name="Doe",
email="john@example.com",
@@ -294,7 +294,7 @@ class TestConvenienceFeatures:
reservation_date="2025-11-14",
)
builder.set_guest(
guest_id="guest_001",
guest_id="1001",
first_name="John",
last_name="Doe",
email="john@example.com",