From 08f85d1b2656d2c16eefa6721effa1d031542e11 Mon Sep 17 00:00:00 2001 From: Jonas Linter <{email_address}> Date: Wed, 3 Dec 2025 10:41:34 +0100 Subject: [PATCH] Holy db migrations batman --- ...c3_remove_composite_fk_from_conversions.py | 51 +++ config/alpinebits.log | 231 ++++++++++ database_schema_analysis.md | 396 ++++++++++++++++++ reset_database.sh | 47 --- reset_db.sh | 28 ++ src/alpine_bits_python/alpine_bits_helpers.py | 4 +- src/alpine_bits_python/api.py | 2 +- src/alpine_bits_python/conversion_service.py | 4 +- src/alpine_bits_python/csv_import.py | 2 +- src/alpine_bits_python/db.py | 33 +- src/alpine_bits_python/db_setup.py | 6 +- src/alpine_bits_python/email_monitoring.py | 4 +- src/alpine_bits_python/reservation_service.py | 2 +- src/alpine_bits_python/schemas.py | 2 +- .../util/migrate_sqlite_to_postgres.py | 2 +- src/alpine_bits_python/webhook_processor.py | 4 +- tests/test_alpine_bits_server_read.py | 12 +- tests/test_api.py | 4 +- tests/test_conversion_service.py | 2 +- 19 files changed, 752 insertions(+), 84 deletions(-) create mode 100644 alembic/versions/2025_12_03_0950-694d52a883c3_remove_composite_fk_from_conversions.py create mode 100644 database_schema_analysis.md delete mode 100644 reset_database.sh create mode 100755 reset_db.sh diff --git a/alembic/versions/2025_12_03_0950-694d52a883c3_remove_composite_fk_from_conversions.py b/alembic/versions/2025_12_03_0950-694d52a883c3_remove_composite_fk_from_conversions.py new file mode 100644 index 0000000..e26cb73 --- /dev/null +++ b/alembic/versions/2025_12_03_0950-694d52a883c3_remove_composite_fk_from_conversions.py @@ -0,0 +1,51 @@ +"""remove_composite_fk_from_conversions + +Revision ID: 694d52a883c3 +Revises: b50c0f45030a +Create Date: 2025-12-03 09:50:18.506030 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '694d52a883c3' +down_revision: Union[str, Sequence[str], None] = 'b50c0f45030a' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(op.f('conversions_hotel_id_guest_id_fkey'), 'conversions', type_='foreignkey') + + # Rename hotel_code to hotel_id (preserving data) and add FK to hotels + op.add_column('reservations', sa.Column('hotel_id', sa.String(), nullable=True)) + op.execute('UPDATE reservations SET hotel_id = hotel_code') + op.drop_column('reservations', 'hotel_code') + + # Add FK constraint without immediate validation (NOT VALID) + # This allows existing rows with non-existent hotel_ids to remain + # Future inserts/updates will still be validated + op.execute( + 'ALTER TABLE reservations ADD CONSTRAINT fk_reservations_hotel_id_hotels ' + 'FOREIGN KEY (hotel_id) REFERENCES hotels (hotel_id) ON DELETE CASCADE NOT VALID' + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + # Drop FK and rename hotel_id back to hotel_code (preserving data) + op.drop_constraint(op.f('fk_reservations_hotel_id_hotels'), 'reservations', type_='foreignkey') + op.add_column('reservations', sa.Column('hotel_code', sa.VARCHAR(), autoincrement=False, nullable=True)) + op.execute('UPDATE reservations SET hotel_code = hotel_id') + op.drop_column('reservations', 'hotel_id') + + op.create_foreign_key(op.f('conversions_hotel_id_guest_id_fkey'), 'conversions', 'conversion_guests', ['hotel_id', 'guest_id'], ['hotel_id', 'guest_id'], ondelete='SET NULL') + # ### end Alembic commands ### diff --git a/config/alpinebits.log b/config/alpinebits.log index 9994643..9b084f3 100644 --- a/config/alpinebits.log +++ b/config/alpinebits.log @@ -392994,3 +392994,234 @@ DETAIL: Key (hotel_id, guest_id)=(39054_001, 28275) is not present in table "co 2025-11-25 12:03:35 - alpine_bits_python.api - INFO - Email service shut down 2025-11-25 12:03:35 - alpine_bits_python.api - INFO - Application shutdown complete 2025-11-25 12:03:35 - alpine_bits_python.worker_coordination - INFO - Released primary worker lock (pid=22943) +2025-12-03 08:59:46 - root - INFO - Logging to file: config/alpinebits.log +2025-12-03 08:59:46 - root - INFO - Logging configured at INFO level +2025-12-03 08:59:46 - alpine_bits_python.notification_service - INFO - Registered notification backend: pushover +2025-12-03 08:59:46 - alpine_bits_python.notification_manager - INFO - Registered pushover backend with priority 0 +2025-12-03 08:59:46 - alpine_bits_python.notification_manager - INFO - Notification service configured with backends: ['pushover'] +2025-12-03 08:59:46 - alpine_bits_python.api - INFO - Application startup initiated (primary_worker=True) +2025-12-03 08:59:46 - alpine_bits_python.db - INFO - Configured database schema: alpinebits +2025-12-03 08:59:46 - alpine_bits_python.db - INFO - Setting PostgreSQL search_path to: alpinebits,public +2025-12-03 08:59:46 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_NOTIF_REPORT +2025-12-03 08:59:46 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_PING +2025-12-03 08:59:46 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_RES_NOTIF_GUEST_REQUESTS +2025-12-03 08:59:46 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_READ +2025-12-03 08:59:46 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: wix_form +2025-12-03 08:59:46 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: generic +2025-12-03 08:59:46 - alpine_bits_python.webhook_processor - INFO - Webhook processors initialized +2025-12-03 08:59:46 - alpine_bits_python.api - INFO - Webhook processors initialized +2025-12-03 08:59:46 - alpine_bits_python.api - INFO - Hotel 39054_001 has no push_endpoint configured +2025-12-03 08:59:46 - alpine_bits_python.api - INFO - Hotel 135 has no push_endpoint configured +2025-12-03 08:59:46 - alpine_bits_python.api - INFO - Hotel 39052_001 has no push_endpoint configured +2025-12-03 08:59:46 - alpine_bits_python.api - INFO - Hotel 39040_001 has no push_endpoint configured +2025-12-03 08:59:46 - alpine_bits_python.api - INFO - Running startup tasks (primary worker)... +2025-12-03 08:59:46 - alpine_bits_python.hotel_service - INFO - Config sync complete: 0 hotels created, 4 updated, 0 endpoints created +2025-12-03 08:59:46 - alpine_bits_python.db_setup - INFO - Config sync: 0 hotels created, 4 updated, 0 endpoints created +2025-12-03 08:59:47 - alpine_bits_python.db_setup - INFO - Backfilling advertising account IDs for existing reservations... +2025-12-03 08:59:47 - alpine_bits_python.db_setup - INFO - Found 4 hotel(s) with account configurations +2025-12-03 08:59:47 - alpine_bits_python.db_setup - INFO - Backfilling usernames for existing acked_requests... +2025-12-03 08:59:47 - alpine_bits_python.db_setup - INFO - Found 4 hotel(s) with usernames in config +2025-12-03 08:59:47 - alpine_bits_python.db_setup - INFO - Checking for stuck webhooks to reprocess... +2025-12-03 08:59:47 - alpine_bits_python.db_setup - INFO - No stuck webhooks found +2025-12-03 08:59:47 - alpine_bits_python.api - INFO - Startup tasks completed +2025-12-03 08:59:47 - alpine_bits_python.api - INFO - Webhook periodic cleanup task started +2025-12-03 08:59:47 - alpine_bits_python.api - INFO - Application startup complete +2025-12-03 08:59:51 - alpine_bits_python.api - INFO - Application shutdown initiated +2025-12-03 08:59:51 - alpine_bits_python.api - INFO - Webhook cleanup task cancelled +2025-12-03 08:59:51 - alpine_bits_python.api - INFO - Webhook cleanup task stopped +2025-12-03 08:59:51 - alpine_bits_python.email_service - INFO - Shutting down email service thread pool +2025-12-03 08:59:51 - alpine_bits_python.email_service - INFO - Email service thread pool shut down complete +2025-12-03 08:59:51 - alpine_bits_python.api - INFO - Email service shut down +2025-12-03 08:59:51 - alpine_bits_python.api - INFO - Application shutdown complete +2025-12-03 08:59:51 - alpine_bits_python.worker_coordination - INFO - Released primary worker lock (pid=9801) +2025-12-03 10:38:22 - root - INFO - Logging to file: config/alpinebits.log +2025-12-03 10:38:22 - root - INFO - Logging configured at INFO level +2025-12-03 10:38:22 - alpine_bits_python.notification_service - INFO - Registered notification backend: pushover +2025-12-03 10:38:22 - alpine_bits_python.notification_manager - INFO - Registered pushover backend with priority 0 +2025-12-03 10:38:22 - alpine_bits_python.notification_manager - INFO - Notification service configured with backends: ['pushover'] +2025-12-03 10:38:22 - alpine_bits_python.api - INFO - Application startup initiated (primary_worker=True) +2025-12-03 10:38:22 - alpine_bits_python.db - INFO - Configured database schema: alpinebits +2025-12-03 10:38:22 - alpine_bits_python.db - INFO - Setting PostgreSQL search_path to: alpinebits,public +2025-12-03 10:38:22 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_NOTIF_REPORT +2025-12-03 10:38:22 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_PING +2025-12-03 10:38:22 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_RES_NOTIF_GUEST_REQUESTS +2025-12-03 10:38:22 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_READ +2025-12-03 10:38:22 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: wix_form +2025-12-03 10:38:22 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: generic +2025-12-03 10:38:22 - alpine_bits_python.webhook_processor - INFO - Webhook processors initialized +2025-12-03 10:38:22 - alpine_bits_python.api - INFO - Webhook processors initialized +2025-12-03 10:38:22 - alpine_bits_python.api - INFO - Hotel 39054_001 has no push_endpoint configured +2025-12-03 10:38:22 - alpine_bits_python.api - INFO - Hotel 135 has no push_endpoint configured +2025-12-03 10:38:22 - alpine_bits_python.api - INFO - Hotel 39052_001 has no push_endpoint configured +2025-12-03 10:38:22 - alpine_bits_python.api - INFO - Hotel 39040_001 has no push_endpoint configured +2025-12-03 10:38:22 - alpine_bits_python.api - INFO - Running startup tasks (primary worker)... +2025-12-03 10:38:22 - alpine_bits_python.hotel_service - INFO - Created hotel: 39054_001 +2025-12-03 10:38:22 - alpine_bits_python.hotel_service - INFO - Created webhook endpoint for hotel 39054_001, type=wix_form, secret=JWreZtpYZIMDALw71zlLStFcQFdZbBXGGhVd379GX6oeDJE2iZLebCi0Sw2d8A0T +2025-12-03 10:38:22 - alpine_bits_python.hotel_service - INFO - Created webhook endpoint for hotel 39054_001, type=generic, secret=BzBT1xmoHA4EIpupE8YOY2r9dfWG4FJY7pEU4eDD_5RW3cKRRMJXLp6JRlY3Egr3 +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created hotel: 135 +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created webhook endpoint for hotel 135, type=wix_form, secret=0vbn5mCJBIRcHtK2DS9AWFebF8LncbpcR0sDJ7zctD3wWgdPZLdiIO-743HwiljT +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created webhook endpoint for hotel 135, type=generic, secret=ci12B1Q81uvSwpyHppL5n1T5tYRXeJnv2cP4OkWH2FoShlMCYWEuvkmxdLhvR50N +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created hotel: 39052_001 +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created webhook endpoint for hotel 39052_001, type=wix_form, secret=V4BcT_XGcGJg7hcHhH2IVupcW4u231R711tdI-eiv15a-cSyaMlRnqrhUqNh0csC +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created webhook endpoint for hotel 39052_001, type=generic, secret=x1M6_NYYXrHEC3aXFPkyglprNC6U5OhBFT4TW9E8SmEnpSRq0xm_ApWv4-Vl-pe3 +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created hotel: 39040_001 +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created webhook endpoint for hotel 39040_001, type=wix_form, secret=5JMgT0EI0CnRgp7jaHE1rCHQwZFMv1t9wn1yWJEBR5j_2Zrcqz_4W5g6pJBvZw4l +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Created webhook endpoint for hotel 39040_001, type=generic, secret=lrYRwnHMq5B1I_XEH7cUoOPx95zzzfrmJcRoh9C_Rd-WD3kl4F0M-UNetAlRbMVU +2025-12-03 10:38:23 - alpine_bits_python.hotel_service - INFO - Config sync complete: 4 hotels created, 0 updated, 8 endpoints created +2025-12-03 10:38:23 - alpine_bits_python.db_setup - INFO - Config sync: 4 hotels created, 0 updated, 8 endpoints created +2025-12-03 10:38:24 - alpine_bits_python.db_setup - INFO - Backfilling advertising account IDs for existing reservations... +2025-12-03 10:38:24 - alpine_bits_python.db_setup - INFO - Found 4 hotel(s) with account configurations +2025-12-03 10:38:40 - root - INFO - Logging to file: config/alpinebits.log +2025-12-03 10:38:40 - root - INFO - Logging configured at INFO level +2025-12-03 10:38:40 - alpine_bits_python.notification_service - INFO - Registered notification backend: pushover +2025-12-03 10:38:40 - alpine_bits_python.notification_manager - INFO - Registered pushover backend with priority 0 +2025-12-03 10:38:40 - alpine_bits_python.notification_manager - INFO - Notification service configured with backends: ['pushover'] +2025-12-03 10:38:40 - alpine_bits_python.api - INFO - Application startup initiated (primary_worker=True) +2025-12-03 10:38:40 - alpine_bits_python.db - INFO - Configured database schema: alpinebits +2025-12-03 10:38:40 - alpine_bits_python.db - INFO - Setting PostgreSQL search_path to: alpinebits,public +2025-12-03 10:38:40 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_NOTIF_REPORT +2025-12-03 10:38:40 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_PING +2025-12-03 10:38:40 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_RES_NOTIF_GUEST_REQUESTS +2025-12-03 10:38:40 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_READ +2025-12-03 10:38:40 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: wix_form +2025-12-03 10:38:40 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: generic +2025-12-03 10:38:40 - alpine_bits_python.webhook_processor - INFO - Webhook processors initialized +2025-12-03 10:38:40 - alpine_bits_python.api - INFO - Webhook processors initialized +2025-12-03 10:38:40 - alpine_bits_python.api - INFO - Hotel 39054_001 has no push_endpoint configured +2025-12-03 10:38:40 - alpine_bits_python.api - INFO - Hotel 135 has no push_endpoint configured +2025-12-03 10:38:40 - alpine_bits_python.api - INFO - Hotel 39052_001 has no push_endpoint configured +2025-12-03 10:38:40 - alpine_bits_python.api - INFO - Hotel 39040_001 has no push_endpoint configured +2025-12-03 10:38:40 - alpine_bits_python.api - INFO - Running startup tasks (primary worker)... +2025-12-03 10:38:40 - alpine_bits_python.hotel_service - INFO - Config sync complete: 0 hotels created, 4 updated, 0 endpoints created +2025-12-03 10:38:40 - alpine_bits_python.db_setup - INFO - Config sync: 0 hotels created, 4 updated, 0 endpoints created +2025-12-03 10:38:41 - alpine_bits_python.db_setup - INFO - Backfilling advertising account IDs for existing reservations... +2025-12-03 10:38:41 - alpine_bits_python.db_setup - INFO - Found 4 hotel(s) with account configurations +2025-12-03 10:38:53 - root - INFO - Logging to file: config/alpinebits.log +2025-12-03 10:38:53 - root - INFO - Logging configured at INFO level +2025-12-03 10:38:53 - alpine_bits_python.notification_service - INFO - Registered notification backend: pushover +2025-12-03 10:38:53 - alpine_bits_python.notification_manager - INFO - Registered pushover backend with priority 0 +2025-12-03 10:38:53 - alpine_bits_python.notification_manager - INFO - Notification service configured with backends: ['pushover'] +2025-12-03 10:38:53 - alpine_bits_python.api - INFO - Application startup initiated (primary_worker=True) +2025-12-03 10:38:53 - alpine_bits_python.db - INFO - Configured database schema: alpinebits +2025-12-03 10:38:53 - alpine_bits_python.db - INFO - Setting PostgreSQL search_path to: alpinebits,public +2025-12-03 10:38:53 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_NOTIF_REPORT +2025-12-03 10:38:53 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_PING +2025-12-03 10:38:53 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_RES_NOTIF_GUEST_REQUESTS +2025-12-03 10:38:53 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_READ +2025-12-03 10:38:53 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: wix_form +2025-12-03 10:38:53 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: generic +2025-12-03 10:38:53 - alpine_bits_python.webhook_processor - INFO - Webhook processors initialized +2025-12-03 10:38:53 - alpine_bits_python.api - INFO - Webhook processors initialized +2025-12-03 10:38:53 - alpine_bits_python.api - INFO - Hotel 39054_001 has no push_endpoint configured +2025-12-03 10:38:53 - alpine_bits_python.api - INFO - Hotel 135 has no push_endpoint configured +2025-12-03 10:38:53 - alpine_bits_python.api - INFO - Hotel 39052_001 has no push_endpoint configured +2025-12-03 10:38:53 - alpine_bits_python.api - INFO - Hotel 39040_001 has no push_endpoint configured +2025-12-03 10:38:53 - alpine_bits_python.api - INFO - Running startup tasks (primary worker)... +2025-12-03 10:38:53 - alpine_bits_python.hotel_service - INFO - Config sync complete: 0 hotels created, 4 updated, 0 endpoints created +2025-12-03 10:38:53 - alpine_bits_python.db_setup - INFO - Config sync: 0 hotels created, 4 updated, 0 endpoints created +2025-12-03 10:38:54 - alpine_bits_python.db_setup - INFO - Backfilling advertising account IDs for existing reservations... +2025-12-03 10:38:54 - alpine_bits_python.db_setup - INFO - Found 4 hotel(s) with account configurations +2025-12-03 10:38:54 - alpine_bits_python.db_setup - INFO - Backfilling usernames for existing acked_requests... +2025-12-03 10:38:54 - alpine_bits_python.db_setup - INFO - Found 4 hotel(s) with usernames in config +2025-12-03 10:38:54 - alpine_bits_python.db_setup - INFO - Checking for stuck webhooks to reprocess... +2025-12-03 10:38:54 - alpine_bits_python.db_setup - INFO - No stuck webhooks found +2025-12-03 10:38:54 - alpine_bits_python.api - INFO - Startup tasks completed +2025-12-03 10:38:54 - alpine_bits_python.api - INFO - Webhook periodic cleanup task started +2025-12-03 10:38:54 - alpine_bits_python.api - INFO - Application startup complete +2025-12-03 10:39:31 - alpine_bits_python.api - INFO - Application shutdown initiated +2025-12-03 10:39:31 - alpine_bits_python.api - INFO - Webhook cleanup task cancelled +2025-12-03 10:39:31 - alpine_bits_python.api - INFO - Webhook cleanup task stopped +2025-12-03 10:39:31 - alpine_bits_python.email_service - INFO - Shutting down email service thread pool +2025-12-03 10:39:31 - alpine_bits_python.email_service - INFO - Email service thread pool shut down complete +2025-12-03 10:39:31 - alpine_bits_python.api - INFO - Email service shut down +2025-12-03 10:39:31 - alpine_bits_python.api - INFO - Application shutdown complete +2025-12-03 10:39:31 - alpine_bits_python.worker_coordination - INFO - Released primary worker lock (pid=34567) +2025-12-03 10:39:34 - root - INFO - Logging to file: config/alpinebits.log +2025-12-03 10:39:34 - root - INFO - Logging configured at INFO level +2025-12-03 10:39:34 - alpine_bits_python.notification_service - INFO - Registered notification backend: pushover +2025-12-03 10:39:34 - alpine_bits_python.notification_manager - INFO - Registered pushover backend with priority 0 +2025-12-03 10:39:34 - alpine_bits_python.notification_manager - INFO - Notification service configured with backends: ['pushover'] +2025-12-03 10:39:34 - alpine_bits_python.api - INFO - Application startup initiated (primary_worker=True) +2025-12-03 10:39:34 - alpine_bits_python.db - INFO - Configured database schema: alpinebits +2025-12-03 10:39:34 - alpine_bits_python.db - INFO - Setting PostgreSQL search_path to: alpinebits,public +2025-12-03 10:39:34 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_NOTIF_REPORT +2025-12-03 10:39:34 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_PING +2025-12-03 10:39:34 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_HOTEL_RES_NOTIF_GUEST_REQUESTS +2025-12-03 10:39:34 - alpine_bits_python.alpinebits_server - INFO - Initializing action instance for AlpineBitsActionName.OTA_READ +2025-12-03 10:39:34 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: wix_form +2025-12-03 10:39:34 - alpine_bits_python.webhook_processor - INFO - Registered webhook processor: generic +2025-12-03 10:39:34 - alpine_bits_python.webhook_processor - INFO - Webhook processors initialized +2025-12-03 10:39:34 - alpine_bits_python.api - INFO - Webhook processors initialized +2025-12-03 10:39:34 - alpine_bits_python.api - INFO - Hotel 39054_001 has no push_endpoint configured +2025-12-03 10:39:34 - alpine_bits_python.api - INFO - Hotel 135 has no push_endpoint configured +2025-12-03 10:39:34 - alpine_bits_python.api - INFO - Hotel 39052_001 has no push_endpoint configured +2025-12-03 10:39:34 - alpine_bits_python.api - INFO - Hotel 39040_001 has no push_endpoint configured +2025-12-03 10:39:34 - alpine_bits_python.api - INFO - Running startup tasks (primary worker)... +2025-12-03 10:39:34 - alpine_bits_python.hotel_service - INFO - Config sync complete: 0 hotels created, 4 updated, 0 endpoints created +2025-12-03 10:39:34 - alpine_bits_python.db_setup - INFO - Config sync: 0 hotels created, 4 updated, 0 endpoints created +2025-12-03 10:39:35 - alpine_bits_python.db_setup - INFO - Backfilling advertising account IDs for existing reservations... +2025-12-03 10:39:35 - alpine_bits_python.db_setup - INFO - Found 4 hotel(s) with account configurations +2025-12-03 10:39:35 - alpine_bits_python.db_setup - INFO - Backfilling usernames for existing acked_requests... +2025-12-03 10:39:35 - alpine_bits_python.db_setup - INFO - Found 4 hotel(s) with usernames in config +2025-12-03 10:39:35 - alpine_bits_python.db_setup - INFO - Checking for stuck webhooks to reprocess... +2025-12-03 10:39:35 - alpine_bits_python.db_setup - INFO - No stuck webhooks found +2025-12-03 10:39:35 - alpine_bits_python.api - INFO - Startup tasks completed +2025-12-03 10:39:35 - alpine_bits_python.api - INFO - Webhook periodic cleanup task started +2025-12-03 10:39:35 - alpine_bits_python.api - INFO - Application startup complete +2025-12-03 10:39:50 - alpine_bits_python.api - INFO - AlpineBits authentication successful for user: bemelman (from config) +2025-12-03 10:39:50 - alpine_bits_python.api - INFO - XML file queued for processing: logs/conversions_import/file_bemelman_20251203_103950.xml by user bemelman (original: file.xml) +2025-12-03 10:39:50 - alpine_bits_python.api - INFO - Starting database processing of file.xml +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Loaded 1764 reservations into cache +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Reservation cache initialized with 6 hotel codes +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Processing deleted reservation: Hotel 39054_001, PMS ID 74423 +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Processing 32 reservations in xml +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Phase 0: Extracted 24 unique guests from 32 reservations +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Phase 1: Successfully upserted 24 guests +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 32770 (pms_id=65675) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 36046 (pms_id=71642) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 34158 (pms_id=68197) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 36811 (pms_id=73332) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 35904 (pms_id=71360) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37465 (pms_id=74400) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37466 (pms_id=74401) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37471 (pms_id=74406) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37472 (pms_id=74407) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37473 (pms_id=74408) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37474 (pms_id=74409) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37475 (pms_id=74410) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37476 (pms_id=74412) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37477 (pms_id=74411) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37478 (pms_id=74413) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37479 (pms_id=74414) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37398 (pms_id=74315) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37212 (pms_id=74028) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37480 (pms_id=74415) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37210 (pms_id=74027) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37481 (pms_id=74416) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37483 (pms_id=74417) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37446 (pms_id=74380) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37437 (pms_id=74369) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37484 (pms_id=74418) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37482 (pms_id=74419) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37486 (pms_id=74420) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37487 (pms_id=74421) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37489 (pms_id=74422) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37485 (pms_id=74424) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37488 (pms_id=74425) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Updated conversion 37490 (pms_id=74426) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Phase 3a: Matched conversion by advertising ID (pms_id=74401, reservation_id=1736) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Phase 3a: Matched conversion by advertising ID (pms_id=74411, reservation_id=1751) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Phase 3a: Matched conversion by advertising ID (pms_id=74027, reservation_id=503) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Phase 3a: Matched conversion by advertising ID (pms_id=74028, reservation_id=503) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Phase 3a: Matched conversion by advertising ID (pms_id=74413, reservation_id=1749) +2025-12-03 10:39:50 - alpine_bits_python.conversion_service - INFO - Phase 3a: Matched conversion by advertising ID (pms_id=74424, reservation_id=1754) +2025-12-03 10:39:54 - alpine_bits_python.conversion_service - INFO - Phase 3b: Found 22138 unique guests from 34438 unmatched conversions +2025-12-03 10:40:12 - alpine_bits_python.api - INFO - Conversion processing complete for file.xml: {'total_reservations': 32, 'deleted_reservations': 1, 'total_daily_sales': 501, 'matched_to_reservation': 6, 'matched_to_customer': 0, 'matched_to_hashed_customer': 0, 'unmatched': 26, 'errors': 0} +2025-12-03 10:41:22 - alpine_bits_python.api - INFO - Application shutdown initiated +2025-12-03 10:41:22 - alpine_bits_python.api - INFO - Webhook cleanup task cancelled +2025-12-03 10:41:22 - alpine_bits_python.api - INFO - Webhook cleanup task stopped +2025-12-03 10:41:22 - alpine_bits_python.email_service - INFO - Shutting down email service thread pool +2025-12-03 10:41:22 - alpine_bits_python.email_service - INFO - Email service thread pool shut down complete +2025-12-03 10:41:22 - alpine_bits_python.api - INFO - Email service shut down +2025-12-03 10:41:22 - alpine_bits_python.api - INFO - Application shutdown complete +2025-12-03 10:41:22 - alpine_bits_python.worker_coordination - INFO - Released primary worker lock (pid=34833) diff --git a/database_schema_analysis.md b/database_schema_analysis.md new file mode 100644 index 0000000..c2f981d --- /dev/null +++ b/database_schema_analysis.md @@ -0,0 +1,396 @@ +# Database Schema Analysis + +## Overview +This document analyzes the database schema for normalization issues, redundancy, and potential improvements. + +## Schema Summary +The database contains 13 tables organized around several core concepts: +- **Customer/Guest Management**: `customers`, `hashed_customers`, `conversion_guests` +- **Reservations**: `reservations`, `conversions`, `conversion_rooms` +- **Hotels**: `hotels`, `hotel_inventory`, `room_availability` +- **Webhooks**: `webhook_endpoints`, `webhook_requests` +- **Tracking**: `acked_requests` + +--- + +## Major Issues Identified + +### 1. **CRITICAL: Dual Customer Systems (Data Duplication)** + +**Problem**: The schema maintains two parallel customer tracking systems: +- `customers` + `hashed_customers` (from Wix forms) +- `conversion_guests` (from PMS) + +**Impact**: +- Same person can exist in both systems with no linkage +- `conversion_guests.hashed_customer_id` attempts to link but this is backward (many-to-one instead of one-to-one) +- Data inconsistency when same guest appears in both sources + +**Details**: +``` +customers (id=1, email="john@example.com") + └─ hashed_customers (id=1, customer_id=1, hashed_email="abc123...") + +conversion_guests (hotel_id="HOTEL1", guest_id=42, guest_email="john@example.com") + └─ hashed_customer_id = NULL (or points to hashed_customers.id=1 after matching) +``` + +**Recommendation**: +- Create a unified `persons` table with a `source` field ("wix", "pms", "merged") +- Both `customers` and `conversion_guests` should reference this unified entity +- Implement proper guest matching/merging logic + +--- + +### 2. **Data Redundancy: Hashed Values Stored Separately** + +**Problem**: `hashed_customers` and `conversion_guests` store hashed values in separate columns alongside originals. + +**Current Structure**: +``` +customers: + - email_address (plaintext) + - phone (plaintext) + +hashed_customers: + - customer_id (FK to customers) + - hashed_email + - hashed_phone + - hashed_given_name + ... +``` + +**Issues**: +- Violates 3NF (derived data stored in separate table) +- Synchronization required between `customers` and `hashed_customers` +- If customer data changes, hashed version can become stale +- Extra JOIN required for every Meta Conversion API call + +**Better Approach**: +Option A: Store hashed values directly in `customers` table as additional columns +Option B: Compute hashes on-the-fly (SHA256 is fast, ~1-2ms per hash) + +**Recommendation**: +- **Short term**: Keep current structure but add triggers to auto-update hashed values +- **Long term**: Move hashed columns into `customers` table directly + +--- + +### 3. **Advertising Account IDs Duplicated Across Tables** + +**Problem**: `meta_account_id` and `google_account_id` appear in 3 places: +- `hotels` table (canonical source) +- `reservations` table (copied at creation time) +- Derived from `fbclid`/`gclid` tracking parameters + +**Current Flow**: +``` +hotels.meta_account_id = "123456" + ↓ +reservation created with fbclid + ↓ +reservations.meta_account_id = "123456" (copied from hotels) +``` + +**Issues**: +- Denormalization without clear benefit +- If hotel's account ID changes, old reservations have stale data +- Mixed source of truth (sometimes from hotels, sometimes from tracking params) + +**Recommendation**: +- Remove `meta_account_id` and `google_account_id` from `reservations` +- Always derive from `hotels` table via JOIN +- If tracking-derived account differs from hotel's account, log a warning + +--- + +### 4. **Hotel Information Duplicated in Reservations** + +**Problem**: `reservations` table stores `hotel_code` and `hotel_name` but has no FK to `hotels` table. + +**Issues**: +- Data can become inconsistent if hotel name changes +- No referential integrity +- Unclear if `hotel_code` matches `hotels.hotel_id` + +**Recommendation**: +- Add `hotel_id` FK column to `reservations` pointing to `hotels.hotel_id` +- Remove `hotel_code` and `hotel_name` columns +- Derive hotel information via JOIN when needed + +--- + +### 5. **Weak Foreign Key Consistency** + +**Problem**: Mixed use of `ON DELETE` policies: +- Some FKs use `SET NULL` (appropriate for nullable relationships) +- Some use `CASCADE` (appropriate for child records) +- Some use `NO ACTION` (prevents deletion, may cause issues) +- `conversions` table has confusing composite FK setup with `hotel_id` and `guest_id` + +**Examples**: +```sql +-- Good: Child data should be deleted with parent +hotel_inventory.hotel_id → hotels.hotel_id (ON DELETE CASCADE) + +-- Questionable: Should webhook requests survive hotel deletion? +webhook_requests.hotel_id → hotels.hotel_id (ON DELETE NO ACTION) + +-- Inconsistent: Why SET NULL vs CASCADE? +reservations.customer_id → customers.id (ON DELETE SET NULL) +reservations.hashed_customer_id → hashed_customers.id (ON DELETE CASCADE) +``` + +**Recommendation**: +Review each FK and establish consistent policies: +- Core data (hotels, customers): SET NULL to preserve historical records +- Supporting data (hashed_customers, inventory): CASCADE +- Transactional data (webhooks, conversions): Decide on retention policy + +--- + +### 6. **Confusing Composite Foreign Key in Conversions** + +**Problem**: The `conversions` table has a composite FK that's incorrectly mapped: + +```python +# In db.py lines 650-655 +__table_args__ = ( + ForeignKeyConstraint( + ["hotel_id", "guest_id"], + ["conversion_guests.hotel_id", "conversion_guests.guest_id"], + ondelete="SET NULL", + ), +) +``` + +**But the database shows**: +``` +Foreign Keys: + hotel_id -> conversion_guests.hotel_id (ON DELETE SET NULL) + guest_id -> conversion_guests.hotel_id (ON DELETE SET NULL) # ← WRONG! + guest_id -> conversion_guests.guest_id (ON DELETE SET NULL) + hotel_id -> conversion_guests.guest_id (ON DELETE SET NULL) # ← WRONG! +``` + +**Impact**: +- Database has 4 FKs instead of 1 composite FK +- Mapping is incorrect (guest_id → hotel_id doesn't make sense) +- Could cause constraint violations or allow orphaned records + +**Recommendation**: +- Fix the composite FK definition in SQLAlchemy +- Run a migration to drop incorrect FKs and recreate properly + +--- + +### 7. **Unclear Relationship Between Reservations and Conversions** + +**Problem**: The relationship between `reservations` (from Wix forms) and `conversions` (from PMS) is complex: + +``` +conversions: + - reservation_id (FK to reservations) - matched by tracking IDs + - customer_id (FK to customers) - matched by guest details + - hashed_customer_id (FK to hashed_customers) - matched by hashed guest details + - guest_id (FK to conversion_guests) - the actual PMS guest +``` + +**Issues**: +- Three different FK fields to three different customer/guest tables +- Matching logic is unclear from schema alone +- `directly_attributable` and `guest_matched` flags indicate matching quality, but this should be more explicit + +**Recommendation**: +- Add a `match_confidence` enum field: "exact_id", "high_confidence", "medium_confidence", "no_match" +- Add `match_method` field to explain how the link was made +- Consider a separate `reservation_conversion_links` table to make the many-to-many relationship explicit + +--- + +### 8. **Room Type Information Scattered** + +**Problem**: Room information appears in multiple places: +- `reservations.room_type_code`, `room_classification_code`, `room_type` +- `conversion_rooms.room_type`, `room_number` +- `hotel_inventory.inv_type_code`, `inv_code`, `room_name` + +**Issues**: +- No clear master data for room types +- Room type codes not standardized across sources +- No FK between `reservations.room_type_code` and `hotel_inventory.inv_type_code` + +**Recommendation**: +- Create a `room_types` reference table linked to hotels +- Add FKs from reservations and conversion_rooms to room_types +- Standardize room type codes across all sources + +--- + +## Normalization Analysis + +### 1st Normal Form (1NF): ✅ PASS +- All columns contain atomic values +- **Exception**: `reservations.children_ages` stores comma-separated values + - Should be: separate `reservation_children` table with age column + +### 2nd Normal Form (2NF): ⚠️ MOSTLY PASS +- All non-key attributes depend on the full primary key +- **Issue**: Some denormalized data exists (hotel names, account IDs in reservations) + +### 3rd Normal Form (3NF): ❌ FAIL +Multiple violations: +- `hashed_customers` stores derived data (hashes) that depend on `customers` +- `reservations.meta_account_id` depends on `hotels` via hotel_code +- `reservations.hotel_name` depends on `hotels` via hotel_code + +--- + +## Data Integrity Issues + +### Missing Foreign Keys +1. **reservations.hotel_code** → should FK to hotels.hotel_id +2. **reservations.room_type_code** → should FK to hotel_inventory +3. **acked_requests.unique_id** → should FK to reservations.unique_id (or be nullable) + +### Missing Indexes +Consider adding for query performance: +1. `customers.email_address` - for lookups during conversion matching +2. `conversions.reservation_date` - for time-based queries +3. `conversion_rooms.total_revenue` - for revenue analytics +4. `reservations.start_date`, `end_date` - for date range queries + +### Missing Constraints +1. **Check constraints** for date logic: + - `reservations.end_date > start_date` + - `conversion_rooms.departure_date > arrival_date` + +2. **Check constraints** for counts: + - `num_adults >= 0`, `num_children >= 0` + +3. **NOT NULL constraints** on critical fields: + - `customers.contact_id` should be NOT NULL (it's the natural key) + - `conversions.hotel_id` is NOT NULL ✓ (good) + +--- + +## Recommendations Priority + +### HIGH PRIORITY (Data Integrity) +1. Fix composite FK in `conversions` table (lines 650-655 in db.py) +2. Add `hotel_id` FK to `reservations` table +3. Add missing NOT NULL constraints on natural keys +4. Add check constraints for date ranges and counts + +### MEDIUM PRIORITY (Normalization) +5. Unify customer/guest systems into a single `persons` entity +6. Remove duplicate account ID fields from `reservations` +7. Remove `hotel_name` from `reservations` (derive via JOIN) +8. Create `reservation_children` table for children_ages + +### LOW PRIORITY (Performance & Cleanup) +9. Move hashed fields into `customers` table (remove `hashed_customers`) +10. Add indexes for common query patterns +11. Create `room_types` reference table +12. Add `match_confidence` and `match_method` to `conversions` + +--- + +## Positive Aspects + +✅ Good use of composite keys (`conversion_guests`, `hotel_inventory`) +✅ Unique constraints on natural keys (`contact_id`, `webhook_secret`) +✅ Proper use of indexes on frequently queried fields +✅ Cascade deletion for child records (inventory, rooms) +✅ Tracking metadata (created_at, updated_at, first_seen, last_seen) +✅ Webhook deduplication via `payload_hash` +✅ JSON storage for flexible data (`conversion_rooms.daily_sales`) + +--- + +## Suggested Refactoring Path + +### Phase 1: Fix Critical Issues (1-2 days) +- Fix composite FK in conversions +- Add hotel_id FK to reservations +- Add missing constraints + +### Phase 2: Normalize Customer Data (3-5 days) +- Create unified persons/guests table +- Migrate existing data +- Update matching logic + +### Phase 3: Clean Up Redundancy (2-3 days) +- Remove duplicate account IDs +- Merge hashed_customers into customers +- Create room_types reference + +### Phase 4: Enhance Tracking (1-2 days) +- Add match_confidence fields +- Improve conversion attribution +- Add missing indexes + +--- + +## Query Examples Affected by Current Issues + +### Issue: Duplicate Customer Data +```sql +-- Current: Find all reservations for a guest (requires checking both systems) +SELECT r.* FROM reservations r +WHERE r.customer_id = ? + OR r.hashed_customer_id IN ( + SELECT id FROM hashed_customers WHERE contact_id = ? + ); + +-- After fix: Simple unified query +SELECT r.* FROM reservations r +WHERE r.person_id = ?; +``` + +### Issue: Missing Hotel FK +```sql +-- Current: Get hotel info for reservation (unreliable) +SELECT r.*, r.hotel_name +FROM reservations r +WHERE r.id = ?; + +-- After fix: Reliable JOIN +SELECT r.*, h.hotel_name, h.meta_account_id +FROM reservations r +JOIN hotels h ON r.hotel_id = h.hotel_id +WHERE r.id = ?; +``` + +### Issue: Hashed Data in Separate Table +```sql +-- Current: Get customer for Meta API (requires JOIN) +SELECT hc.hashed_email, hc.hashed_phone +FROM reservations r +JOIN hashed_customers hc ON r.hashed_customer_id = hc.id +WHERE r.id = ?; + +-- After fix: Direct access +SELECT c.hashed_email, c.hashed_phone +FROM reservations r +JOIN customers c ON r.customer_id = c.id +WHERE r.id = ?; +``` + +--- + +## Conclusion + +The schema is **functional but has significant normalization and consistency issues**. The main problems are: + +1. **Dual customer tracking systems** that should be unified +2. **Redundant storage of derived data** (hashes, account IDs) +3. **Missing foreign key relationships** (hotels, room types) +4. **Inconsistent deletion policies** across foreign keys +5. **Broken composite foreign key** in conversions table + +The database violates 3NF in several places and could benefit from a refactoring effort. However, the issues are primarily architectural rather than critical bugs, so the system can continue operating while improvements are made incrementally. + +**Estimated effort to fix all issues**: 1-2 weeks of development + testing +**Risk level**: Medium (requires data migration and careful FK updates) +**Recommended approach**: Incremental fixes starting with high-priority items diff --git a/reset_database.sh b/reset_database.sh deleted file mode 100644 index a43691c..0000000 --- a/reset_database.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Reset database and initialize Alembic from scratch - -echo "=== Database Reset Script ===" -echo "This will drop all tables and reinitialize with Alembic" -echo "" -read -p "Are you sure? (type 'yes' to continue): " confirm - -if [ "$confirm" != "yes" ]; then - echo "Aborted." - exit 1 -fi - -echo "" -echo "Step 1: Dropping all tables in the database..." -echo "Connect to your database and run:" -echo "" -echo " -- For PostgreSQL:" -echo " DROP SCHEMA public CASCADE;" -echo " CREATE SCHEMA public;" -echo " GRANT ALL ON SCHEMA public TO ;" -echo " GRANT ALL ON SCHEMA public TO public;" -echo "" -echo " -- Or if using a custom schema (e.g., alpinebits):" -echo " DROP SCHEMA alpinebits CASCADE;" -echo " CREATE SCHEMA alpinebits;" -echo "" -echo "Press Enter after you've run the SQL commands..." -read - -echo "" -echo "Step 2: Running Alembic migrations..." -uv run alembic upgrade head - -if [ $? -eq 0 ]; then - echo "" - echo "=== Success! ===" - echo "Database has been reset and migrations applied." - echo "" - echo "Current migration status:" - uv run alembic current -else - echo "" - echo "=== Error ===" - echo "Migration failed. Check the error messages above." - exit 1 -fi diff --git a/reset_db.sh b/reset_db.sh new file mode 100755 index 0000000..85e0dfb --- /dev/null +++ b/reset_db.sh @@ -0,0 +1,28 @@ +#!/bin/bash + + +# Recreate the database: run DROP and CREATE in separate psql calls (DROP DATABASE cannot run inside a transaction block) +if ! docker exec -i meta_timescaledb psql -U meta_user -d postgres -c "DROP DATABASE IF EXISTS meta_insights;"; then + echo "Error: failed to drop database 'meta_insights'." >&2 + exit 1 +fi + +if ! docker exec -i meta_timescaledb psql -U meta_user -d postgres -c "CREATE DATABASE meta_insights;"; then + echo "Error: failed to create database 'meta_insights'." >&2 + exit 1 +fi + +# then import dump specified by argument only if previous commands succeeded +if [ -n "$1" ]; then + DUMP_FILE="$1" + if [ ! -r "$DUMP_FILE" ]; then + echo "Error: dump file '$DUMP_FILE' does not exist or is not readable." >&2 + exit 2 + fi + + echo "Importing dump from $DUMP_FILE" + if ! docker exec -i meta_timescaledb psql -U meta_user -d meta_insights < "$DUMP_FILE"; then + echo "Error: failed to import dump '$DUMP_FILE' into 'meta_insights'." >&2 + exit 3 + fi +fi \ No newline at end of file diff --git a/src/alpine_bits_python/alpine_bits_helpers.py b/src/alpine_bits_python/alpine_bits_helpers.py index 27fad1e..ced4dcd 100644 --- a/src/alpine_bits_python/alpine_bits_helpers.py +++ b/src/alpine_bits_python/alpine_bits_helpers.py @@ -768,9 +768,9 @@ def _process_single_reservation( hotel_reservation_id=[hotel_res_id] ) - if reservation.hotel_code is None: + if reservation.hotel_id is None: raise ValueError("Reservation hotel_code is None") - hotel_code = str(reservation.hotel_code) + hotel_code = str(reservation.hotel_id) hotel_name = None if reservation.hotel_name is None else str(reservation.hotel_name) basic_property_info = HotelReservation.ResGlobalInfo.BasicPropertyInfo( diff --git a/src/alpine_bits_python/api.py b/src/alpine_bits_python/api.py index faab88f..27ef88a 100644 --- a/src/alpine_bits_python/api.py +++ b/src/alpine_bits_python/api.py @@ -138,7 +138,7 @@ async def push_listener(customer: DBCustomer, reservation: DBReservation, hotel) server: AlpineBitsServer = app.state.alpine_bits_server hotel_id = hotel["hotel_id"] - reservation_hotel_id = reservation.hotel_code + reservation_hotel_id = reservation.hotel_id # Double-check hotel matching (should be guaranteed by dispatcher) if hotel_id != reservation_hotel_id: diff --git a/src/alpine_bits_python/conversion_service.py b/src/alpine_bits_python/conversion_service.py index 1b78b98..43847f6 100644 --- a/src/alpine_bits_python/conversion_service.py +++ b/src/alpine_bits_python/conversion_service.py @@ -642,7 +642,7 @@ class ConversionService: # Organize by hotel_code for efficient lookup for reservation in reservations: - hotel_code = reservation.hotel_code + hotel_code = reservation.hotel_id if hotel_code not in self._reservation_cache: self._reservation_cache[hotel_code] = [] # Cache the hashed customer - prefer direct relationship, fall back to customer relationship @@ -1095,7 +1095,7 @@ class ConversionService: # Add hotel filter if available if hotel_id: - query = query.where(Reservation.hotel_code == hotel_id) + query = query.where(Reservation.hotel_id == hotel_id) # Execute query db_result = await session.execute(query) diff --git a/src/alpine_bits_python/csv_import.py b/src/alpine_bits_python/csv_import.py index 22539e5..cf50d0c 100644 --- a/src/alpine_bits_python/csv_import.py +++ b/src/alpine_bits_python/csv_import.py @@ -472,7 +472,7 @@ class CSVImporter: num_adults=num_adults, num_children=num_children, children_ages=children_ages, - hotel_code=final_hotel_code, + hotel_id=final_hotel_code, hotel_name=final_hotel_name, offer=str(row.get("room_offer", "")).strip() or None, user_comment=str(row.get("message", "")).strip() or None, diff --git a/src/alpine_bits_python/db.py b/src/alpine_bits_python/db.py index ec8a499..f367e8c 100644 --- a/src/alpine_bits_python/db.py +++ b/src/alpine_bits_python/db.py @@ -27,7 +27,7 @@ from sqlalchemy.ext.asyncio import ( async_sessionmaker, create_async_engine, ) -from sqlalchemy.orm import backref, declarative_base, relationship +from sqlalchemy.orm import backref, declarative_base, foreign, relationship from .const import WebhookStatus from .logging_config import get_logger @@ -435,7 +435,13 @@ class ConversionGuest(Base): last_seen = Column(DateTime(timezone=True)) # Relationships - conversions = relationship("Conversion", back_populates="guest") + conversions = relationship( + "Conversion", + back_populates="guest", + foreign_keys="[Conversion.hotel_id, Conversion.guest_id]", + primaryjoin="and_(ConversionGuest.hotel_id == foreign(Conversion.hotel_id), " + "ConversionGuest.guest_id == foreign(Conversion.guest_id))", + ) hashed_customer = relationship("HashedCustomer", backref="conversion_guests") @staticmethod @@ -541,8 +547,8 @@ class Reservation(Base): # Advertising account IDs (stored conditionally based on fbclid/gclid presence) meta_account_id = Column(String) google_account_id = Column(String) - # Add hotel_code and hotel_name for XML - hotel_code = Column(String) + # Add hotel_id and hotel_name for XML + hotel_id = Column(String, ForeignKey("hotels.hotel_id", ondelete="CASCADE")) hotel_name = Column(String) # RoomTypes fields (optional) room_type_code = Column(String) @@ -569,7 +575,7 @@ class AckedRequest(Base): ) # Username of the client making the request unique_id = Column( String, index=True - ) # Should match Reservation.form_id or another unique field + ) # Matches the md5_unique_id in Reservation timestamp = Column(DateTime(timezone=True)) @@ -646,13 +652,10 @@ class Conversion(Base): created_at = Column(DateTime(timezone=True)) # When this record was imported updated_at = Column(DateTime(timezone=True)) # When this record was last updated - # Composite foreign key constraint for ConversionGuest (hotel_id, guest_id) + # Table constraints + # Note: The relationship to ConversionGuest is handled via SQLAlchemy ORM + # by matching (hotel_id, guest_id) pairs, no DB-level FK constraint needed __table_args__ = ( - ForeignKeyConstraint( - ["hotel_id", "guest_id"], - ["conversion_guests.hotel_id", "conversion_guests.guest_id"], - ondelete="SET NULL", - ), UniqueConstraint( "hotel_id", "pms_reservation_id", name="uq_conversion_hotel_reservation" ), @@ -662,7 +665,13 @@ class Conversion(Base): reservation = relationship("Reservation", backref="conversions") customer = relationship("Customer", backref="conversions") hashed_customer = relationship("HashedCustomer", backref="conversions") - guest = relationship("ConversionGuest", back_populates="conversions") + guest = relationship( + "ConversionGuest", + back_populates="conversions", + foreign_keys="[Conversion.hotel_id, Conversion.guest_id]", + primaryjoin="and_(Conversion.hotel_id == ConversionGuest.hotel_id, " + "Conversion.guest_id == ConversionGuest.guest_id)", + ) conversion_rooms = relationship( "ConversionRoom", back_populates="conversion", cascade="all, delete-orphan" ) diff --git a/src/alpine_bits_python/db_setup.py b/src/alpine_bits_python/db_setup.py index b56a28a..eaf3538 100644 --- a/src/alpine_bits_python/db_setup.py +++ b/src/alpine_bits_python/db_setup.py @@ -115,7 +115,7 @@ async def backfill_advertising_account_ids( sql = text( "UPDATE reservations " "SET meta_account_id = :meta_account " - "WHERE hotel_code = :hotel_id " + "WHERE hotel_id = :hotel_id " "AND fbclid IS NOT NULL " "AND fbclid != '' " "AND (meta_account_id IS NULL OR meta_account_id = '')" @@ -141,7 +141,7 @@ async def backfill_advertising_account_ids( sql = text( "UPDATE reservations " "SET google_account_id = :google_account " - "WHERE hotel_code = :hotel_id " + "WHERE hotel_id = :hotel_id " "AND gclid IS NOT NULL " "AND gclid != '' " "AND (google_account_id IS NULL OR google_account_id = '')" @@ -215,7 +215,7 @@ async def backfill_acked_requests_username( UPDATE acked_requests SET username = :username WHERE unique_id IN ( - SELECT md5_unique_id FROM reservations WHERE hotel_code = :hotel_id + SELECT md5_unique_id FROM reservations WHERE hotel_id = :hotel_id ) AND username IS NULL """ diff --git a/src/alpine_bits_python/email_monitoring.py b/src/alpine_bits_python/email_monitoring.py index 0c7ce55..cb004a9 100644 --- a/src/alpine_bits_python/email_monitoring.py +++ b/src/alpine_bits_python/email_monitoring.py @@ -523,10 +523,10 @@ class ReservationStatsCollector: async with self.async_sessionmaker() as session: # Query reservations created in the reporting period result = await session.execute( - select(Reservation.hotel_code, func.count(Reservation.id)) + select(Reservation.hotel_id, func.count(Reservation.id)) .where(Reservation.created_at >= period_start) .where(Reservation.created_at < period_end) - .group_by(Reservation.hotel_code) + .group_by(Reservation.hotel_id) ) hotel_counts = dict(result.all()) diff --git a/src/alpine_bits_python/reservation_service.py b/src/alpine_bits_python/reservation_service.py index 12c0a2a..c04752e 100644 --- a/src/alpine_bits_python/reservation_service.py +++ b/src/alpine_bits_python/reservation_service.py @@ -181,7 +181,7 @@ class ReservationService: if end_date: filters.append(Reservation.created_at <= end_date) if hotel_code: - filters.append(Reservation.hotel_code == hotel_code) + filters.append(Reservation.hotel_id == hotel_code) if filters: query = query.where(and_(*filters)) diff --git a/src/alpine_bits_python/schemas.py b/src/alpine_bits_python/schemas.py index 852cf84..638c546 100644 --- a/src/alpine_bits_python/schemas.py +++ b/src/alpine_bits_python/schemas.py @@ -131,7 +131,7 @@ class ReservationData(BaseModel): num_adults: int = Field(..., ge=1) num_children: int = Field(0, ge=0, le=10) children_ages: list[int] = Field(default_factory=list) - hotel_code: str = Field(..., min_length=1, max_length=50) + hotel_id: str = Field(..., min_length=1, max_length=50) hotel_name: str | None = Field(None, max_length=200) offer: str | None = Field(None, max_length=500) user_comment: str | None = Field(None, max_length=2000) diff --git a/src/alpine_bits_python/util/migrate_sqlite_to_postgres.py b/src/alpine_bits_python/util/migrate_sqlite_to_postgres.py index 5eefc1b..10094f5 100644 --- a/src/alpine_bits_python/util/migrate_sqlite_to_postgres.py +++ b/src/alpine_bits_python/util/migrate_sqlite_to_postgres.py @@ -306,7 +306,7 @@ async def migrate_data( user_comment=reservation.user_comment, fbclid=reservation.fbclid, gclid=reservation.gclid, - hotel_code=reservation.hotel_code, + hotel_code=reservation.hotel_id, hotel_name=reservation.hotel_name, room_type_code=reservation.room_type_code, room_classification_code=reservation.room_classification_code, diff --git a/src/alpine_bits_python/webhook_processor.py b/src/alpine_bits_python/webhook_processor.py index e051c7e..633e916 100644 --- a/src/alpine_bits_python/webhook_processor.py +++ b/src/alpine_bits_python/webhook_processor.py @@ -247,7 +247,7 @@ async def process_wix_form_submission( num_adults=num_adults, num_children=num_children, children_ages=children_ages, - hotel_code=hotel_code, + hotel_id=hotel_code, hotel_name=hotel_name, offer=offer, created_at=submissionTime, @@ -575,7 +575,7 @@ async def process_generic_webhook_submission( "num_adults": num_adults, "num_children": num_children, "children_ages": children_ages, - "hotel_code": hotel_code, + "hotel_id": hotel_code, "hotel_name": hotel_name, "offer": selected_offers_str, "utm_source": utm_source, diff --git a/tests/test_alpine_bits_server_read.py b/tests/test_alpine_bits_server_read.py index e2ba282..5ca83a3 100644 --- a/tests/test_alpine_bits_server_read.py +++ b/tests/test_alpine_bits_server_read.py @@ -98,7 +98,7 @@ def sample_reservation(sample_customer): user_comment="Late check-in requested", fbclid="PAZXh0bgNhZW0BMABhZGlkAasmYBTNE3QBp1jWuJ9zIpfEGRJMP63fMAMI405yvG5EtH-OT0PxSkAbBJaudFHR6cMtkdHu_aem_fopaFtECyVPNW9fmWfEkyA", gclid="", - hotel_code="HOTEL123", + hotel_id="HOTEL123", hotel_name="Alpine Paradise Resort", ) data = reservation.model_dump(exclude_none=True) @@ -136,7 +136,7 @@ def minimal_reservation(minimal_customer): num_adults=1, num_children=0, children_ages=[], - hotel_code="HOTEL123", + hotel_id="HOTEL123", created_at=datetime(2024, 12, 2, 12, 0, 0, tzinfo=UTC), hotel_name="Alpine Paradise Resort", ) @@ -403,7 +403,7 @@ class TestEdgeCases: num_adults=1, num_children=0, children_ages="", - hotel_code="HOTEL123", + hotel_id="HOTEL123", created_at=datetime.now(UTC), ) @@ -434,7 +434,7 @@ class TestEdgeCases: num_adults=2, num_children=0, children_ages=[], - hotel_code="HOTEL123", + hotel_id="HOTEL123", created_at=datetime.now(UTC), utm_source="facebook", utm_medium="social", @@ -851,7 +851,7 @@ class TestAcknowledgments: num_adults=2, num_children=0, children_ages=[], - hotel_code="HOTEL123", + hotel_id="HOTEL123", hotel_name="Alpine Paradise Resort", created_at=datetime(2024, 11, 1, 12, 0, 0, tzinfo=UTC), ) @@ -863,7 +863,7 @@ class TestAcknowledgments: num_adults=2, num_children=1, children_ages=[10], - hotel_code="HOTEL123", + hotel_id="HOTEL123", hotel_name="Alpine Paradise Resort", created_at=datetime(2024, 11, 15, 10, 0, 0, tzinfo=UTC), ) diff --git a/tests/test_api.py b/tests/test_api.py index b67c89b..1fcafac 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -523,7 +523,7 @@ class TestGenericWebhookEndpoint: (r for r in reservations if r.customer_id == customer.id), None ) assert reservation is not None, "Reservation should be created" - assert reservation.hotel_code == "HOTEL123" + assert reservation.hotel_id == "HOTEL123" assert reservation.hotel_name == "Test Hotel" assert reservation.num_adults == 2 assert reservation.num_children == 1 @@ -614,7 +614,7 @@ class TestGenericWebhookEndpoint: result = await session.execute(select(Reservation)) reservations = result.scalars().all() reservation = next( - (r for r in reservations if r.hotel_code == "HOTEL123"), None + (r for r in reservations if r.hotel_id == "HOTEL123"), None ) assert reservation is not None, "Reservation should be created" assert reservation.num_children == 3 diff --git a/tests/test_conversion_service.py b/tests/test_conversion_service.py index 9aa1ec2..8df6f34 100644 --- a/tests/test_conversion_service.py +++ b/tests/test_conversion_service.py @@ -747,7 +747,7 @@ class TestHashedMatchingLogic: reservation = Reservation( customer_id=customer.id, unique_id="res_6", - hotel_code="hotel_1", + hotel_id="hotel_1", ) test_db_session.add(reservation) await test_db_session.commit()