From 8f2565b5a9e2474f3b1f425564f78ba995cccb5a Mon Sep 17 00:00:00 2001 From: Jonas Linter <{email_address}> Date: Tue, 14 Oct 2025 14:28:47 +0200 Subject: [PATCH] Generic webhook now gets saved to database --- src/alpine_bits_python/api.py | 353 ++++++++++++++++++++++++++++++---- tests/test_api.py | 241 +++++++++++++++++++++-- 2 files changed, 542 insertions(+), 52 deletions(-) diff --git a/src/alpine_bits_python/api.py b/src/alpine_bits_python/api.py index 6269c54..b4b3c89 100644 --- a/src/alpine_bits_python/api.py +++ b/src/alpine_bits_python/api.py @@ -2,6 +2,7 @@ import asyncio import gzip import json import os +import traceback import urllib.parse from collections import defaultdict from datetime import date, datetime @@ -480,6 +481,249 @@ async def process_wix_form_submission(request: Request, data: dict[str, Any], db } +async def process_generic_webhook_submission( + request: Request, data: dict[str, Any], db +): + """Process generic webhook submissions with nested structure. + + Expected structure: + { + "hotel_data": {"hotelname": "...", "hotelcode": "..."}, + "form_data": { + "sprache": "de/it/en", + "anreise": "DD.MM.YYYY", + "abreise": "DD.MM.YYYY", + "erwachsene": "N", + "kinder": "N", + "alter": {"1": "age1", "2": "age2", ...}, + "anrede": "...", + "name": "...", + "nachname": "...", + "mail": "...", + "tel": "...", + "nachricht": "..." + }, + "tracking_data": { + "utm_source": "...", + "utm_medium": "...", + "utm_campaign": "...", + "utm_content": "...", + "utm_term": "...", + "fbclid": "...", + "gclid": "..." + }, + "timestamp": "ISO8601" + } + """ + timestamp = datetime.now().isoformat() + _LOGGER.info("Processing generic webhook submission at %s", timestamp) + + # Extract nested data + hotel_data = data.get("hotel_data", {}) + form_data = data.get("form_data", {}) + tracking_data = data.get("tracking_data", {}) + + # Extract hotel information + hotel_code = hotel_data.get("hotelcode") + hotel_name = hotel_data.get("hotelname") + + if not hotel_code: + _LOGGER.warning( + "No hotel_code provided in webhook data, using default" + ) + hotel_code = request.app.state.config.get("default_hotel_code", "123") + + if not hotel_name: + hotel_name = ( + request.app.state.config.get("default_hotel_name") or "Frangart Inn" + ) + + # Extract customer information + first_name = form_data.get("name") + last_name = form_data.get("nachname") + email = form_data.get("mail") + phone_number = form_data.get("tel") + name_prefix = form_data.get("anrede") + language = form_data.get("sprache", "de")[:2] + user_comment = form_data.get("nachricht", "") + + # Parse dates - handle DD.MM.YYYY format + start_date_str = form_data.get("anreise") + end_date_str = form_data.get("abreise") + + if not start_date_str or not end_date_str: + raise HTTPException( + status_code=400, + detail="Missing required dates (anreise/abreise)" + ) + + try: + # Parse DD.MM.YYYY format using strptime + start_date = datetime.strptime(start_date_str, "%d.%m.%Y").date() + end_date = datetime.strptime(end_date_str, "%d.%m.%Y").date() + except ValueError as e: + _LOGGER.error( + "Error parsing dates: start=%s, end=%s, error=%s", + start_date_str, + end_date_str, + e, + ) + raise HTTPException( + status_code=400, detail=f"Invalid date format: {e}" + ) from e + + # Extract room/guest info + num_adults = int(form_data.get("erwachsene", 2)) + num_children = int(form_data.get("kinder", 0)) + + # Extract children ages from nested structure + children_ages = [] + if num_children > 0: + alter_data = form_data.get("alter", {}) + for i in range(1, num_children + 1): + age_str = alter_data.get(str(i)) + if age_str: + try: + children_ages.append(int(age_str)) + except ValueError: + _LOGGER.warning( + "Invalid age value for child %d: %s", i, age_str + ) + + # Extract tracking information + utm_source = tracking_data.get("utm_source") + utm_medium = tracking_data.get("utm_medium") + utm_campaign = tracking_data.get("utm_campaign") + utm_term = tracking_data.get("utm_term") + utm_content = tracking_data.get("utm_content") + fbclid = tracking_data.get("fbclid") + gclid = tracking_data.get("gclid") + + # Parse submission timestamp + submission_time = data.get("timestamp") + try: + if submission_time: + # Handle ISO8601 format with timezone + if submission_time.endswith("Z"): + submission_time = datetime.fromisoformat(submission_time[:-1]) + elif "+" in submission_time: + # Remove timezone info (e.g., +02:00) + submission_time = datetime.fromisoformat( + submission_time.split("+")[0] + ) + else: + submission_time = datetime.fromisoformat(submission_time) + except Exception as e: + _LOGGER.exception("Error parsing submission timestamp: %s", e) + submission_time = None + + # Generate unique ID + unique_id = generate_unique_id() + + # Use CustomerService to handle customer creation/update with hashing + customer_service = CustomerService(db) + + customer_data = { + "given_name": first_name, + "surname": last_name, + "contact_id": None, + "name_prefix": name_prefix if name_prefix != "--" else None, + "email_address": email, + "phone": phone_number if phone_number else None, + "email_newsletter": False, + "address_line": None, + "city_name": None, + "postal_code": None, + "country_code": None, + "gender": None, + "birth_date": None, + "language": language, + "address_catalog": False, + "name_title": None, + } + + # Create/update customer + db_customer = await customer_service.get_or_create_customer(customer_data) + + # Create reservation + reservation_kwargs = { + "unique_id": unique_id, + "start_date": start_date, + "end_date": end_date, + "num_adults": num_adults, + "num_children": num_children, + "children_ages": children_ages, + "hotel_code": hotel_code, + "hotel_name": hotel_name, + "offer": None, + "utm_source": utm_source, + "utm_medium": utm_medium, + "utm_campaign": utm_campaign, + "utm_term": utm_term, + "utm_content": utm_content, + "user_comment": user_comment, + "fbclid": fbclid, + "gclid": gclid, + } + + # Only include created_at if we have a valid submission_time + if submission_time: + reservation_kwargs["created_at"] = submission_time + + reservation = ReservationData(**reservation_kwargs) + + if reservation.md5_unique_id is None: + raise HTTPException( + status_code=400, detail="Failed to generate md5_unique_id" + ) + + # Use ReservationService to create reservation + reservation_service = ReservationService(db) + db_reservation = await reservation_service.create_reservation( + reservation, db_customer.id + ) + + async def push_event(): + # Fire event for listeners (push, etc.) - hotel-specific dispatch + dispatcher = getattr(request.app.state, "event_dispatcher", None) + if dispatcher: + # Get hotel_code from reservation to target the right listeners + hotel_code = getattr(db_reservation, "hotel_code", None) + if hotel_code and hotel_code.strip(): + await dispatcher.dispatch_for_hotel( + "form_processed", hotel_code, db_customer, db_reservation + ) + _LOGGER.info( + "Dispatched form_processed event for hotel %s", hotel_code + ) + else: + _LOGGER.warning( + "No hotel_code in reservation, skipping push notifications" + ) + + # Create task and store reference to prevent garbage collection + task = asyncio.create_task(push_event()) + # Add done callback to log any exceptions + task.add_done_callback( + lambda t: t.exception() if not t.cancelled() else None + ) + + _LOGGER.info( + "Successfully processed generic webhook: customer_id=%s, " + "reservation_id=%s", + db_customer.id, + db_reservation.id, + ) + + return { + "status": "success", + "message": "Generic webhook data processed successfully", + "customer_id": db_customer.id, + "reservation_id": db_reservation.id, + "timestamp": timestamp, + } + + async def validate_basic_auth( credentials: HTTPBasicCredentials = Depends(security_basic), ) -> str: @@ -601,57 +845,98 @@ async def handle_wix_form_test( @api_router.post("/webhook/generic") @webhook_limiter.limit(WEBHOOK_RATE_LIMIT) -async def handle_generic_webhook(request: Request, data: dict[str, Any]): +async def handle_generic_webhook( + request: Request, db_session=Depends(get_async_session) +): """Handle generic webhook endpoint for receiving JSON payloads. - Logs the data to file for later analysis. Does not process the data - or save to database since the structure is not yet known. + Supports gzip compression, extracts customer and reservation data, + saves to database, and triggers push notifications. No authentication required for this endpoint. """ + # Store data for error logging if needed + data = None + try: timestamp = datetime.now().isoformat() _LOGGER.info("Received generic webhook data at %s", timestamp) - # Create log entry with metadata - log_entry = { + # Get the raw body content + body = await request.body() + + if not body: + raise HTTPException( + status_code=400, detail="ERROR: No content provided" + ) + + # Check if content is gzip compressed + content_encoding = request.headers.get("content-encoding", "").lower() + is_gzipped = content_encoding == "gzip" + + # Decompress if gzipped + if is_gzipped: + try: + body = gzip.decompress(body) + _LOGGER.info("Successfully decompressed gzip content") + except Exception as e: + raise HTTPException( + status_code=400, + detail=f"ERROR: Failed to decompress gzip content: {e}", + ) from e + + # Parse JSON + try: + data = json.loads(body.decode("utf-8")) + except json.JSONDecodeError as e: + raise HTTPException( + status_code=400, + detail=f"ERROR: Invalid JSON content: {e}", + ) from e + + # Process the webhook data and save to database + await process_generic_webhook_submission(request, data, db_session) + + return { + "status": "success", + "message": "Generic webhook data received and processed successfully", "timestamp": timestamp, - "client_ip": request.client.host if request.client else "unknown", - "headers": dict(request.headers), - "data": data, - "origin_header": request.headers.get("origin"), } - # Create logs directory if it doesn't exist - logs_dir = Path("logs/generic_webhooks") - if not logs_dir.exists(): - logs_dir.mkdir(parents=True, mode=0o755, exist_ok=True) - _LOGGER.info("Created directory: %s", logs_dir) - - # Generate log filename with timestamp - log_filename = ( - logs_dir / f"webhook_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - ) - - # Write log file - with log_filename.open("w", encoding="utf-8") as f: - json.dump(log_entry, f, indent=2, default=str, ensure_ascii=False) - - _LOGGER.info("Generic webhook data logged to: %s", log_filename) - + except HTTPException: + raise except Exception as e: _LOGGER.exception("Error in handle_generic_webhook") + + # Log error data to file asynchronously (only on error) + error_log_entry = { + "timestamp": datetime.now().isoformat(), + "client_ip": request.client.host if request.client else "unknown", + "headers": dict(request.headers), + "data": data, # Include the parsed data if available + "error": str(e), + "traceback": traceback.format_exc(), + } + + # Use asyncio to run file I/O in thread pool to avoid blocking + error_logs_dir = Path("logs/errors") + await asyncio.to_thread( + error_logs_dir.mkdir, parents=True, mode=0o755, exist_ok=True + ) + + error_log_filename = error_logs_dir / ( + f"generic_webhook_error_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + ) + await asyncio.to_thread( + error_log_filename.write_text, + json.dumps(error_log_entry, indent=2, default=str, ensure_ascii=False), + encoding="utf-8", + ) + + _LOGGER.error("Error data logged to: %s", error_log_filename) raise HTTPException( status_code=500, detail="Error processing generic webhook data" ) from e - else: - return { - "status": "success", - "message": "Generic webhook data received successfully", - "data_logged_to": str(log_filename), - "timestamp": timestamp, - "note": "Data logged for later analysis", - } @api_router.put("/hoteldata/conversions_import/{filename:path}") diff --git a/tests/test_api.py b/tests/test_api.py index f70e736..d361147 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -352,16 +352,40 @@ class TestWixWebhookEndpoint: class TestGenericWebhookEndpoint: """Test generic webhook endpoint.""" - def test_generic_webhook_success(self, client): - """Test successful generic webhook submission.""" + def test_generic_webhook_success_with_real_data(self, client): + """Test successful generic webhook submission with real form data.""" + unique_id = uuid.uuid4().hex[:8] test_data = { - "event_type": "test_event", - "data": { - "key1": "value1", - "key2": "value2", - "nested": {"foo": "bar"}, + "hotel_data": { + "hotelname": "Bemelmans", + "hotelcode": "39054_001" }, - "metadata": {"source": "test_system"}, + "form_data": { + "sprache": "it", + "anreise": "14.10.2025", + "abreise": "15.10.2025", + "erwachsene": "1", + "kinder": "2", + "alter": { + "1": "2", + "2": "4" + }, + "anrede": "Herr", + "name": "Armin", + "nachname": "Wieser", + "mail": f"test.{unique_id}@example.com", + "tel": "+391234567890", + "nachricht": "Test message" + }, + "tracking_data": { + "utm_source": "ig", + "utm_medium": "Instagram_Stories", + "utm_campaign": "Conversions_Apartment_Bemelmans_ITA", + "utm_content": "Grafik_1_Apartments_Bemelmans", + "utm_term": "Cold_Traffic_Conversions_Apartment_Bemelmans_ITA", + "fbclid": "test_fbclid_123" + }, + "timestamp": "2025-10-14T12:20:08+02:00" } response = client.post("/api/webhook/generic", json=test_data) @@ -370,20 +394,202 @@ class TestGenericWebhookEndpoint: data = response.json() assert data["status"] == "success" assert "timestamp" in data - assert "data_logged_to" in data - assert "generic_webhooks" in data["data_logged_to"] - assert data["note"] == "Data logged for later analysis" + assert data["message"] == "Generic webhook data received and processed successfully" + + def test_generic_webhook_creates_customer_and_reservation(self, client): + """Test that webhook creates customer and reservation in database.""" + unique_id = uuid.uuid4().hex[:8] + test_data = { + "hotel_data": { + "hotelname": "Test Hotel", + "hotelcode": "TEST123" + }, + "form_data": { + "sprache": "de", + "anreise": "25.12.2025", + "abreise": "31.12.2025", + "erwachsene": "2", + "kinder": "1", + "alter": {"1": "8"}, + "anrede": "Frau", + "name": "Maria", + "nachname": "Schmidt", + "mail": f"maria.{unique_id}@example.com", + "tel": "+491234567890", + "nachricht": "Looking forward to our stay" + }, + "tracking_data": { + "utm_source": "google", + "utm_medium": "cpc", + "utm_campaign": "winter2025" + }, + "timestamp": "2025-10-14T10:00:00Z" + } + + response = client.post("/api/webhook/generic", json=test_data) + assert response.status_code == 200 + + # Verify data was saved to database + async def check_db(): + engine = client.app.state.engine + async_session = async_sessionmaker(engine, expire_on_commit=False) + async with async_session() as session: + from sqlalchemy import select + + # Check customer was created + result = await session.execute(select(Customer)) + customers = result.scalars().all() + # Find the customer we just created + customer = next( + (c for c in customers if c.email_address == f"maria.{unique_id}@example.com"), + None + ) + assert customer is not None, "Customer should be created" + assert customer.given_name == "Maria" + assert customer.surname == "Schmidt" + assert customer.phone == "+491234567890" + assert customer.language == "de" + assert customer.name_prefix == "Frau" + + # Check reservation was created + result = await session.execute(select(Reservation)) + reservations = result.scalars().all() + reservation = next( + (r for r in reservations if r.customer_id == customer.id), + None + ) + assert reservation is not None, "Reservation should be created" + assert reservation.hotel_code == "TEST123" + assert reservation.hotel_name == "Test Hotel" + assert reservation.num_adults == 2 + assert reservation.num_children == 1 + # children_ages is stored as CSV string + children_ages = [int(age) for age in reservation.children_ages.split(",") if age] + assert len(children_ages) == 1 + assert children_ages[0] == 8 + assert reservation.utm_source == "google" + assert reservation.utm_campaign == "winter2025" + + import asyncio + asyncio.run(check_db()) + + def test_generic_webhook_missing_dates(self, client): + """Test webhook with missing required dates.""" + test_data = { + "hotel_data": {"hotelname": "Test", "hotelcode": "123"}, + "form_data": { + "sprache": "de", + "name": "John", + "nachname": "Doe", + "mail": "john@example.com" + # Missing anreise and abreise + }, + "tracking_data": {} + } + + response = client.post("/api/webhook/generic", json=test_data) + # HTTPException with 400 is raised, then caught and returns 500 + assert response.status_code in [400, 500] + + def test_generic_webhook_invalid_date_format(self, client): + """Test webhook with invalid date format.""" + test_data = { + "hotel_data": {"hotelname": "Test", "hotelcode": "123"}, + "form_data": { + "sprache": "en", + "anreise": "2025-10-14", # Wrong format, should be DD.MM.YYYY + "abreise": "2025-10-15", + "erwachsene": "2", + "kinder": "0", + "name": "Jane", + "nachname": "Doe", + "mail": "jane@example.com" + }, + "tracking_data": {} + } + + response = client.post("/api/webhook/generic", json=test_data) + # HTTPException with 400 is raised, then caught and returns 500 + assert response.status_code in [400, 500] + + def test_generic_webhook_with_children_ages(self, client): + """Test webhook properly handles children ages.""" + unique_id = uuid.uuid4().hex[:8] + test_data = { + "hotel_data": {"hotelname": "Family Hotel", "hotelcode": "FAM001"}, + "form_data": { + "sprache": "it", + "anreise": "01.08.2025", + "abreise": "15.08.2025", + "erwachsene": "2", + "kinder": "3", + "alter": { + "1": "5", + "2": "8", + "3": "12" + }, + "anrede": "--", # Should be filtered out + "name": "Paolo", + "nachname": "Rossi", + "mail": f"paolo.{unique_id}@example.com", + "tel": "", # Empty phone + "nachricht": "" + }, + "tracking_data": { + "fbclid": "test_fb_123", + "gclid": "test_gc_456" + } + } + + response = client.post("/api/webhook/generic", json=test_data) + assert response.status_code == 200 + + # Verify children ages were stored correctly + async def check_db(): + engine = client.app.state.engine + async_session = async_sessionmaker(engine, expire_on_commit=False) + async with async_session() as session: + from sqlalchemy import select + + result = await session.execute(select(Reservation)) + reservations = result.scalars().all() + reservation = next( + (r for r in reservations if r.hotel_code == "FAM001"), + None + ) + assert reservation is not None + assert reservation.num_children == 3 + # children_ages is stored as CSV string + children_ages = [ + int(age) for age in reservation.children_ages.split(",") if age + ] + assert children_ages == [5, 8, 12] + assert reservation.fbclid == "test_fb_123" + assert reservation.gclid == "test_gc_456" + + # Check customer with empty phone and -- prefix + result = await session.execute(select(Customer)) + customers = result.scalars().all() + customer = next( + (c for c in customers if c.email_address == f"paolo.{unique_id}@example.com"), + None + ) + assert customer is not None + assert customer.phone is None # Empty phone should be None + assert customer.name_prefix is None # -- should be filtered out + + import asyncio + asyncio.run(check_db()) def test_generic_webhook_empty_payload(self, client): """Test generic webhook with empty payload.""" response = client.post("/api/webhook/generic", json={}) - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" + # Should fail gracefully with error logging (400 or 500) + assert response.status_code in [400, 500] def test_generic_webhook_complex_nested_data(self, client): - """Test generic webhook with complex nested data structures.""" + """Test generic webhook logs complex nested data structures.""" complex_data = { "arrays": [1, 2, 3], "nested": {"level1": {"level2": {"level3": "deep"}}}, @@ -392,9 +598,8 @@ class TestGenericWebhookEndpoint: response = client.post("/api/webhook/generic", json=complex_data) - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" + # This should fail to process but succeed in logging (400 or 500) + assert response.status_code in [400, 500] class TestAlpineBitsServerEndpoint: