Added date_fields

2025-10-21 17:16:30 +02:00
parent 3fe923f5a7
commit 6e4cc7ed1d
5 changed files with 293 additions and 26 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
    image: timescale/timescaledb:latest-pg16
    container_name: meta_timescaledb
    ports:
-      - "5432:5432"
+      - "5555:5432"
    environment:
      POSTGRES_DB: meta_insights
      POSTGRES_USER: meta_user
@@ -19,22 +19,22 @@ services:
      retries: 5
    restart: unless-stopped

-  # # Optional: Grafana for visualization
-  # grafana:
-  #   image: grafana/grafana:latest
-  #   container_name: meta_grafana
-  #   ports:
-  #     - "3000:3000"
-  #   environment:
-  #     GF_SECURITY_ADMIN_USER: admin
-  #     GF_SECURITY_ADMIN_PASSWORD: admin
-  #     GF_INSTALL_PLUGINS: grafana-clock-panel
-  #   volumes:
-  #     - grafana_data:/var/lib/grafana
-  #   depends_on:
-  #     timescaledb:
-  #       condition: service_healthy
-  #   restart: unless-stopped
+  # Optional: Grafana for visualization
+  grafana:
+    image: grafana/grafana:latest
+    container_name: meta_grafana
+    ports:
+      - "3555:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_INSTALL_PLUGINS: grafana-clock-panel
+    volumes:
+      - grafana_data:/var/lib/grafana
+    depends_on:
+      timescaledb:
+        condition: service_healthy
+    restart: unless-stopped

 volumes:
  timescale_data:
--- a/src/meta_api_grabber/database.py
+++ b/src/meta_api_grabber/database.py
@@ -235,8 +235,9 @@ class TimescaleDBClient:
        query = """
            INSERT INTO account_insights (
                time, account_id, impressions, clicks, spend, reach, frequency,
-                ctr, cpc, cpm, cpp, actions, cost_per_action_type, date_preset, fetched_at
-            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, NOW())
+                ctr, cpc, cpm, cpp, actions, cost_per_action_type, date_preset,
+                date_start, date_stop, fetched_at
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, NOW())
            ON CONFLICT (time, account_id)
            DO UPDATE SET
                impressions = EXCLUDED.impressions,
@@ -250,6 +251,8 @@ class TimescaleDBClient:
                cpp = EXCLUDED.cpp,
                actions = EXCLUDED.actions,
                cost_per_action_type = EXCLUDED.cost_per_action_type,
+                date_start = EXCLUDED.date_start,
+                date_stop = EXCLUDED.date_stop,
                fetched_at = NOW()
        """

@@ -264,6 +267,15 @@ class TimescaleDBClient:
        cpm = float(data.get("cpm", 0)) if data.get("cpm") else None
        cpp = float(data.get("cpp", 0)) if data.get("cpp") else None

+        # Extract date range from Meta API response and convert to date objects
+        from datetime import date as Date
+        date_start = None
+        date_stop = None
+        if data.get("date_start"):
+            date_start = Date.fromisoformat(data["date_start"])  # "2025-10-21" -> date object
+        if data.get("date_stop"):
+            date_stop = Date.fromisoformat(data["date_stop"])
+
        # Store actions as JSONB
        import json
        actions = json.dumps(data.get("actions", [])) if data.get("actions") else None
@@ -273,7 +285,8 @@ class TimescaleDBClient:
            await conn.execute(
                query,
                time, account_id, impressions, clicks, spend, reach, frequency,
-                ctr, cpc, cpm, cpp, actions, cost_per_action, date_preset
+                ctr, cpc, cpm, cpp, actions, cost_per_action, date_preset,
+                date_start, date_stop
            )

    async def insert_campaign_insights(
@@ -297,8 +310,8 @@ class TimescaleDBClient:
        query = """
            INSERT INTO campaign_insights (
                time, campaign_id, account_id, impressions, clicks, spend, reach,
-                ctr, cpc, cpm, actions, date_preset, fetched_at
-            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NOW())
+                ctr, cpc, cpm, actions, date_preset, date_start, date_stop, fetched_at
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, NOW())
            ON CONFLICT (time, campaign_id)
            DO UPDATE SET
                impressions = EXCLUDED.impressions,
@@ -309,6 +322,8 @@ class TimescaleDBClient:
                cpc = EXCLUDED.cpc,
                cpm = EXCLUDED.cpm,
                actions = EXCLUDED.actions,
+                date_start = EXCLUDED.date_start,
+                date_stop = EXCLUDED.date_stop,
                fetched_at = NOW()
        """

@@ -320,6 +335,15 @@ class TimescaleDBClient:
        cpc = float(data.get("cpc", 0)) if data.get("cpc") else None
        cpm = float(data.get("cpm", 0)) if data.get("cpm") else None

+        # Extract date range from Meta API response and convert to date objects
+        from datetime import date as Date
+        date_start = None
+        date_stop = None
+        if data.get("date_start"):
+            date_start = Date.fromisoformat(data["date_start"])
+        if data.get("date_stop"):
+            date_stop = Date.fromisoformat(data["date_stop"])
+
        import json
        actions = json.dumps(data.get("actions", [])) if data.get("actions") else None

@@ -327,7 +351,7 @@ class TimescaleDBClient:
            await conn.execute(
                query,
                time, campaign_id, account_id, impressions, clicks, spend, reach,
-                ctr, cpc, cpm, actions, date_preset
+                ctr, cpc, cpm, actions, date_preset, date_start, date_stop
            )

    async def insert_adset_insights(
@@ -353,8 +377,8 @@ class TimescaleDBClient:
        query = """
            INSERT INTO adset_insights (
                time, adset_id, campaign_id, account_id, impressions, clicks, spend, reach,
-                ctr, cpc, cpm, actions, date_preset, fetched_at
-            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW())
+                ctr, cpc, cpm, actions, date_preset, date_start, date_stop, fetched_at
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW())
            ON CONFLICT (time, adset_id)
            DO UPDATE SET
                impressions = EXCLUDED.impressions,
@@ -365,6 +389,8 @@ class TimescaleDBClient:
                cpc = EXCLUDED.cpc,
                cpm = EXCLUDED.cpm,
                actions = EXCLUDED.actions,
+                date_start = EXCLUDED.date_start,
+                date_stop = EXCLUDED.date_stop,
                fetched_at = NOW()
        """

@@ -376,6 +402,15 @@ class TimescaleDBClient:
        cpc = float(data.get("cpc", 0)) if data.get("cpc") else None
        cpm = float(data.get("cpm", 0)) if data.get("cpm") else None

+        # Extract date range from Meta API response and convert to date objects
+        from datetime import date as Date
+        date_start = None
+        date_stop = None
+        if data.get("date_start"):
+            date_start = Date.fromisoformat(data["date_start"])
+        if data.get("date_stop"):
+            date_stop = Date.fromisoformat(data["date_stop"])
+
        import json
        actions = json.dumps(data.get("actions", [])) if data.get("actions") else None

@@ -383,7 +418,7 @@ class TimescaleDBClient:
            await conn.execute(
                query,
                time, adset_id, campaign_id, account_id, impressions, clicks, spend, reach,
-                ctr, cpc, cpm, actions, date_preset
+                ctr, cpc, cpm, actions, date_preset, date_start, date_stop
            )

    # ========================================================================
--- a/src/meta_api_grabber/db_schema.sql
+++ b/src/meta_api_grabber/db_schema.sql
@@ -4,6 +4,20 @@
 -- Enable TimescaleDB extension (run as superuser)
 -- CREATE EXTENSION IF NOT EXISTS timescaledb;

+-- ============================================================================
+-- MIGRATIONS (Add new columns to existing tables)
+-- ============================================================================
+
+-- Add date_start and date_stop columns (idempotent - safe to run multiple times)
+ALTER TABLE IF EXISTS account_insights ADD COLUMN IF NOT EXISTS date_start DATE;
+ALTER TABLE IF EXISTS account_insights ADD COLUMN IF NOT EXISTS date_stop DATE;
+
+ALTER TABLE IF EXISTS campaign_insights ADD COLUMN IF NOT EXISTS date_start DATE;
+ALTER TABLE IF EXISTS campaign_insights ADD COLUMN IF NOT EXISTS date_stop DATE;
+
+ALTER TABLE IF EXISTS adset_insights ADD COLUMN IF NOT EXISTS date_start DATE;
+ALTER TABLE IF EXISTS adset_insights ADD COLUMN IF NOT EXISTS date_stop DATE;
+
 -- ============================================================================
 -- METADATA TABLES (Regular PostgreSQL tables for caching)
 -- ============================================================================
@@ -71,6 +85,8 @@ CREATE TABLE IF NOT EXISTS account_insights (

    -- Metadata
    date_preset VARCHAR(50),
+    date_start DATE,  -- Actual start date of the data range from Meta API
+    date_stop DATE,   -- Actual end date of the data range from Meta API
    fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),

    -- Composite primary key
@@ -110,6 +126,8 @@ CREATE TABLE IF NOT EXISTS campaign_insights (

    -- Metadata
    date_preset VARCHAR(50),
+    date_start DATE,  -- Actual start date of the data range from Meta API
+    date_stop DATE,   -- Actual end date of the data range from Meta API
    fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),

    PRIMARY KEY (time, campaign_id)
@@ -150,6 +168,8 @@ CREATE TABLE IF NOT EXISTS adset_insights (

    -- Metadata
    date_preset VARCHAR(50),
+    date_start DATE,  -- Actual start date of the data range from Meta API
+    date_stop DATE,   -- Actual end date of the data range from Meta API
    fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),

    PRIMARY KEY (time, adset_id)
--- a/src/meta_api_grabber/test_date_fields.py
+++ b/src/meta_api_grabber/test_date_fields.py
@@ -0,0 +1,125 @@
+"""
+Test that date_start and date_stop are properly stored in the database.
+"""
+
+import asyncio
+import os
+from datetime import datetime, timezone
+from dotenv import load_dotenv
+from facebook_business.adobjects.adaccount import AdAccount
+from facebook_business.adobjects.adsinsights import AdsInsights
+from facebook_business.api import FacebookAdsApi
+
+from meta_api_grabber.database import TimescaleDBClient
+
+
+async def test_date_fields():
+    """Test that date fields are stored correctly."""
+    load_dotenv()
+
+    access_token = os.getenv("META_ACCESS_TOKEN")
+    app_secret = os.getenv("META_APP_SECRET")
+    app_id = os.getenv("META_APP_ID")
+    account_id = "act_238334370765317"
+
+    if not all([access_token, app_secret, app_id]):
+        print("❌ Missing required environment variables")
+        return 1
+
+    # Initialize Facebook Ads API
+    FacebookAdsApi.init(
+        app_id=app_id,
+        app_secret=app_secret,
+        access_token=access_token,
+    )
+
+    print("="*70)
+    print("TESTING DATE_START AND DATE_STOP STORAGE")
+    print("="*70)
+    print()
+
+    # Connect to database
+    print("Connecting to database...")
+    db = TimescaleDBClient()
+    await db.connect()
+
+    try:
+        print("Fetching insights for 'today'...")
+        ad_account = AdAccount(account_id)
+
+        fields = [
+            AdsInsights.Field.impressions,
+            AdsInsights.Field.clicks,
+            AdsInsights.Field.spend,
+            AdsInsights.Field.date_start,
+            AdsInsights.Field.date_stop,
+        ]
+
+        params = {
+            "date_preset": "today",
+            "level": "account",
+        }
+
+        insights = ad_account.get_insights(fields=fields, params=params)
+
+        # Store in database
+        timestamp = datetime.now(timezone.utc)
+        for insight in insights:
+            insight_dict = dict(insight)
+            print(f"\nAPI Response:")
+            print(f"  date_start: {insight_dict.get('date_start')}")
+            print(f"  date_stop: {insight_dict.get('date_stop')}")
+            print(f"  impressions: {insight_dict.get('impressions')}")
+
+            await db.insert_account_insights(
+                time=timestamp,
+                account_id=account_id,
+                data=insight_dict,
+                date_preset="today",
+            )
+            print("\n✓ Stored in database")
+
+        # Verify from database
+        print("\nQuerying database...")
+        async with db.pool.acquire() as conn:
+            row = await conn.fetchrow("""
+                SELECT date_start, date_stop, date_preset, impressions, spend
+                FROM account_insights
+                WHERE account_id = $1
+                ORDER BY time DESC
+                LIMIT 1
+            """, account_id)
+
+            if row:
+                print("\n✓ Retrieved from database:")
+                print(f"  date_start: {row['date_start']}")
+                print(f"  date_stop: {row['date_stop']}")
+                print(f"  date_preset: {row['date_preset']}")
+                print(f"  impressions: {row['impressions']}")
+                print(f"  spend: {row['spend']}")
+
+                print("\n" + "="*70)
+                print("✓ TEST PASSED - Date fields are stored correctly!")
+                print("="*70)
+                print("\nYou can now query historical data by date_stop:")
+                print("  - For clean daily trends, use: GROUP BY date_stop")
+                print("  - For latest value per day, use: ORDER BY time DESC with date_stop")
+                print()
+            else:
+                print("\n❌ No data found in database")
+                return 1
+
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+    finally:
+        await db.close()
+
+    return 0
+
+
+if __name__ == "__main__":
+    exit(asyncio.run(test_date_fields()))
--- a/src/meta_api_grabber/test_today_preset.py
+++ b/src/meta_api_grabber/test_today_preset.py
@@ -0,0 +1,87 @@
+"""
+Test script to check what date_start and date_stop look like for "today" preset.
+"""
+
+import os
+import json
+from dotenv import load_dotenv
+from facebook_business.adobjects.adaccount import AdAccount
+from facebook_business.adobjects.adsinsights import AdsInsights
+from facebook_business.api import FacebookAdsApi
+
+
+def test_today_preset():
+    """Test the 'today' date preset to see date_start and date_stop values."""
+    load_dotenv()
+
+    access_token = os.getenv("META_ACCESS_TOKEN")
+    app_secret = os.getenv("META_APP_SECRET")
+    app_id = os.getenv("META_APP_ID")
+
+    if not all([access_token, app_secret, app_id]):
+        print("❌ Missing required environment variables")
+        return 1
+
+    # Initialize Facebook Ads API
+    FacebookAdsApi.init(
+        app_id=app_id,
+        app_secret=app_secret,
+        access_token=access_token,
+    )
+
+    # Use the first account we know exists
+    account_id = "act_238334370765317"
+    ad_account = AdAccount(account_id)
+
+    print("="*70)
+    print("TESTING DATE_PRESET='TODAY'")
+    print("="*70)
+    print(f"Account: {account_id}")
+    print()
+
+    # Request with date_preset="today"
+    fields = [
+        AdsInsights.Field.impressions,
+        AdsInsights.Field.clicks,
+        AdsInsights.Field.spend,
+        AdsInsights.Field.date_start,
+        AdsInsights.Field.date_stop,
+    ]
+
+    params = {
+        "date_preset": "today",
+        "level": "account",
+    }
+
+    print("Making API request with date_preset='today'...")
+    insights = ad_account.get_insights(fields=fields, params=params)
+
+    print("\nResponse:")
+    print("-"*70)
+    for insight in insights:
+        insight_dict = dict(insight)
+        print(json.dumps(insight_dict, indent=2))
+        print()
+        print("Key observations:")
+        print(f"  date_start: {insight_dict.get('date_start')}")
+        print(f"  date_stop:  {insight_dict.get('date_stop')}")
+        print(f"  Are they the same? {insight_dict.get('date_start') == insight_dict.get('date_stop')}")
+        print(f"  impressions: {insight_dict.get('impressions')}")
+        print(f"  spend: {insight_dict.get('spend')}")
+
+    print()
+    print("="*70)
+    print("CONCLUSION")
+    print("="*70)
+    print("For 'today' preset:")
+    print("  - date_start and date_stop should both be today's date")
+    print("  - Metrics are cumulative from midnight to now")
+    print("  - Multiple collections during the day will have same dates")
+    print("    but increasing metric values")
+    print()
+
+    return 0
+
+
+if __name__ == "__main__":
+    exit(test_today_preset())