diff --git a/docker-compose.yml b/docker-compose.yml index 1ab4652..57c3360 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: timescale/timescaledb:latest-pg16 container_name: meta_timescaledb ports: - - "5432:5432" + - "5555:5432" environment: POSTGRES_DB: meta_insights POSTGRES_USER: meta_user @@ -19,22 +19,22 @@ services: retries: 5 restart: unless-stopped - # # Optional: Grafana for visualization - # grafana: - # image: grafana/grafana:latest - # container_name: meta_grafana - # ports: - # - "3000:3000" - # environment: - # GF_SECURITY_ADMIN_USER: admin - # GF_SECURITY_ADMIN_PASSWORD: admin - # GF_INSTALL_PLUGINS: grafana-clock-panel - # volumes: - # - grafana_data:/var/lib/grafana - # depends_on: - # timescaledb: - # condition: service_healthy - # restart: unless-stopped + # Optional: Grafana for visualization + grafana: + image: grafana/grafana:latest + container_name: meta_grafana + ports: + - "3555:3000" + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_INSTALL_PLUGINS: grafana-clock-panel + volumes: + - grafana_data:/var/lib/grafana + depends_on: + timescaledb: + condition: service_healthy + restart: unless-stopped volumes: timescale_data: diff --git a/src/meta_api_grabber/database.py b/src/meta_api_grabber/database.py index 5f2dc71..e18a174 100644 --- a/src/meta_api_grabber/database.py +++ b/src/meta_api_grabber/database.py @@ -235,8 +235,9 @@ class TimescaleDBClient: query = """ INSERT INTO account_insights ( time, account_id, impressions, clicks, spend, reach, frequency, - ctr, cpc, cpm, cpp, actions, cost_per_action_type, date_preset, fetched_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, NOW()) + ctr, cpc, cpm, cpp, actions, cost_per_action_type, date_preset, + date_start, date_stop, fetched_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, NOW()) ON CONFLICT (time, account_id) DO UPDATE SET impressions = EXCLUDED.impressions, @@ -250,6 +251,8 @@ class TimescaleDBClient: cpp = EXCLUDED.cpp, actions = EXCLUDED.actions, cost_per_action_type = EXCLUDED.cost_per_action_type, + date_start = EXCLUDED.date_start, + date_stop = EXCLUDED.date_stop, fetched_at = NOW() """ @@ -264,6 +267,15 @@ class TimescaleDBClient: cpm = float(data.get("cpm", 0)) if data.get("cpm") else None cpp = float(data.get("cpp", 0)) if data.get("cpp") else None + # Extract date range from Meta API response and convert to date objects + from datetime import date as Date + date_start = None + date_stop = None + if data.get("date_start"): + date_start = Date.fromisoformat(data["date_start"]) # "2025-10-21" -> date object + if data.get("date_stop"): + date_stop = Date.fromisoformat(data["date_stop"]) + # Store actions as JSONB import json actions = json.dumps(data.get("actions", [])) if data.get("actions") else None @@ -273,7 +285,8 @@ class TimescaleDBClient: await conn.execute( query, time, account_id, impressions, clicks, spend, reach, frequency, - ctr, cpc, cpm, cpp, actions, cost_per_action, date_preset + ctr, cpc, cpm, cpp, actions, cost_per_action, date_preset, + date_start, date_stop ) async def insert_campaign_insights( @@ -297,8 +310,8 @@ class TimescaleDBClient: query = """ INSERT INTO campaign_insights ( time, campaign_id, account_id, impressions, clicks, spend, reach, - ctr, cpc, cpm, actions, date_preset, fetched_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NOW()) + ctr, cpc, cpm, actions, date_preset, date_start, date_stop, fetched_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, NOW()) ON CONFLICT (time, campaign_id) DO UPDATE SET impressions = EXCLUDED.impressions, @@ -309,6 +322,8 @@ class TimescaleDBClient: cpc = EXCLUDED.cpc, cpm = EXCLUDED.cpm, actions = EXCLUDED.actions, + date_start = EXCLUDED.date_start, + date_stop = EXCLUDED.date_stop, fetched_at = NOW() """ @@ -320,6 +335,15 @@ class TimescaleDBClient: cpc = float(data.get("cpc", 0)) if data.get("cpc") else None cpm = float(data.get("cpm", 0)) if data.get("cpm") else None + # Extract date range from Meta API response and convert to date objects + from datetime import date as Date + date_start = None + date_stop = None + if data.get("date_start"): + date_start = Date.fromisoformat(data["date_start"]) + if data.get("date_stop"): + date_stop = Date.fromisoformat(data["date_stop"]) + import json actions = json.dumps(data.get("actions", [])) if data.get("actions") else None @@ -327,7 +351,7 @@ class TimescaleDBClient: await conn.execute( query, time, campaign_id, account_id, impressions, clicks, spend, reach, - ctr, cpc, cpm, actions, date_preset + ctr, cpc, cpm, actions, date_preset, date_start, date_stop ) async def insert_adset_insights( @@ -353,8 +377,8 @@ class TimescaleDBClient: query = """ INSERT INTO adset_insights ( time, adset_id, campaign_id, account_id, impressions, clicks, spend, reach, - ctr, cpc, cpm, actions, date_preset, fetched_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW()) + ctr, cpc, cpm, actions, date_preset, date_start, date_stop, fetched_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW()) ON CONFLICT (time, adset_id) DO UPDATE SET impressions = EXCLUDED.impressions, @@ -365,6 +389,8 @@ class TimescaleDBClient: cpc = EXCLUDED.cpc, cpm = EXCLUDED.cpm, actions = EXCLUDED.actions, + date_start = EXCLUDED.date_start, + date_stop = EXCLUDED.date_stop, fetched_at = NOW() """ @@ -376,6 +402,15 @@ class TimescaleDBClient: cpc = float(data.get("cpc", 0)) if data.get("cpc") else None cpm = float(data.get("cpm", 0)) if data.get("cpm") else None + # Extract date range from Meta API response and convert to date objects + from datetime import date as Date + date_start = None + date_stop = None + if data.get("date_start"): + date_start = Date.fromisoformat(data["date_start"]) + if data.get("date_stop"): + date_stop = Date.fromisoformat(data["date_stop"]) + import json actions = json.dumps(data.get("actions", [])) if data.get("actions") else None @@ -383,7 +418,7 @@ class TimescaleDBClient: await conn.execute( query, time, adset_id, campaign_id, account_id, impressions, clicks, spend, reach, - ctr, cpc, cpm, actions, date_preset + ctr, cpc, cpm, actions, date_preset, date_start, date_stop ) # ======================================================================== diff --git a/src/meta_api_grabber/db_schema.sql b/src/meta_api_grabber/db_schema.sql index be32dce..5d14cb2 100644 --- a/src/meta_api_grabber/db_schema.sql +++ b/src/meta_api_grabber/db_schema.sql @@ -4,6 +4,20 @@ -- Enable TimescaleDB extension (run as superuser) -- CREATE EXTENSION IF NOT EXISTS timescaledb; +-- ============================================================================ +-- MIGRATIONS (Add new columns to existing tables) +-- ============================================================================ + +-- Add date_start and date_stop columns (idempotent - safe to run multiple times) +ALTER TABLE IF EXISTS account_insights ADD COLUMN IF NOT EXISTS date_start DATE; +ALTER TABLE IF EXISTS account_insights ADD COLUMN IF NOT EXISTS date_stop DATE; + +ALTER TABLE IF EXISTS campaign_insights ADD COLUMN IF NOT EXISTS date_start DATE; +ALTER TABLE IF EXISTS campaign_insights ADD COLUMN IF NOT EXISTS date_stop DATE; + +ALTER TABLE IF EXISTS adset_insights ADD COLUMN IF NOT EXISTS date_start DATE; +ALTER TABLE IF EXISTS adset_insights ADD COLUMN IF NOT EXISTS date_stop DATE; + -- ============================================================================ -- METADATA TABLES (Regular PostgreSQL tables for caching) -- ============================================================================ @@ -71,6 +85,8 @@ CREATE TABLE IF NOT EXISTS account_insights ( -- Metadata date_preset VARCHAR(50), + date_start DATE, -- Actual start date of the data range from Meta API + date_stop DATE, -- Actual end date of the data range from Meta API fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -- Composite primary key @@ -110,6 +126,8 @@ CREATE TABLE IF NOT EXISTS campaign_insights ( -- Metadata date_preset VARCHAR(50), + date_start DATE, -- Actual start date of the data range from Meta API + date_stop DATE, -- Actual end date of the data range from Meta API fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (time, campaign_id) @@ -150,6 +168,8 @@ CREATE TABLE IF NOT EXISTS adset_insights ( -- Metadata date_preset VARCHAR(50), + date_start DATE, -- Actual start date of the data range from Meta API + date_stop DATE, -- Actual end date of the data range from Meta API fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (time, adset_id) diff --git a/src/meta_api_grabber/test_date_fields.py b/src/meta_api_grabber/test_date_fields.py new file mode 100644 index 0000000..d9064c6 --- /dev/null +++ b/src/meta_api_grabber/test_date_fields.py @@ -0,0 +1,125 @@ +""" +Test that date_start and date_stop are properly stored in the database. +""" + +import asyncio +import os +from datetime import datetime, timezone +from dotenv import load_dotenv +from facebook_business.adobjects.adaccount import AdAccount +from facebook_business.adobjects.adsinsights import AdsInsights +from facebook_business.api import FacebookAdsApi + +from meta_api_grabber.database import TimescaleDBClient + + +async def test_date_fields(): + """Test that date fields are stored correctly.""" + load_dotenv() + + access_token = os.getenv("META_ACCESS_TOKEN") + app_secret = os.getenv("META_APP_SECRET") + app_id = os.getenv("META_APP_ID") + account_id = "act_238334370765317" + + if not all([access_token, app_secret, app_id]): + print("❌ Missing required environment variables") + return 1 + + # Initialize Facebook Ads API + FacebookAdsApi.init( + app_id=app_id, + app_secret=app_secret, + access_token=access_token, + ) + + print("="*70) + print("TESTING DATE_START AND DATE_STOP STORAGE") + print("="*70) + print() + + # Connect to database + print("Connecting to database...") + db = TimescaleDBClient() + await db.connect() + + try: + print("Fetching insights for 'today'...") + ad_account = AdAccount(account_id) + + fields = [ + AdsInsights.Field.impressions, + AdsInsights.Field.clicks, + AdsInsights.Field.spend, + AdsInsights.Field.date_start, + AdsInsights.Field.date_stop, + ] + + params = { + "date_preset": "today", + "level": "account", + } + + insights = ad_account.get_insights(fields=fields, params=params) + + # Store in database + timestamp = datetime.now(timezone.utc) + for insight in insights: + insight_dict = dict(insight) + print(f"\nAPI Response:") + print(f" date_start: {insight_dict.get('date_start')}") + print(f" date_stop: {insight_dict.get('date_stop')}") + print(f" impressions: {insight_dict.get('impressions')}") + + await db.insert_account_insights( + time=timestamp, + account_id=account_id, + data=insight_dict, + date_preset="today", + ) + print("\n✓ Stored in database") + + # Verify from database + print("\nQuerying database...") + async with db.pool.acquire() as conn: + row = await conn.fetchrow(""" + SELECT date_start, date_stop, date_preset, impressions, spend + FROM account_insights + WHERE account_id = $1 + ORDER BY time DESC + LIMIT 1 + """, account_id) + + if row: + print("\n✓ Retrieved from database:") + print(f" date_start: {row['date_start']}") + print(f" date_stop: {row['date_stop']}") + print(f" date_preset: {row['date_preset']}") + print(f" impressions: {row['impressions']}") + print(f" spend: {row['spend']}") + + print("\n" + "="*70) + print("✓ TEST PASSED - Date fields are stored correctly!") + print("="*70) + print("\nYou can now query historical data by date_stop:") + print(" - For clean daily trends, use: GROUP BY date_stop") + print(" - For latest value per day, use: ORDER BY time DESC with date_stop") + print() + else: + print("\n❌ No data found in database") + return 1 + + except Exception as e: + print(f"\n❌ Error: {e}") + import traceback + traceback.print_exc() + return 1 + + finally: + await db.close() + + return 0 + + +if __name__ == "__main__": + exit(asyncio.run(test_date_fields())) diff --git a/src/meta_api_grabber/test_today_preset.py b/src/meta_api_grabber/test_today_preset.py new file mode 100644 index 0000000..99b285f --- /dev/null +++ b/src/meta_api_grabber/test_today_preset.py @@ -0,0 +1,87 @@ +""" +Test script to check what date_start and date_stop look like for "today" preset. +""" + +import os +import json +from dotenv import load_dotenv +from facebook_business.adobjects.adaccount import AdAccount +from facebook_business.adobjects.adsinsights import AdsInsights +from facebook_business.api import FacebookAdsApi + + +def test_today_preset(): + """Test the 'today' date preset to see date_start and date_stop values.""" + load_dotenv() + + access_token = os.getenv("META_ACCESS_TOKEN") + app_secret = os.getenv("META_APP_SECRET") + app_id = os.getenv("META_APP_ID") + + if not all([access_token, app_secret, app_id]): + print("❌ Missing required environment variables") + return 1 + + # Initialize Facebook Ads API + FacebookAdsApi.init( + app_id=app_id, + app_secret=app_secret, + access_token=access_token, + ) + + # Use the first account we know exists + account_id = "act_238334370765317" + ad_account = AdAccount(account_id) + + print("="*70) + print("TESTING DATE_PRESET='TODAY'") + print("="*70) + print(f"Account: {account_id}") + print() + + # Request with date_preset="today" + fields = [ + AdsInsights.Field.impressions, + AdsInsights.Field.clicks, + AdsInsights.Field.spend, + AdsInsights.Field.date_start, + AdsInsights.Field.date_stop, + ] + + params = { + "date_preset": "today", + "level": "account", + } + + print("Making API request with date_preset='today'...") + insights = ad_account.get_insights(fields=fields, params=params) + + print("\nResponse:") + print("-"*70) + for insight in insights: + insight_dict = dict(insight) + print(json.dumps(insight_dict, indent=2)) + print() + print("Key observations:") + print(f" date_start: {insight_dict.get('date_start')}") + print(f" date_stop: {insight_dict.get('date_stop')}") + print(f" Are they the same? {insight_dict.get('date_start') == insight_dict.get('date_stop')}") + print(f" impressions: {insight_dict.get('impressions')}") + print(f" spend: {insight_dict.get('spend')}") + + print() + print("="*70) + print("CONCLUSION") + print("="*70) + print("For 'today' preset:") + print(" - date_start and date_stop should both be today's date") + print(" - Metrics are cumulative from midnight to now") + print(" - Multiple collections during the day will have same dates") + print(" but increasing metric values") + print() + + return 0 + + +if __name__ == "__main__": + exit(test_today_preset())