Updated view queries

CLAUDE.md

@@ -1,2 +1,4 @@
 Uv managed python project that grabs data from the meta api and saves it in a timescaledb database. 
 
+Always use uv run to execute python related stuff
+

src/meta_api_grabber/database.py

@@ -310,7 +310,6 @@ class TimescaleDBClient:
         account_id: str,
         data: Dict[str, Any],
         date_preset: str = "today",
-        cache_metadata: bool = True,
     ):
         """
         Insert campaign-level insights data.
@@ -321,10 +320,9 @@ class TimescaleDBClient:
             account_id: Ad account ID
             data: Insights data dictionary from Meta API
             date_preset: Date preset used
-            cache_metadata: If True, automatically cache campaign metadata from insights data
         """
-        # Cache campaign metadata if requested and available in the insights data
-        if cache_metadata and data.get("campaign_name"):
+        # Auto-cache campaign metadata if available in the insights data
+        if data.get("campaign_name"):
             await self.upsert_campaign(
                 campaign_id=campaign_id,
                 account_id=account_id,
@@ -389,7 +387,6 @@ class TimescaleDBClient:
         account_id: str,
         data: Dict[str, Any],
         date_preset: str = "today",
-        cache_metadata: bool = True,
     ):
         """
         Insert ad set level insights data.
@@ -401,21 +398,18 @@ class TimescaleDBClient:
             account_id: Ad account ID
             data: Insights data dictionary from Meta API
             date_preset: Date preset used
-            cache_metadata: If True, automatically cache adset/campaign metadata from insights data
         """
-        # Cache metadata if requested and available in the insights data
-        if cache_metadata:
-            # Cache adset metadata if available
-            # Note: Campaign should already exist from cache_campaigns_metadata or grab_campaign_insights
-            # If it doesn't exist, the foreign key constraint will fail with a clear error
-            # This is intentional - we should never silently create campaigns with 'Unknown' names
-            if data.get("adset_name"):
-                await self.upsert_adset(
-                    adset_id=adset_id,
-                    campaign_id=campaign_id,
-                    adset_name=data["adset_name"],
-                    status=None,  # Not available in insights response
-                )
+        # Auto-cache adset metadata if available in the insights data
+        # Note: Campaign should already exist from cache_campaigns_metadata or grab_campaign_insights
+        # If it doesn't exist, the foreign key constraint will fail with a clear error
+        # This is intentional - we should never silently create campaigns with 'Unknown' names
+        if data.get("adset_name"):
+            await self.upsert_adset(
+                adset_id=adset_id,
+                campaign_id=campaign_id,
+                adset_name=data["adset_name"],
+                status=None,  # Not available in insights response
+            )
 
         query = """
             INSERT INTO adset_insights (

src/meta_api_grabber/db_schema.sql

@@ -20,7 +20,6 @@ ALTER TABLE IF EXISTS campaign_insights ADD COLUMN IF NOT EXISTS cost_per_action
 
 ALTER TABLE IF EXISTS adset_insights ADD COLUMN IF NOT EXISTS date_start DATE;
 ALTER TABLE IF EXISTS adset_insights ADD COLUMN IF NOT EXISTS date_stop DATE;
-ALTER TABLE IF EXISTS adset_insights ADD COLUMN IF NOT EXISTS account_currency VARCHAR(3);
 
 ALTER TABLE IF EXISTS campaign_insights_by_country ADD COLUMN IF NOT EXISTS frequency NUMERIC(10, 4);
 ALTER TABLE IF EXISTS campaign_insights_by_country ADD COLUMN IF NOT EXISTS cpp NUMERIC(10, 4);

src/meta_api_grabber/scheduled_grabber.py

@@ -438,7 +438,6 @@ class ScheduledInsightsGrabber:
         end_date: Optional[date] = None,
         breakdowns: Optional[list] = None,
         limit: Optional[int] = None,
-        cache_metadata: bool = False,
         required_fields: Optional[dict] = None,
         extra_data_processor=None,
     ) -> tuple[int, Optional[date]]:
@@ -455,7 +454,6 @@ class ScheduledInsightsGrabber:
             end_date: End date for custom date range (optional)
             breakdowns: List of breakdown fields (optional)
             limit: Maximum number of results (optional)
-            cache_metadata: Whether to cache metadata (for campaign/adset levels)
             required_fields: Dict of field_name -> label for validation before insert
             extra_data_processor: Optional callable to process/add extra data to insight_dict
 
@@ -529,14 +527,27 @@ class ScheduledInsightsGrabber:
             # Compute appropriate timestamp based on date_start and account timezone
             timestamp = self._compute_timestamp(date_start_str, account_timezone)
 
-            # Call the appropriate database insert function
-            await db_insert_func(
-                time=timestamp,
-                account_id=account_id,
-                data=insight_dict,
-                date_preset=date_preset_for_db,
-                cache_metadata=cache_metadata,
-            )
+            # Build kwargs for insert function based on level
+            kwargs = {
+                "time": timestamp,
+                "account_id": account_id,
+                "data": insight_dict,
+                "date_preset": date_preset_for_db,
+            }
+
+            # Add level-specific parameters
+            if level == "campaign":
+                kwargs["campaign_id"] = insight_dict.get("campaign_id")
+            elif level == "adset":
+                kwargs["adset_id"] = insight_dict.get("adset_id")
+                kwargs["campaign_id"] = insight_dict.get("campaign_id")
+
+            # Add country for breakdown queries
+            if "country" in insight_dict:
+                kwargs["country"] = insight_dict.get("country")
+
+            # Call the appropriate database insert function with level-specific parameters
+            await db_insert_func(**kwargs)
             count += 1
 
         return count, date_start_value
@@ -602,7 +613,6 @@ class ScheduledInsightsGrabber:
             db_insert_func=self.db.insert_campaign_insights,
             date_preset=date_preset,
             limit=limit,
-            cache_metadata=True,
             required_fields={"campaign_id": "campaign_id"},
         )
 
@@ -640,7 +650,6 @@ class ScheduledInsightsGrabber:
             db_insert_func=self.db.insert_adset_insights,
             date_preset=date_preset,
             limit=limit,
-            cache_metadata=True,
             required_fields={"adset_id": "adset_id", "campaign_id": "campaign_id"},
         )
 
@@ -759,7 +768,6 @@ class ScheduledInsightsGrabber:
             start_date=start_date,
             end_date=end_date,
             limit=limit,
-            cache_metadata=True,
             required_fields={"campaign_id": "campaign_id"},
         )
 
@@ -808,7 +816,6 @@ class ScheduledInsightsGrabber:
             start_date=start_date,
             end_date=end_date,
             limit=limit,
-            cache_metadata=True,
             required_fields={"adset_id": "adset_id", "campaign_id": "campaign_id"},
         )
 

src/meta_api_grabber/view_manager.py

@@ -93,6 +93,8 @@ class ViewManager:
                 "account_insights_flattened",
                 "campaign_insights_flattened",
                 "campaign_insights_by_country_flattened",
+                #"campaign_insights_by_device_flattened",
+                #"campaign_insights_by_gender_flattened",
             ]
 
         async with self.pool.acquire() as conn:

src/meta_api_grabber/views/account_insights_by_device.sql

@@ -0,0 +1,20 @@
+--- account insights by gender
+
+
+DROP VIEW IF EXISTS account_insights_by_device CASCADE;
+
+CREATE VIEW account_insights_by_device AS
+SELECT 
+    time, 
+    account_id, 
+    device_platform, 
+    SUM(impressions) AS impressions, 
+    SUM(clicks) AS clicks,
+    SUM(spend) AS spend, 
+    SUM(link_click) AS link_click, 
+    SUM(landing_page_view) AS landing_page_view,
+    SUM(lead) AS lead
+FROM campaign_insights_by_device_flattened
+GROUP BY time, account_id, device_platform;
+
+

src/meta_api_grabber/views/account_insights_by_gender.sql

@@ -0,0 +1,53 @@
+
+
+
+DROP VIEW IF EXISTS account_insights_by_gender CASCADE;
+
+CREATE VIEW account_insights_by_gender AS
+SELECT 
+    time, 
+    account_id, 
+    gender, 
+    SUM(impressions) AS impressions, 
+    SUM(clicks) AS clicks,
+    SUM(spend) AS spend, 
+    SUM(link_click) AS link_click, 
+    SUM(landing_page_view) AS landing_page_view,
+    SUM(lead) AS lead
+FROM campaign_insights_by_gender
+GROUP BY time, account_id, gender;
+
+
+DROP VIEW IF EXISTS account_insights_by_age CASCADE;
+
+CREATE VIEW account_insights_by_age AS
+SELECT 
+    time, 
+    account_id, 
+    age, 
+    SUM(impressions) AS impressions, 
+    SUM(clicks) AS clicks,
+    SUM(spend) AS spend, 
+    SUM(link_click) AS link_click, 
+    SUM(landing_page_view) AS landing_page_view,
+    SUM(lead) AS lead
+FROM campaign_insights_by_age
+GROUP BY time, account_id, age;
+
+DROP VIEW IF EXISTS account_insights_by_gender_and_age CASCADE;
+
+CREATE VIEW account_insights_by_gender_and_age AS
+SELECT 
+    time, 
+    account_id,
+    gender, 
+    age, 
+    SUM(impressions) AS impressions, 
+    SUM(clicks) AS clicks,
+    SUM(spend) AS spend, 
+    SUM(link_click) AS link_click, 
+    SUM(landing_page_view) AS landing_page_view,
+    SUM(lead) AS lead
+FROM campaign_insights_by_gender_and_age
+GROUP BY time, account_id, age, gender;
+

src/meta_api_grabber/views/account_insights_google.sql

@@ -0,0 +1,55 @@
+DROP VIEW IF EXISTS g_account_insights CASCADE;
+CREATE VIEW g_account_insights AS
+SELECT 
+    time,
+    account_id,
+    clicks,
+    impressions,
+    interactions,
+    cost_micros,
+    cost_micros / 1000000.0 as cost,
+    leads,
+    engagements,
+    customer_currency_code,
+    account_name,
+    
+    -- CTR (Click-Through Rate)
+    (clicks::numeric / impressions_nz) * 100 as ctr,
+    
+    -- CPM (Cost Per Mille) in micros and standard units
+    (cost_micros::numeric / impressions_nz) * 1000 as cpm_micros,
+    (cost_micros::numeric / impressions_nz) * 1000 / 1000000.0 as cpm,
+    
+    -- CPC (Cost Per Click) in micros and standard units
+    cost_micros::numeric / clicks_nz as cpc_micros,
+    cost_micros::numeric / clicks_nz / 1000000.0 as cpc,
+    
+    -- CPL (Cost Per Lead) in micros and standard units
+    cost_micros::numeric / leads_nz as cpl_micros,
+    cost_micros::numeric / leads_nz / 1000000.0 as cpl,
+    
+    -- Conversion Rate
+    (leads::numeric / clicks_nz) * 100 as conversion_rate,
+    
+    -- Engagement Rate
+    (engagements::numeric / impressions_nz) * 100 as engagement_rate
+    
+FROM (
+    SELECT 
+        segments_date as time, 
+        customer_id as account_id, 
+        sum(metrics_clicks) as clicks, 
+        sum(metrics_impressions) as impressions, 
+        sum(metrics_interactions) as interactions, 
+        sum(metrics_cost_micros) as cost_micros,
+        sum(metrics_conversions) as leads, 
+        sum(metrics_engagements) as engagements,
+        customer_currency_code,
+        customer_descriptive_name as account_name,
+        -- Null-safe denominators
+        NULLIF(sum(metrics_clicks), 0) as clicks_nz,
+        NULLIF(sum(metrics_impressions), 0) as impressions_nz,
+        NULLIF(sum(metrics_conversions), 0) as leads_nz
+    FROM google.account_performance_report
+    GROUP BY account_id, time, customer_currency_code, account_name
+) base;

src/meta_api_grabber/views/campaign_insights.sql

@@ -3,10 +3,9 @@
 DROP MATERIALIZED VIEW IF EXISTS campaign_insights_flattened CASCADE;
 
 CREATE MATERIALIZED VIEW campaign_insights_flattened AS
-SELECT 
-    time,
-    account_id,
-	campaign_id,
+ SELECT date_start AS "time",
+    concat('act_', account_id) AS account_id,
+    campaign_id,
     impressions,
     clicks,
     spend,
@@ -14,22 +13,18 @@ SELECT
     ctr,
     cpc,
     cpm,
-    date_preset,
     date_start,
     date_stop,
-    fetched_at,
-    (SELECT (value->>'value')::numeric 
-     FROM jsonb_array_elements(actions) 
-     WHERE value->>'action_type' = 'link_click') AS link_click,
-    (SELECT (value->>'value')::numeric 
-     FROM jsonb_array_elements(actions) 
-     WHERE value->>'action_type' = 'landing_page_view') AS landing_page_view,
-    (SELECT (value->>'value')::numeric 
-     FROM jsonb_array_elements(actions) 
-     WHERE value->>'action_type' = 'lead') AS lead
-
-
-FROM campaign_insights;
+    ( SELECT (jsonb_array_elements.value ->> 'value'::text)::numeric AS "numeric"
+           FROM jsonb_array_elements(customcampaign_insights.actions) jsonb_array_elements(value)
+          WHERE (jsonb_array_elements.value ->> 'action_type'::text) = 'link_click'::text) AS link_click,
+    ( SELECT (jsonb_array_elements.value ->> 'value'::text)::numeric AS "numeric"
+           FROM jsonb_array_elements(customcampaign_insights.actions) jsonb_array_elements(value)
+          WHERE (jsonb_array_elements.value ->> 'action_type'::text) = 'landing_page_view'::text) AS landing_page_view,
+    ( SELECT (jsonb_array_elements.value ->> 'value'::text)::numeric AS "numeric"
+           FROM jsonb_array_elements(customcampaign_insights.actions) jsonb_array_elements(value)
+          WHERE (jsonb_array_elements.value ->> 'action_type'::text) = 'lead'::text) AS lead
+   FROM meta.customcampaign_insights;
 
 CREATE INDEX idx_campaign_insights_flat_date ON campaign_insights_flattened(date_start, date_stop);
 

src/meta_api_grabber/views/campaign_insights_by_country.sql

@@ -3,22 +3,20 @@
 DROP MATERIALIZED VIEW IF EXISTS campaign_insights_by_country_flattened CASCADE;
 
 CREATE MATERIALIZED VIEW campaign_insights_by_country_flattened AS
-SELECT
-    time,
-    account_id,
+ SELECT date_start AS "time",
+    concat('act_', account_id) AS account_id,
     campaign_id,
     country,
     impressions,
     clicks,
     spend,
     reach,
+    frequency,
     ctr,
     cpc,
     cpm,
-    date_preset,
     date_start,
     date_stop,
-    fetched_at,
     (SELECT (value->>'value')::numeric
      FROM jsonb_array_elements(actions)
      WHERE value->>'action_type' = 'link_click') AS link_click,
@@ -29,7 +27,7 @@ SELECT
      FROM jsonb_array_elements(actions)
      WHERE value->>'action_type' = 'lead') AS lead
 
-FROM campaign_insights_by_country;
+   FROM meta.custom_campaign_country;
 
 CREATE INDEX idx_campaign_insights_by_country_flat_date ON campaign_insights_by_country_flattened(date_start, date_stop);
 

src/meta_api_grabber/views/campaign_insights_by_device.sql

@@ -0,0 +1,32 @@
+--- campaign insights by device
+
+DROP MATERIALIZED VIEW IF EXISTS campaign_insights_by_device_flattened CASCADE;
+
+CREATE MATERIALIZED VIEW campaign_insights_by_device_flattened AS
+SELECT date_start AS "time",
+    concat('act_', account_id) AS account_id,
+    campaign_id,
+    device_platform,
+    impressions,
+    clicks,
+    spend,
+    reach,
+    frequency,
+    date_start,
+    date_stop,
+    (SELECT (value->>'value')::numeric
+     FROM jsonb_array_elements(actions)
+     WHERE value->>'action_type' = 'link_click') AS link_click,
+    (SELECT (value->>'value')::numeric
+     FROM jsonb_array_elements(actions)
+     WHERE value->>'action_type' = 'landing_page_view') AS landing_page_view,
+    (SELECT (value->>'value')::numeric
+     FROM jsonb_array_elements(actions)
+     WHERE value->>'action_type' = 'lead') AS lead
+     
+   FROM meta.custom_campaign_device;
+
+CREATE INDEX idx_campaign_insights_by_device_flat_date ON campaign_insights_by_device_flattened(date_start, date_stop);
+
+CREATE UNIQUE INDEX idx_campaign_insights_by_device_flat_unique ON campaign_insights_by_device_flattened(time, campaign_id, device_platform);
+REFRESH MATERIALIZED VIEW CONCURRENTLY campaign_insights_by_device_flattened;

src/meta_api_grabber/views/campaign_insights_by_gender.sql

@@ -0,0 +1,71 @@
+--- campaign insights by country
+
+DROP MATERIALIZED VIEW IF EXISTS campaign_insights_by_gender_and_age CASCADE;
+
+CREATE MATERIALIZED VIEW campaign_insights_by_gender_and_age AS
+SELECT date_start AS "time",
+    concat('act_', account_id) AS account_id,
+    campaign_id,
+    gender,
+	  age,
+    impressions,
+    clicks,
+    spend,
+    reach,
+    frequency,
+    ctr,
+    cpc,
+    cpm,
+    date_start,
+    date_stop,
+    (SELECT (value->>'value')::numeric
+     FROM jsonb_array_elements(actions)
+     WHERE value->>'action_type' = 'link_click') AS link_click,
+    (SELECT (value->>'value')::numeric
+     FROM jsonb_array_elements(actions)
+     WHERE value->>'action_type' = 'landing_page_view') AS landing_page_view,
+    (SELECT (value->>'value')::numeric
+     FROM jsonb_array_elements(actions)
+     WHERE value->>'action_type' = 'lead') AS lead
+     
+   FROM meta.custom_campaign_gender;
+
+CREATE INDEX idx_campaign_insights_by_gender_and_age_date ON campaign_insights_by_gender_and_age(date_start, date_stop);
+
+CREATE UNIQUE INDEX idx_campaign_insights_by_gender_and_age_unique ON campaign_insights_by_gender_and_age(time, campaign_id, gender, age);
+REFRESH MATERIALIZED VIEW CONCURRENTLY campaign_insights_by_gender_and_age;
+
+
+DROP VIEW IF EXISTS campaign_insights_by_gender CASCADE;
+
+create view campaign_insights_by_gender as 
+Select time, 
+sum(clicks) as clicks, 
+sum(link_click) as link_click,
+sum(lead) as lead,
+sum(landing_page_view) as landing_page_view,
+sum(spend) as spend, 
+sum(reach) as reach, 
+sum(impressions) as impressions, 
+gender,
+campaign_id,
+account_id 
+from campaign_insights_by_gender_and_age 
+group by time, gender, account_id, campaign_id, date_start, date_stop;
+
+DROP VIEW IF EXISTS campaign_insights_by_age CASCADE;
+
+create view campaign_insights_by_age as 
+Select time, 
+sum(clicks) as clicks, 
+sum(link_click) as link_click,
+sum(lead) as lead,
+sum(landing_page_view) as landing_page_view,
+sum(spend) as spend, 
+sum(reach) as reach, 
+sum(impressions) as impressions, 
+age,
+campaign_id,
+account_id 
+from campaign_insights_by_gender_and_age 
+group by time, age, account_id, campaign_id, date_start, date_stop;