diff --git a/RATE_LIMITER_ENHANCEMENTS.md b/RATE_LIMITER_ENHANCEMENTS.md index e824cc8..4d249af 100644 --- a/RATE_LIMITER_ENHANCEMENTS.md +++ b/RATE_LIMITER_ENHANCEMENTS.md @@ -24,7 +24,7 @@ Tracks application-level rate limits across all users. ``` ### 2. **X-Ad-Account-Usage** (Ad Account Specific) -Tracks rate limits for specific ad accounts. +Tracks rate limits for specific ad accounts. **Stored per account ID** to support multiple accounts. **Fields:** - `acc_id_util_pct`: Percentage of ad account usage (0-100) @@ -40,6 +40,8 @@ Tracks rate limits for specific ad accounts. } ``` +**Note:** Metrics are tracked separately for each ad account in a dictionary keyed by account ID (e.g., `act_123456789`). + ### 3. **X-Business-Use-Case-Usage** (Business Use Case Limits) Tracks rate limits per business use case (ads_insights, ads_management, etc.). @@ -156,6 +158,17 @@ logging.basicConfig( ) ``` +### Updating Usage with Account ID +When calling `update_usage()`, you can optionally provide an account ID to track per-account metrics: + +```python +# Option 1: Provide account_id explicitly +limiter.update_usage(response, account_id='act_123456789') + +# Option 2: Let the limiter try to extract it from the response +limiter.update_usage(response) # Will attempt to extract account_id +``` + ### Access New Metrics All metrics are available through the `get_stats()` method: @@ -163,10 +176,14 @@ All metrics are available through the `get_stats()` method: stats = limiter.get_stats() print(f"App call count: {stats['app_call_count']}%") -print(f"Ad account usage: {stats['ad_account_usage_pct']}%") -print(f"Reset in: {stats['reset_time_duration']}s") print(f"Regain access in: {stats['estimated_time_to_regain_access']} min") -print(f"API tier: {stats['ads_api_access_tier']}") + +# Per-account metrics +for account_id, usage in stats['ad_account_usage'].items(): + print(f"Account {account_id}:") + print(f" Usage: {usage['acc_id_util_pct']}%") + print(f" Reset in: {usage['reset_time_duration']}s") + print(f" API tier: {usage['ads_api_access_tier']}") # Business use case details for buc in stats['buc_usage']: diff --git a/src/meta_api_grabber/rate_limiter.py b/src/meta_api_grabber/rate_limiter.py index be596e8..ebcc5ab 100644 --- a/src/meta_api_grabber/rate_limiter.py +++ b/src/meta_api_grabber/rate_limiter.py @@ -59,10 +59,9 @@ class MetaRateLimiter: self.app_total_cputime: float = 0.0 self.app_total_time: float = 0.0 - # X-Ad-Account-Usage (ad account specific) - self.ad_account_usage_pct: float = 0.0 - self.reset_time_duration: int = 0 # seconds until reset - self.ads_api_access_tier: Optional[str] = None + # X-Ad-Account-Usage (ad account specific) - tracked per account + # Key: account_id (e.g., "act_123456789"), Value: dict with metrics + self.ad_account_usage: Dict[str, Dict[str, Any]] = {} # X-Business-Use-Case-Usage (business use case limits) self.buc_usage: List[Dict[str, Any]] = [] @@ -128,7 +127,7 @@ class MetaRateLimiter: logger.debug(f"Failed to parse X-App-Usage header: {e}") return {'call_count': 0.0, 'total_time': 0.0, 'total_cputime': 0.0} - def parse_x_ad_account_usage(self, response: Any) -> Dict[str, Any]: + def parse_x_ad_account_usage(self, response: Any) -> Optional[Dict[str, Any]]: """ Parse X-Ad-Account-Usage header (Ad account specific limits). @@ -142,7 +141,8 @@ class MetaRateLimiter: response: API response object Returns: - Dictionary with acc_id_util_pct, reset_time_duration, ads_api_access_tier + Dictionary with metrics, or None if header not present. + To determine account_id, check response object or URL. """ try: headers = self._get_headers(response) @@ -160,7 +160,41 @@ class MetaRateLimiter: return result except Exception as e: logger.debug(f"Failed to parse X-Ad-Account-Usage header: {e}") - return {'acc_id_util_pct': 0.0, 'reset_time_duration': 0, 'ads_api_access_tier': None} + return None + + def _extract_account_id(self, response: Any) -> Optional[str]: + """ + Extract account ID from response object. + + Tries multiple methods to find the account ID from the response. + + Args: + response: API response object + + Returns: + Account ID string (e.g., "act_123456789") or None + """ + # Try to get account_id from response attributes + if hasattr(response, 'account_id'): + return response.account_id + if hasattr(response, '_data') and isinstance(response._data, dict): + return response._data.get('account_id') + + # Try to get from parent object + if hasattr(response, '_parent_object'): + parent = response._parent_object + if hasattr(parent, 'get_id'): + return parent.get_id() + if hasattr(parent, '_data') and isinstance(parent._data, dict): + return parent._data.get('account_id') or parent._data.get('id') + + # Try to get from API context + if hasattr(response, '_api_context'): + context = response._api_context + if hasattr(context, 'account_id'): + return context.account_id + + return None def parse_x_business_use_case_usage(self, response: Any) -> List[Dict[str, Any]]: """ @@ -233,18 +267,20 @@ class MetaRateLimiter: logger.debug(f"Failed to parse x-fb-ads-insights-throttle header: {e}") return {'app_id_util_pct': 0.0, 'acc_id_util_pct': 0.0} - def update_usage(self, response: Any): + def update_usage(self, response: Any, account_id: Optional[str] = None): """ Update usage statistics from all API response headers. Parses and updates metrics from: - X-App-Usage - - X-Ad-Account-Usage + - X-Ad-Account-Usage (per account) - X-Business-Use-Case-Usage - x-fb-ads-insights-throttle (legacy) Args: response: API response object + account_id: Optional account ID (e.g., "act_123456789"). + If not provided, will attempt to extract from response. """ # Parse all headers app_usage = self.parse_x_app_usage(response) @@ -257,10 +293,20 @@ class MetaRateLimiter: self.app_total_cputime = app_usage['total_cputime'] self.app_total_time = app_usage['total_time'] - # Update X-Ad-Account-Usage metrics - self.ad_account_usage_pct = ad_account_usage['acc_id_util_pct'] - self.reset_time_duration = ad_account_usage['reset_time_duration'] - self.ads_api_access_tier = ad_account_usage['ads_api_access_tier'] + # Update X-Ad-Account-Usage metrics (per account) + if ad_account_usage: + # Try to get account_id + if not account_id: + account_id = self._extract_account_id(response) + + # Use 'unknown' as fallback if we can't determine account + if not account_id: + account_id = 'unknown' + logger.debug("Could not determine account_id, using 'unknown'") + + # Store usage for this account + self.ad_account_usage[account_id] = ad_account_usage + logger.debug(f"Updated ad account usage for {account_id}") # Update X-Business-Use-Case-Usage metrics self.buc_usage = buc_usage @@ -292,11 +338,14 @@ class MetaRateLimiter: if self.app_total_time > self.throttle_threshold: warnings.append(f"App total time: {self.app_total_time:.1f}%") - # Check X-Ad-Account-Usage - if self.ad_account_usage_pct > self.throttle_threshold: - warnings.append(f"Ad account: {self.ad_account_usage_pct:.1f}%") - if self.reset_time_duration > 0: - warnings.append(f"Resets in {self.reset_time_duration}s") + # Check X-Ad-Account-Usage (per account) + for account_id, usage in self.ad_account_usage.items(): + acc_pct = usage.get('acc_id_util_pct', 0) + if acc_pct > self.throttle_threshold: + warnings.append(f"Account {account_id}: {acc_pct:.1f}%") + reset_time = usage.get('reset_time_duration', 0) + if reset_time > 0: + warnings.append(f"Resets in {reset_time}s") # Check X-Business-Use-Case-Usage for buc in self.buc_usage: @@ -328,11 +377,14 @@ class MetaRateLimiter: self.app_call_count, self.app_total_cputime, self.app_total_time, - self.ad_account_usage_pct, self.legacy_app_usage_pct, self.legacy_account_usage_pct, ] + # Add ad account usage percentages (per account) + for usage in self.ad_account_usage.values(): + usage_values.append(usage.get('acc_id_util_pct', 0)) + # Add BUC usage percentages for buc in self.buc_usage: usage_values.extend([ @@ -373,12 +425,15 @@ class MetaRateLimiter: logger.info(f"Using BUC estimated_time_to_regain_access: {self.estimated_time_to_regain_access} min ({delay}s)") return min(delay, self.max_retry_delay) - # If we have reset_time_duration from Ad Account header, consider it - if self.reset_time_duration > 0 and self.ad_account_usage_pct >= 90: - # Use a fraction of reset_time_duration as delay - delay = min(self.reset_time_duration * 0.5, self.max_retry_delay) - logger.info(f"Using Ad Account reset_time_duration: {self.reset_time_duration}s (delay: {delay}s)") - return delay + # Check if any ad account has reset_time_duration and high usage + for account_id, usage in self.ad_account_usage.items(): + acc_pct = usage.get('acc_id_util_pct', 0) + reset_time = usage.get('reset_time_duration', 0) + if reset_time > 0 and acc_pct >= 90: + # Use a fraction of reset_time_duration as delay + delay = min(reset_time * 0.5, self.max_retry_delay) + logger.info(f"Using Ad Account {account_id} reset_time_duration: {reset_time}s (delay: {delay}s)") + return delay # Progressive delay based on usage # 75% = base_delay, 90% = 2x, 95% = 5x, 99% = 10x @@ -518,10 +573,8 @@ class MetaRateLimiter: 'app_total_cputime': self.app_total_cputime, 'app_total_time': self.app_total_time, - # X-Ad-Account-Usage metrics - 'ad_account_usage_pct': self.ad_account_usage_pct, - 'reset_time_duration': self.reset_time_duration, - 'ads_api_access_tier': self.ads_api_access_tier, + # X-Ad-Account-Usage metrics (per account) + 'ad_account_usage': self.ad_account_usage, # X-Business-Use-Case-Usage metrics 'buc_usage': self.buc_usage, @@ -558,12 +611,15 @@ class MetaRateLimiter: output.append(f" Total Time: {stats['app_total_time']:.1f}%") output.append("") - # X-Ad-Account-Usage - output.append("X-Ad-Account-Usage:") - output.append(f" Account Usage: {stats['ad_account_usage_pct']:.1f}%") - output.append(f" Reset Time Duration: {stats['reset_time_duration']}s") - output.append(f" API Access Tier: {stats['ads_api_access_tier'] or 'N/A'}") - output.append("") + # X-Ad-Account-Usage (per account) + if stats['ad_account_usage']: + output.append("X-Ad-Account-Usage (Per Account):") + for account_id, usage in stats['ad_account_usage'].items(): + output.append(f" Account: {account_id}") + output.append(f" Usage: {usage.get('acc_id_util_pct', 0):.1f}%") + output.append(f" Reset Time: {usage.get('reset_time_duration', 0)}s") + output.append(f" API Access Tier: {usage.get('ads_api_access_tier') or 'N/A'}") + output.append("") # X-Business-Use-Case-Usage if stats['buc_usage']: diff --git a/test_rate_limiter.py b/test_rate_limiter.py index a47590f..b1024f3 100644 --- a/test_rate_limiter.py +++ b/test_rate_limiter.py @@ -54,8 +54,8 @@ async def test_rate_limiter(): limiter.update_usage(response1) limiter.print_stats() - # Test 2: X-Ad-Account-Usage header - print("\n--- Test 2: X-Ad-Account-Usage Header ---") + # Test 2: X-Ad-Account-Usage header (first account) + print("\n--- Test 2: X-Ad-Account-Usage Header (Account 1) ---") response2 = MockResponse(headers={ 'x-ad-account-usage': json.dumps({ 'acc_id_util_pct': 78.5, @@ -63,7 +63,19 @@ async def test_rate_limiter(): 'ads_api_access_tier': 'development_access' }) }) - limiter.update_usage(response2) + limiter.update_usage(response2, account_id='act_123456789') + limiter.print_stats() + + # Test 2b: X-Ad-Account-Usage header (second account) + print("\n--- Test 2b: X-Ad-Account-Usage Header (Account 2) ---") + response2b = MockResponse(headers={ + 'x-ad-account-usage': json.dumps({ + 'acc_id_util_pct': 45.2, + 'reset_time_duration': 80, + 'ads_api_access_tier': 'standard_access' + }) + }) + limiter.update_usage(response2b, account_id='act_987654321') limiter.print_stats() # Test 3: X-Business-Use-Case-Usage header