jonas/meta_api_grabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://gitea.99tales.net/jonas/meta_api_grabber

Browse Source
This commit is contained in:
Jonas Linter
2025-11-04 10:53:27 +00:00
parent df6da80320 f962a1a83d
commit 9ff1ee31d0
3 changed files with 127 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
RATE_LIMITER_ENHANCEMENTS.md
View File

@@ -24,7 +24,7 @@ Tracks application-level rate limits across all users.
```
### 2. **X-Ad-Account-Usage** (Ad Account Specific)
Tracks rate limits for specific ad accounts.
Tracks rate limits for specific ad accounts. **Stored per account ID** to support multiple accounts.
**Fields:**
- `acc_id_util_pct`: Percentage of ad account usage (0-100)
@@ -40,6 +40,8 @@ Tracks rate limits for specific ad accounts.
}
```
**Note:** Metrics are tracked separately for each ad account in a dictionary keyed by account ID (e.g., `act_123456789`).
### 3. **X-Business-Use-Case-Usage** (Business Use Case Limits)
Tracks rate limits per business use case (ads_insights, ads_management, etc.).
@@ -156,6 +158,17 @@ logging.basicConfig(
)
```
### Updating Usage with Account ID
When calling `update_usage()`, you can optionally provide an account ID to track per-account metrics:
```python
# Option 1: Provide account_id explicitly
limiter.update_usage(response, account_id='act_123456789')
# Option 2: Let the limiter try to extract it from the response
limiter.update_usage(response) # Will attempt to extract account_id
```
### Access New Metrics
All metrics are available through the `get_stats()` method:
@@ -163,10 +176,14 @@ All metrics are available through the `get_stats()` method:
stats = limiter.get_stats()
print(f"App call count: {stats['app_call_count']}%")
print(f"Ad account usage: {stats['ad_account_usage_pct']}%")
print(f"Reset in: {stats['reset_time_duration']}s")
print(f"Regain access in: {stats['estimated_time_to_regain_access']} min")
print(f"API tier: {stats['ads_api_access_tier']}")
# Per-account metrics
for account_id, usage in stats['ad_account_usage'].items():
print(f"Account {account_id}:")
print(f" Usage: {usage['acc_id_util_pct']}%")
print(f" Reset in: {usage['reset_time_duration']}s")
print(f" API tier: {usage['ads_api_access_tier']}")
# Business use case details
for buc in stats['buc_usage']:

120
src/meta_api_grabber/rate_limiter.py
View File

@@ -59,10 +59,9 @@ class MetaRateLimiter:
self.app_total_cputime: float = 0.0
self.app_total_time: float = 0.0
# X-Ad-Account-Usage (ad account specific)
self.ad_account_usage_pct: float = 0.0
self.reset_time_duration: int = 0 # seconds until reset
self.ads_api_access_tier: Optional[str] = None
# X-Ad-Account-Usage (ad account specific) - tracked per account
# Key: account_id (e.g., "act_123456789"), Value: dict with metrics
self.ad_account_usage: Dict[str, Dict[str, Any]] = {}
# X-Business-Use-Case-Usage (business use case limits)
self.buc_usage: List[Dict[str, Any]] = []
@@ -128,7 +127,7 @@ class MetaRateLimiter:
logger.debug(f"Failed to parse X-App-Usage header: {e}")
return {'call_count': 0.0, 'total_time': 0.0, 'total_cputime': 0.0}
def parse_x_ad_account_usage(self, response: Any) -> Dict[str, Any]:
def parse_x_ad_account_usage(self, response: Any) -> Optional[Dict[str, Any]]:
"""
Parse X-Ad-Account-Usage header (Ad account specific limits).
@@ -142,7 +141,8 @@ class MetaRateLimiter:
response: API response object
Returns:
Dictionary with acc_id_util_pct, reset_time_duration, ads_api_access_tier
Dictionary with metrics, or None if header not present.
To determine account_id, check response object or URL.
"""
try:
headers = self._get_headers(response)
@@ -160,7 +160,41 @@ class MetaRateLimiter:
return result
except Exception as e:
logger.debug(f"Failed to parse X-Ad-Account-Usage header: {e}")
return {'acc_id_util_pct': 0.0, 'reset_time_duration': 0, 'ads_api_access_tier': None}
return None
def _extract_account_id(self, response: Any) -> Optional[str]:
"""
Extract account ID from response object.
Tries multiple methods to find the account ID from the response.
Args:
response: API response object
Returns:
Account ID string (e.g., "act_123456789") or None
"""
# Try to get account_id from response attributes
if hasattr(response, 'account_id'):
return response.account_id
if hasattr(response, '_data') and isinstance(response._data, dict):
return response._data.get('account_id')
# Try to get from parent object
if hasattr(response, '_parent_object'):
parent = response._parent_object
if hasattr(parent, 'get_id'):
return parent.get_id()
if hasattr(parent, '_data') and isinstance(parent._data, dict):
return parent._data.get('account_id') or parent._data.get('id')
# Try to get from API context
if hasattr(response, '_api_context'):
context = response._api_context
if hasattr(context, 'account_id'):
return context.account_id
return None
def parse_x_business_use_case_usage(self, response: Any) -> List[Dict[str, Any]]:
"""
@@ -233,18 +267,20 @@ class MetaRateLimiter:
logger.debug(f"Failed to parse x-fb-ads-insights-throttle header: {e}")
return {'app_id_util_pct': 0.0, 'acc_id_util_pct': 0.0}
def update_usage(self, response: Any):
def update_usage(self, response: Any, account_id: Optional[str] = None):
"""
Update usage statistics from all API response headers.
Parses and updates metrics from:
- X-App-Usage
- X-Ad-Account-Usage
- X-Ad-Account-Usage (per account)
- X-Business-Use-Case-Usage
- x-fb-ads-insights-throttle (legacy)
Args:
response: API response object
account_id: Optional account ID (e.g., "act_123456789").
If not provided, will attempt to extract from response.
"""
# Parse all headers
app_usage = self.parse_x_app_usage(response)
@@ -257,10 +293,20 @@ class MetaRateLimiter:
self.app_total_cputime = app_usage['total_cputime']
self.app_total_time = app_usage['total_time']
# Update X-Ad-Account-Usage metrics
self.ad_account_usage_pct = ad_account_usage['acc_id_util_pct']
self.reset_time_duration = ad_account_usage['reset_time_duration']
self.ads_api_access_tier = ad_account_usage['ads_api_access_tier']
# Update X-Ad-Account-Usage metrics (per account)
if ad_account_usage:
# Try to get account_id
if not account_id:
account_id = self._extract_account_id(response)
# Use 'unknown' as fallback if we can't determine account
if not account_id:
account_id = 'unknown'
logger.debug("Could not determine account_id, using 'unknown'")
# Store usage for this account
self.ad_account_usage[account_id] = ad_account_usage
logger.debug(f"Updated ad account usage for {account_id}")
# Update X-Business-Use-Case-Usage metrics
self.buc_usage = buc_usage
@@ -292,11 +338,14 @@ class MetaRateLimiter:
if self.app_total_time > self.throttle_threshold:
warnings.append(f"App total time: {self.app_total_time:.1f}%")
# Check X-Ad-Account-Usage
if self.ad_account_usage_pct > self.throttle_threshold:
warnings.append(f"Ad account: {self.ad_account_usage_pct:.1f}%")
if self.reset_time_duration > 0:
warnings.append(f"Resets in {self.reset_time_duration}s")
# Check X-Ad-Account-Usage (per account)
for account_id, usage in self.ad_account_usage.items():
acc_pct = usage.get('acc_id_util_pct', 0)
if acc_pct > self.throttle_threshold:
warnings.append(f"Account {account_id}: {acc_pct:.1f}%")
reset_time = usage.get('reset_time_duration', 0)
if reset_time > 0:
warnings.append(f"Resets in {reset_time}s")
# Check X-Business-Use-Case-Usage
for buc in self.buc_usage:
@@ -328,11 +377,14 @@ class MetaRateLimiter:
self.app_call_count,
self.app_total_cputime,
self.app_total_time,
self.ad_account_usage_pct,
self.legacy_app_usage_pct,
self.legacy_account_usage_pct,
]
# Add ad account usage percentages (per account)
for usage in self.ad_account_usage.values():
usage_values.append(usage.get('acc_id_util_pct', 0))
# Add BUC usage percentages
for buc in self.buc_usage:
usage_values.extend([
@@ -373,11 +425,14 @@ class MetaRateLimiter:
logger.info(f"Using BUC estimated_time_to_regain_access: {self.estimated_time_to_regain_access} min ({delay}s)")
return min(delay, self.max_retry_delay)
# If we have reset_time_duration from Ad Account header, consider it
if self.reset_time_duration > 0 and self.ad_account_usage_pct >= 90:
# Check if any ad account has reset_time_duration and high usage
for account_id, usage in self.ad_account_usage.items():
acc_pct = usage.get('acc_id_util_pct', 0)
reset_time = usage.get('reset_time_duration', 0)
if reset_time > 0 and acc_pct >= 90:
# Use a fraction of reset_time_duration as delay
delay = min(self.reset_time_duration * 0.5, self.max_retry_delay)
logger.info(f"Using Ad Account reset_time_duration: {self.reset_time_duration}s (delay: {delay}s)")
delay = min(reset_time * 0.5, self.max_retry_delay)
logger.info(f"Using Ad Account {account_id} reset_time_duration: {reset_time}s (delay: {delay}s)")
return delay
# Progressive delay based on usage
@@ -518,10 +573,8 @@ class MetaRateLimiter:
'app_total_cputime': self.app_total_cputime,
'app_total_time': self.app_total_time,
# X-Ad-Account-Usage metrics
'ad_account_usage_pct': self.ad_account_usage_pct,
'reset_time_duration': self.reset_time_duration,
'ads_api_access_tier': self.ads_api_access_tier,
# X-Ad-Account-Usage metrics (per account)
'ad_account_usage': self.ad_account_usage,
# X-Business-Use-Case-Usage metrics
'buc_usage': self.buc_usage,
@@ -558,11 +611,14 @@ class MetaRateLimiter:
output.append(f" Total Time: {stats['app_total_time']:.1f}%")
output.append("")
# X-Ad-Account-Usage
output.append("X-Ad-Account-Usage:")
output.append(f" Account Usage: {stats['ad_account_usage_pct']:.1f}%")
output.append(f" Reset Time Duration: {stats['reset_time_duration']}s")
output.append(f" API Access Tier: {stats['ads_api_access_tier'] or 'N/A'}")
# X-Ad-Account-Usage (per account)
if stats['ad_account_usage']:
output.append("X-Ad-Account-Usage (Per Account):")
for account_id, usage in stats['ad_account_usage'].items():
output.append(f" Account: {account_id}")
output.append(f" Usage: {usage.get('acc_id_util_pct', 0):.1f}%")
output.append(f" Reset Time: {usage.get('reset_time_duration', 0)}s")
output.append(f" API Access Tier: {usage.get('ads_api_access_tier') or 'N/A'}")
output.append("")
# X-Business-Use-Case-Usage

18
test_rate_limiter.py
View File

@@ -54,8 +54,8 @@ async def test_rate_limiter():
limiter.update_usage(response1)
limiter.print_stats()
# Test 2: X-Ad-Account-Usage header
print("\n--- Test 2: X-Ad-Account-Usage Header ---")
# Test 2: X-Ad-Account-Usage header (first account)
print("\n--- Test 2: X-Ad-Account-Usage Header (Account 1) ---")
response2 = MockResponse(headers={
'x-ad-account-usage': json.dumps({
'acc_id_util_pct': 78.5,
@@ -63,7 +63,19 @@ async def test_rate_limiter():
'ads_api_access_tier': 'development_access'
})
})
limiter.update_usage(response2)
limiter.update_usage(response2, account_id='act_123456789')
limiter.print_stats()
# Test 2b: X-Ad-Account-Usage header (second account)
print("\n--- Test 2b: X-Ad-Account-Usage Header (Account 2) ---")
response2b = MockResponse(headers={
'x-ad-account-usage': json.dumps({
'acc_id_util_pct': 45.2,
'reset_time_duration': 80,
'ads_api_access_tier': 'standard_access'
})
})
limiter.update_usage(response2b, account_id='act_987654321')
limiter.print_stats()
# Test 3: X-Business-Use-Case-Usage header