import requests
import sys
import json
sys.path.insert(0, r'C:\Users\Roel\clawd\scripts\demandgen-pipeline')
from config import WEBGAINS_API, PUBLISHER_ID, webgains_headers

# Try smaller pages to see if we can paginate properly
cid = 1622230
url = f'{WEBGAINS_API}/auth/publishers/{PUBLISHER_ID}/campaigns/{cid}/feeds/products'

cursor = None
total = 0
seen_ids = set()
feeds_seen = set()
for page in range(100):
    params = {'size': 100, 'fields[]': ['title','brand','price','id']}
    if cursor:
        params['search_after'] = cursor
    r = requests.get(url, headers=webgains_headers(), params=params)
    d = r.json()
    products = d.get('data', [])
    if not products:
        print(f'No products on page {page+1}')
        break
    new_cursor = d.get('pagination', {}).get('search_cursor', '')
    if new_cursor == cursor:
        print(f'Cursor unchanged on page {page+1}')
        break
    cursor = new_cursor
    
    new_products = 0
    for p in products:
        pid = p.get('id')
        if pid not in seen_ids:
            seen_ids.add(pid)
            new_products += 1
        # Track feed from cursor
    
    total += len(products)
    cursor_parts = cursor.split('_')
    if len(cursor_parts) >= 2:
        feeds_seen.add(f'{cursor_parts[0]}_{cursor_parts[1]}')
    
    if page < 5 or page % 10 == 0:
        print(f'Page {page+1}: {len(products)} ({new_products} new), total: {total}, unique: {len(seen_ids)}, cursor: {cursor}')
    
    if len(products) < 100:
        print(f'Last page: {page+1}')
        break

print(f'\nTotal fetched: {total}')
print(f'Unique products: {len(seen_ids)}')
print(f'Feeds seen: {feeds_seen}')