import requests, sys, json
sys.path.insert(0, r'C:\Users\Roel\clawd\scripts\demandgen-pipeline')
from config import WEBGAINS_API, PUBLISHER_ID, webgains_headers

all_products = {}
fields = ['title','brand','price','id','ean','colour','image_link','link','description','rrp','currency','in_stock','category_name','model_number']

for cid in [1622230, 1622240, 1622250, 1622260]:
    url = f'{WEBGAINS_API}/auth/publishers/{PUBLISHER_ID}/campaigns/{cid}/feeds/products'
    for size in [1000, 500]:
        r = requests.get(url, headers=webgains_headers(), params={'size': size, 'fields[]': fields})
        d = r.json()
        prods = d.get('data', [])
        if prods:
            new = 0
            for p in prods:
                pid = str(p.get('id', ''))
                if pid and pid not in all_products:
                    all_products[pid] = p
                    new += 1
            print(f'Campaign {cid}, size={size}: {len(prods)} returned, {new} new (total: {len(all_products)})')
            break
        print(f'Campaign {cid}, size={size}: 0 returned, trying smaller...')

print(f'\nTotal unique products: {len(all_products)}')
brands = {}
for p in all_products.values():
    b = p.get('brand', 'Unknown') or 'Unknown'
    brands[b] = brands.get(b, 0) + 1
print('Brands:')
for b, c in sorted(brands.items(), key=lambda x: -x[1]):
    print(f'  {b:25}: {c}')

# Save all products to JSON for import
with open('scripts/webgains-products.json', 'w') as f:
    json.dump(list(all_products.values()), f)
print(f'\nSaved {len(all_products)} products to scripts/webgains-products.json')