#!/usr/bin/env python3 """ SneakerPicks — Deeplink/Affiliate Link Checker Checks if affiliate links still work by sending HTTP HEAD requests. Usage: python3 scripts/deeplink-audit.py # check 100 random offers python3 scripts/deeplink-audit.py --sample 500 # check 500 python3 scripts/deeplink-audit.py --feed "JD Sports" # check specific feed python3 scripts/deeplink-audit.py --fix # mark broken as out-of-stock """ import argparse import os import sys import time from concurrent.futures import ThreadPoolExecutor, as_completed from threading import Semaphore from urllib.parse import urlparse import psycopg2 import requests TIMEOUT = 10 MAX_WORKERS = 5 USER_AGENT = "SneakerPicks Link Checker/1.0" # Rate limit: max 5 requests/sec via semaphore + sleep rate_semaphore = Semaphore(MAX_WORKERS) def get_connection(): url = os.environ.get("DATABASE_URL") if not url: print("ERROR: DATABASE_URL environment variable not set") sys.exit(1) return psycopg2.connect(url) def check_link(offer_id: int, url: str) -> dict: """Check a single link. Returns result dict.""" rate_semaphore.acquire() try: try: resp = requests.head( url, timeout=TIMEOUT, allow_redirects=True, headers={"User-Agent": USER_AGENT}, ) status = resp.status_code final_url = resp.url if status in (404, 410): result = "broken" elif status >= 500: result = "error" elif status in (301, 302, 303, 307, 308) or final_url != url: result = "redirected" elif 200 <= status < 400: result = "ok" else: result = "error" return { "offer_id": offer_id, "url": url, "status": status, "final_url": final_url, "result": result, } except requests.RequestException as e: return { "offer_id": offer_id, "url": url, "status": None, "final_url": None, "result": "error", "error": str(e), } finally: time.sleep(0.2) # ~5 req/s across 5 workers rate_semaphore.release() def run(sample: int, feed: str | None, fix: bool): conn = get_connection() cur = conn.cursor() # Build query query = "SELECT id, product_url FROM product_offers WHERE in_stock = true" params: list = [] if feed: query += " AND feed_id = (SELECT id FROM feeds WHERE name = %s LIMIT 1)" params.append(feed) query += " ORDER BY RANDOM() LIMIT %s" params.append(sample) cur.execute(query, params) offers = cur.fetchall() print(f"Checking {len(offers)} offers" + (f" from feed '{feed}'" if feed else "") + "...") results = {"ok": 0, "redirected": 0, "broken": 0, "error": 0} broken_ids: list[int] = [] with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: futures = {executor.submit(check_link, oid, url): oid for oid, url in offers} for i, future in enumerate(as_completed(futures), 1): r = future.result() results[r["result"]] += 1 if r["result"] == "broken": broken_ids.append(r["offer_id"]) print(f" BROKEN [{r['status']}] {r['url']}") elif r["result"] == "error": err = r.get("error", f"HTTP {r['status']}") print(f" ERROR {r['url'][:80]} — {err}") if i % 50 == 0: print(f" ... {i}/{len(offers)} checked") print(f"\nResults: ✓ ok={results['ok']} ↪ redirected={results['redirected']} " f"✗ broken={results['broken']} ⚠ error={results['error']}") if broken_ids and fix: cur.execute(""" UPDATE product_offers SET in_stock = false WHERE id = ANY(%s) """, (broken_ids,)) conn.commit() print(f" ✓ Marked {cur.rowcount} broken offers as out-of-stock") elif broken_ids and not fix: print(f" {len(broken_ids)} broken offers found. Use --fix to mark as out-of-stock.") cur.close() conn.close() if __name__ == "__main__": parser = argparse.ArgumentParser(description="SneakerPicks deeplink audit") parser.add_argument("--sample", type=int, default=100, help="Number of offers to check (default: 100)") parser.add_argument("--feed", type=str, default=None, help="Filter by feed name") parser.add_argument("--fix", action="store_true", help="Mark broken links as out-of-stock") args = parser.parse_args() run(sample=args.sample, feed=args.feed, fix=args.fix)