INNER CODE UNIT · Python
all_raw_assets
nubenetes/awesome-kubernetes · src/main.py:198
all_raw_assets = raw_social + trending
if not all_raw_assets:
log_event("[!] No new links found to process.")
return
# 4. Expansion, Resilient Health Check and Initial Deduplication
log_event(f"[*] Expanding, verifying and deduplicating {len(all_raw_assets)} raw links...")
semaphore = asyncio.Semaphore(15)
# User-Agent rotation for resilient discovery
user_agents = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
]
fallback_user_agents = [
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/124.0 Mobile/15E148 Safari/605.1.15",