INNER CODE UNIT · Python

process_asset

nubenetes/awesome-kubernetes · src/main.py:219

    async def process_asset(asset, idx):
        async with semaphore:
            # 1. Expand URL
            expanded_url = await resolve_url(asset["url"])
            asset["url"] = expanded_url
            
            # 2. Resilient Health Check (Identity Rotation & Domain failure stats - Recommendation #2)
            parsed = urlparse(expanded_url)
            domain = parsed.netloc.lower()
            
            domain_info = health_learning["domains"].setdefault(domain, {"attempts": 0, "failures": 0, "consecutive_failures": 0, "success_rate": 100.0})
            consecutive_failures = domain_info.get("consecutive_failures", 0)
            success_rate = domain_info.get("success_rate", 100.0)
            
            timeout_val = 12.0
            if consecutive_failures >= 3 or success_rate < 50.0:
                timeout_val = 3.0
                ua = fallback_user_agents[idx % len(fallback_user_agents)]

View source record →