INNER CODE UNIT · Python

ua

nubenetes/awesome-kubernetes · src/main.py:238

                ua = user_agents[idx % len(user_agents)]
                
            headers = {"User-Agent": ua, "Referer": "https://www.google.com/"}
            
            domain_info["attempts"] = domain_info.get("attempts", 0) + 1
            
            # NOTE: All domains must be checked to ensure the link isn't a 404.
            try:
                async with httpx.AsyncClient(headers=headers, follow_redirects=True, timeout=timeout_val, verify=False) as client:
                    resp = await client.get(expanded_url)
                    if resp.status_code == 404:
                        asset["health"] = "dead" # Definitively dead
                        domain_info["failures"] = domain_info.get("failures", 0) + 1
                        domain_info["consecutive_failures"] = domain_info.get("consecutive_failures", 0) + 1
                    else:
                        asset["health"] = "online"
                        domain_info["consecutive_failures"] = 0
            except Exception as e:

View source record →