INNER CODE UNIT · Python
ua
nubenetes/awesome-kubernetes · src/main.py:236
ua = fallback_user_agents[idx % len(fallback_user_agents)]
else:
ua = user_agents[idx % len(user_agents)]
headers = {"User-Agent": ua, "Referer": "https://www.google.com/"}
domain_info["attempts"] = domain_info.get("attempts", 0) + 1
# NOTE: All domains must be checked to ensure the link isn't a 404.
try:
async with httpx.AsyncClient(headers=headers, follow_redirects=True, timeout=timeout_val, verify=False) as client:
resp = await client.get(expanded_url)
if resp.status_code == 404:
asset["health"] = "dead" # Definitively dead
domain_info["failures"] = domain_info.get("failures", 0) + 1
domain_info["consecutive_failures"] = domain_info.get("consecutive_failures", 0) + 1
else:
asset["health"] = "online"