INNER CODE UNIT · Python

fetch_content

essamamdani/search-result-scraper-markdown · main.py:60

def fetch_content(url):
    proxies = get_proxies()
    def fetch_normal_content(url):
        try:
            response = httpx.get(url, headers=HEADERS, timeout=REQUEST_TIMEOUT, proxies=proxies, follow_redirects=True)
            response.raise_for_status()
            return response.text
        except httpx.RequestError as e:
            print(f"An error occurred while requesting {url}: {e}")
        except httpx.HTTPStatusError as e:
            print(f"HTTP error occurred: {e}")
        return None

    def fetch_browserless_content(url):
        try:
            browserless_url = f"{BROWSERLESS_URL}/content"
            params = {
                "headless": False,

View source record →