INNER CODE UNIT · Python
fetch_normal_content
essamamdani/search-result-scraper-markdown · main.py:62
def fetch_normal_content(url):
try:
response = httpx.get(url, headers=HEADERS, timeout=REQUEST_TIMEOUT, proxies=proxies, follow_redirects=True)
response.raise_for_status()
return response.text
except httpx.RequestError as e:
print(f"An error occurred while requesting {url}: {e}")
except httpx.HTTPStatusError as e:
print(f"HTTP error occurred: {e}")
return None
def fetch_browserless_content(url):
try:
browserless_url = f"{BROWSERLESS_URL}/content"
params = {
"headless": False,
"stealth": True,
}