INNER CODE UNIT · Python
post_fetch
XDeadHackerX/The_spy_job · requisitos/theHarvester/theHarvester/lib/core.py:237
async def post_fetch(cls, url, headers='', data='', params='', json=False, proxy=False):
if len(headers) == 0:
headers = {'User-Agent': Core.get_user_agent()}
timeout = aiohttp.ClientTimeout(total=720)
# by default timeout is 5 minutes, changed to 12 minutes
# results are well worth the wait
try:
if proxy:
proxy = str(random.choice(cls().proxy_list))
if params != "":
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
async with session.get(url, params=params, proxy=proxy) as response:
await asyncio.sleep(2)
return await response.text() if json is False else await response.json()
else:
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
async with session.get(url, proxy=proxy) as response:
await asyncio.sleep(2)