INNER CODE UNIT · Python

AsyncFetcher

XDeadHackerX/The_spy_job · requisitos/theHarvester/theHarvester/lib/core.py:233

class AsyncFetcher:
    proxy_list = Core.proxy_list()

    @classmethod
    async def post_fetch(cls, url, headers='', data='', params='', json=False, proxy=False):
        if len(headers) == 0:
            headers = {'User-Agent': Core.get_user_agent()}
        timeout = aiohttp.ClientTimeout(total=720)
        # by default timeout is 5 minutes, changed to 12 minutes
        # results are well worth the wait
        try:
            if proxy:
                proxy = str(random.choice(cls().proxy_list))
                if params != "":
                    async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
                        async with session.get(url, params=params, proxy=proxy) as response:
                            await asyncio.sleep(2)
                            return await response.text() if json is False else await response.json()

View source record →