INNER CODE UNIT · Python

t

trevorhobenshield/twitter-api-client · scripts/update.py:128

            t = min(random.random() * (b ** i), m)
            logger.info(f'Retrying in {f"{t:.2f}"} seconds\n{e}')
            await asyncio.sleep(t)


def download(urls: list[str], out: str = 'tmp', sz: int = None, fname_fn: partial = None, **kwargs) -> Generator:
    async def get(client: AsyncClient, sem: Semaphore, url: str):
        fname = url.split('/')[-1] if not fname_fn else fname_fn(url)
        async with aiofiles.open(f'{_out}/{fname}', 'wb') as fp:
            r = await backoff(client.get, sem, url, **kwargs)
            async for chunk in r.aiter_bytes(sz):
                await fp.write(chunk)
            return r

    _out = mkdir(out)
    return (partial(get, url=u) for u in urls)

View source record →