INNER CODE UNIT · Python

download

trevorhobenshield/twitter-api-client · scripts/update.py:133

def download(urls: list[str], out: str = 'tmp', sz: int = None, fname_fn: partial = None, **kwargs) -> Generator:
    async def get(client: AsyncClient, sem: Semaphore, url: str):
        fname = url.split('/')[-1] if not fname_fn else fname_fn(url)
        async with aiofiles.open(f'{_out}/{fname}', 'wb') as fp:
            r = await backoff(client.get, sem, url, **kwargs)
            async for chunk in r.aiter_bytes(sz):
                await fp.write(chunk)
            return r

    _out = mkdir(out)
    return (partial(get, url=u) for u in urls)


def send(cfgs: list[dict], **kwargs) -> Generator:
    async def f(client: AsyncClient, sem: Semaphore, cfg: dict) -> Response:
        return await backoff(client.request, sem, **cfg, **kwargs)

    return (partial(f, cfg=cfg) for cfg in cfgs)

View source record →