INNER CODE UNIT · Python

upload_and_update_dataset_to_gcs_format_async

openfoodfacts/openfoodfacts-ai · category-detection/main.py:355

async def upload_and_update_dataset_to_gcs_format_async(
    dataset_path: Path,
    output_path: Path,
    bucket_name: str,
    max_concurrent_uploads: int = 30,
    base_image_dir: Path | None = None,
    from_key: str | None = None,
):
    limiter = asyncio.Semaphore(max_concurrent_uploads)
    ignore = True if from_key is None else False
    missing_files = 0
    async with aiohttp.ClientSession() as session:
        async with asyncio.TaskGroup() as tg:
            async with aiofiles.open(dataset_path, "r") as input_file, aiofiles.open(
                output_path, "wb"
            ) as output_file:
                async with limiter:
                    tasks = set()

View source record →