INNER CODE UNIT · Python

_load_dataset_from_hf

paperswithbacktest/pwb-toolbox · pwb_toolbox/datasets/__init__.py:103

def _load_dataset_from_hf(dataset_name: str, split: str, hf_token: str, symbols=None) -> pd.DataFrame:
    repo_id = f"paperswithbacktest/{dataset_name}"
    api = HfApi(token=hf_token)
    repo_files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
    parquet_files = _list_hf_split_parquet_files(repo_files, split)
    if not parquet_files:
        raise ValueError(f"No parquet files found for split '{split}' in dataset '{repo_id}'.")

    iterator = tqdm(parquet_files, desc=f"HF {dataset_name}", unit="file") if len(parquet_files) > 1 else parquet_files
    local_files = [
        hf_hub_download(
            repo_id=repo_id,
            repo_type="dataset",
            filename=file_name,
            token=hf_token,
        )
        for file_name in iterator
    ]

View source record →