INNER CODE UNIT · Python

_read_parquet_files

paperswithbacktest/pwb-toolbox · pwb_toolbox/datasets/__init__.py:61

def _read_parquet_files(paths: list[str], desc: str, symbols=None) -> pd.DataFrame:
    if not paths:
        raise ValueError("No parquet files provided.")

    iterator = tqdm(paths, desc=desc, unit="file") if len(paths) > 1 else paths
    frames = []
    for path in iterator:
        if isinstance(path, str) and path.startswith(("http://", "https://")):
            resp = requests.get(path, timeout=120)
            resp.raise_for_status()
            frames.append(_read_one_parquet(BytesIO(resp.content), symbols))
        else:
            frames.append(_read_one_parquet(path, symbols))
    return pd.concat(frames, ignore_index=True) if len(frames) > 1 else frames[0]


def _load_dataset_from_pwb(dataset_name: str, split: str, pwb_api_key: str, symbols=None) -> pd.DataFrame:
    base_url = "https://data.paperswithbacktest.com/v1/datasets"

View source record →