INNER CODE UNIT · Python
_load_dataset_from_pwb
paperswithbacktest/pwb-toolbox · pwb_toolbox/datasets/__init__.py:77
def _load_dataset_from_pwb(dataset_name: str, split: str, pwb_api_key: str, symbols=None) -> pd.DataFrame:
base_url = "https://data.paperswithbacktest.com/v1/datasets"
resp = requests.get(f"{base_url}/{dataset_name}", params={"pwb_api_key": pwb_api_key, "split": split}, timeout=30)
resp.raise_for_status()
files = resp.json().get("files", [])
if not files:
raise ValueError(f"No files available for dataset '{dataset_name}' and split '{split}'")
print(f"Downloading {len(files)} parquet files from PWB...")
return _read_parquet_files(files, desc=f"PWB {dataset_name}", symbols=symbols)
def _list_hf_split_parquet_files(repo_files: list[str], split: str) -> list[str]:
patterns = [
re.compile(rf"(^|/){re.escape(split)}-\d{{5}}-of-\d{{5}}\.parquet$"),
re.compile(rf"(^|/){re.escape(split)}-\d{{4}}-\d{{2}}\.parquet$"),
re.compile(rf"(^|/){re.escape(split)}\.parquet$"),
]