INNER CODE UNIT · Python
_list_hf_split_parquet_files
paperswithbacktest/pwb-toolbox · pwb_toolbox/datasets/__init__.py:89
def _list_hf_split_parquet_files(repo_files: list[str], split: str) -> list[str]:
patterns = [
re.compile(rf"(^|/){re.escape(split)}-\d{{5}}-of-\d{{5}}\.parquet$"),
re.compile(rf"(^|/){re.escape(split)}-\d{{4}}-\d{{2}}\.parquet$"),
re.compile(rf"(^|/){re.escape(split)}\.parquet$"),
]
matched = [f for f in repo_files if any(p.search(f) for p in patterns)]
if matched:
return sorted(matched)
# Fallback for unusual naming conventions.
return sorted([f for f in repo_files if f.endswith(".parquet") and f"/{split}" in f])
def _load_dataset_from_hf(dataset_name: str, split: str, hf_token: str, symbols=None) -> pd.DataFrame:
repo_id = f"paperswithbacktest/{dataset_name}"
api = HfApi(token=hf_token)
repo_files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")