INNER CODE UNIT · Python
_read_one_parquet
paperswithbacktest/pwb-toolbox · pwb_toolbox/datasets/__init__.py:33
def _read_one_parquet(source, symbols=None) -> pd.DataFrame:
"""Read a single parquet source, pushing the symbol filter down when possible.
``source`` is a local path (``str``) or a file-like object. When ``symbols``
is provided, the scalar ``symbol`` column is filtered via parquet predicate
pushdown (which prunes row groups, so a whole shard is never materialised),
and the news-style list column ``symbols`` is filtered row-wise right after
the read. Either way memory stays bounded to a single shard's matching rows
instead of the full dataset, which is what makes large sharded datasets
(e.g. the 75 GB 1-minute prices) loadable when a symbol subset is requested.
"""
filters = None
if symbols:
schema = pq.read_schema(source)
if "symbol" in schema.names:
filters = [("symbol", "in", list(symbols))]
if hasattr(source, "seek"):
source.seek(0)