INNER CODE UNIT · Python
_symbols_overlap
paperswithbacktest/pwb-toolbox · pwb_toolbox/datasets/__init__.py:26
def _symbols_overlap(cell, symbol_set) -> bool:
"""True if a news-style ``symbols`` list cell shares any ticker with ``symbol_set``."""
if cell is None:
return False
return any(symbol in symbol_set for symbol in cell)
def _read_one_parquet(source, symbols=None) -> pd.DataFrame:
"""Read a single parquet source, pushing the symbol filter down when possible.
``source`` is a local path (``str``) or a file-like object. When ``symbols``
is provided, the scalar ``symbol`` column is filtered via parquet predicate
pushdown (which prunes row groups, so a whole shard is never materialised),
and the news-style list column ``symbols`` is filtered row-wise right after
the read. Either way memory stays bounded to a single shard's matching rows
instead of the full dataset, which is what makes large sharded datasets
(e.g. the 75 GB 1-minute prices) loadable when a symbol subset is requested.
"""