INNER CODE UNIT · Python
Chunker
Laurent00TT/PharosRAG · src/chunker/core.py:243
class Chunker:
"""The chunker component. `chunk(elements, ...)` -> ChunkResult. Stateless config holder."""
def __init__(self, target=None, min_tokens=None, max_tokens=None, budgets=None, page_grouped=None):
self._override = None
if target is not None:
self._override = (min_tokens or 200, target, max_tokens or 1500)
self.budgets = {**BUDGETS, **(budgets or {})}
self.page_grouped = page_grouped if page_grouped is not None else PAGE_GROUPED
def _budget(self, doc_type):
return self._override or self.budgets.get(doc_type, DEFAULT_BUDGET)
def chunk(self, elements: list[Element], *, doc_id: str, doc_type: str | None = None,
lang: str = "en", doc_meta: dict | None = None, acl: dict | None = None) -> ChunkResult:
budget = self._budget(doc_type)
banners = _banner_texts(elements) # round-2: drop per-page running banners
kept = [e for e in elements