INNER CODE UNIT · Python
_sentence_split
Laurent00TT/PharosRAG · src/chunker/core.py:196
def _sentence_split(text, hi, lang):
parts = re.split(r"(?<=[。！？.!?])\s+", text)
out, cur = [], ""
for p in parts:
if est_tokens(cur + p, lang) > hi and cur:
out.append(cur.strip()); cur = ""
cur += p + " "
if cur.strip():
out.append(cur.strip())
return out or [text]
def assemble_text(blocks, lang, budget):
"""Greedy accumulate consecutive same-section text blocks to a token budget. THE knob."""
lo, target, hi = budget
groups, cur, cur_tok = [], [], 0.0
def flush():