INNER CODE UNIT · Python

assemble_text

Laurent00TT/PharosRAG · src/chunker/core.py:208

def assemble_text(blocks, lang, budget):
    """Greedy accumulate consecutive same-section text blocks to a token budget. THE knob."""
    lo, target, hi = budget
    groups, cur, cur_tok = [], [], 0.0

    def flush():
        nonlocal cur, cur_tok
        if cur:
            groups.append(cur); cur = []; cur_tok = 0.0

    for b in blocks:
        bt = est_tokens(b["text"], lang)
        if bt > hi:
            flush()
            for piece in _sentence_split(b["text"], hi, lang):
                groups.append([{**b, "text": piece}])
            continue
        if cur and not b.get("merge") and cur_tok + bt > target:

View source record →