INNER CODE UNIT · Python

flush

Laurent00TT/PharosRAG · src/chunker/core.py:213

    def flush():
        nonlocal cur, cur_tok
        if cur:
            groups.append(cur); cur = []; cur_tok = 0.0

    for b in blocks:
        bt = est_tokens(b["text"], lang)
        if bt > hi:
            flush()
            for piece in _sentence_split(b["text"], hi, lang):
                groups.append([{**b, "text": piece}])
            continue
        if cur and not b.get("merge") and cur_tok + bt > target:
            flush()
        cur.append(b); cur_tok += bt
    flush()

    merged = []

View source record →