INNER CODE UNIT · Python

npages

Laurent00TT/PharosRAG · src/chunker/core.py:123

    npages = max((e.page for e in elements), default=0) + 1
    if npages < 4:
        return set()
    pages_of, freq = defaultdict(set), Counter()
    for e in elements:
        if e.kind in NOISE_KINDS:
            continue
        t = _norm(e.text)
        if t and not t.endswith((":", "：")):
            pages_of[t].add(e.page); freq[t] += 1
    return {t for t in freq if freq[t] >= 3 and len(pages_of[t]) >= npages * 0.5}


def _structural_nonheading(txt):
    """Text-shape rejects shared by the leveling pass and the reset-aware pre-pass."""
    return bool(TOC_RE.search(txt) or TOC_TAIL_RE.match(txt)
                or len(txt) > MAX_HEADING_LEN or BULLET_RE.match(txt))

View source record →