INNER CODE UNIT · Python
_banner_texts
Laurent00TT/PharosRAG · src/chunker/core.py:115
def _banner_texts(elements):
"""Running banners/watermarks parser mislabeled as content: identical text appearing on
>= half the pages (review round-2). Page-fraction (not raw count) avoids killing legit
recurring headings ('SALARIES AND EXPENSES' x4, 'Engine Sensors' x6 are NOT banners).
Round-3: a banner must NOT end with a colon — colon labels like 'Prompt:' / 'GPT-4V:'
recur on 60% of pages in dialogue/multimodal papers but are CONTENT (speaker turns),
not page furniture. (bbox position-stability would be a stronger signal — deferred.)"""
npages = max((e.page for e in elements), default=0) + 1
if npages < 4:
return set()
pages_of, freq = defaultdict(set), Counter()
for e in elements:
if e.kind in NOISE_KINDS:
continue
t = _norm(e.text)
if t and not t.endswith((":", "：")):
pages_of[t].add(e.page); freq[t] += 1