INNER CODE UNIT · Python
extract_keywords
kangarooking/cangjie-skill · scripts/build_chunks.py:106
def extract_keywords(text: str, limit: int = 12) -> list[str]:
"""确定性关键词预筛：词频最高的中文单字组成的双字串 + 英文词。够 FTS5 使用即可。"""
freq: dict[str, int] = {}
for m in re.finditer(r"[\u4e00-\u9fff]{2,6}|[A-Za-z][A-Za-z0-9-]{2,}", text):
w = m.group(0).lower()
freq[w] = freq.get(w, 0) + 1
return [w for w, _ in sorted(freq.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]]
def group_chunks(elements: list[dict], source_id: str, version_id: str, max_chars: int) -> list[dict]:
chunks: list[dict] = []
cur: list[dict] = []
cur_path: list[str] = []
def flush():
nonlocal cur
if not cur:
return