INNER CODE UNIT · Python

cjk_bigram

kangarooking/cangjie-skill · scripts/build_index.py:22

def cjk_bigram(text: str) -> str:
    """把连续中文串展开为 bigram 词序列，使 FTS5 能做中文子串匹配。"""

    def expand(m: re.Match) -> str:
        s = m.group(0)
        if len(s) == 1:
            return s
        return " ".join(s[i : i + 2] for i in range(len(s) - 1))

    return CJK_RE.sub(expand, text)


def build(chunks_path: Path, db_path: Path) -> int:
    chunks = [json.loads(line) for line in chunks_path.read_text(encoding="utf-8").splitlines() if line.strip()]
    db_path.parent.mkdir(parents=True, exist_ok=True)
    db_path.unlink(missing_ok=True)
    con = sqlite3.connect(db_path)
    con.execute("CREATE TABLE chunks (seq INTEGER PRIMARY KEY, chunk_id TEXT, heading_path TEXT, text TEXT, keywords TEXT)")

View source record →