INNER CODE UNIT · Python
build
kangarooking/cangjie-skill · scripts/build_index.py:34
def build(chunks_path: Path, db_path: Path) -> int:
chunks = [json.loads(line) for line in chunks_path.read_text(encoding="utf-8").splitlines() if line.strip()]
db_path.parent.mkdir(parents=True, exist_ok=True)
db_path.unlink(missing_ok=True)
con = sqlite3.connect(db_path)
con.execute("CREATE TABLE chunks (seq INTEGER PRIMARY KEY, chunk_id TEXT, heading_path TEXT, text TEXT, keywords TEXT)")
con.execute("CREATE VIRTUAL TABLE chunks_fts USING fts5(chunk_id, heading_path, body, keywords)")
for seq, c in enumerate(chunks):
hp = " / ".join(c["heading_path"])
con.execute("INSERT INTO chunks VALUES (?,?,?,?,?)",
(seq, c["chunk_id"], hp, c["text"], " ".join(c.get("keywords", []))))
con.execute("INSERT INTO chunks_fts VALUES (?,?,?,?)",
(c["chunk_id"], cjk_bigram(hp), cjk_bigram(c["text"]), cjk_bigram(" ".join(c.get("keywords", [])))))
con.commit()
con.close()
print(f"索引已建: {db_path}（{len(chunks)} chunks）")
return 0