INNER CODE UNIT · Python
docs2pickle
junruxiong/IncarnaMind · docs2db.py:129
def docs2pickle(docs: List[Document], suffix: str = ""):
"""Serializes a list of Document objects to a pickle file.
Args:
docs (Document): List of Document objects.
suffix (str, optional): Suffix for the pickle file. Defaults to "".
"""
for doc in docs:
doc.page_content = clean_text(doc.page_content)
name = f"pickle_{suffix}"
if not os.path.exists(embedding_store_path):
os.makedirs(embedding_store_path)
with open(f"{embedding_store_path}/docs_{name}.pkl", "wb") as file:
pickle.dump(docs, file)
def split_doc(
doc: List[Document], chunk_size: int, chunk_overlap: int, chunk_idx_name: str