INNER CODE UNIT · Python
split_doc
junruxiong/IncarnaMind · docs2db.py:145
def split_doc(
doc: List[Document], chunk_size: int, chunk_overlap: int, chunk_idx_name: str
):
"""Splits a document into smaller chunks based on the provided size and overlap.
Args:
doc (Document): Document to be split.
chunk_size (int): Size of each chunk.
chunk_overlap (int): Overlap between adjacent chunks.
chunk_idx_name (str): Metadata key for storing chunk indices.
Returns:
list: List of Document objects representing the chunks.
"""
data_splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
length_function=tiktoken_len,