INNER CODE UNIT · Python
merge_chunks
junruxiong/IncarnaMind · docs2db.py:260
def merge_chunks(doc: Document, scale_factor: int, chunk_idx_name: str):
"""Merges adjacent chunks into larger chunks based on a scaling factor.
Args:
doc (Document): List of Document objects.
scale_factor (int): The number of small chunks to merge into a larger chunk.
chunk_idx_name (str): Metadata key for storing chunk indices.
Returns:
list: List of Document objects representing the merged chunks.
"""
merged_doc = []
page_content = ""
metadata_list = []
chunk_idx = 0
for idx, item in enumerate(doc):
page_content += item.page_content