INNER CODE UNIT · Python
process_files
junruxiong/IncarnaMind · docs2db.py:296
def process_files():
"""Main function for processing files. Loads, tokenizes, and saves document data."""
with Pool() as pool:
chunks_small = []
chunks_medium = []
file_names = []
with tqdm(total=len(files_path), desc="Processing files", ncols=80) as pbar:
for doc in pool.imap_unordered(load_file, files_path):
file_name_with_extension = os.path.basename(doc[0].metadata["source"])
# file_name, _ = os.path.splitext(file_name_with_extension)
chunk_split_small = split_doc(
doc=doc,
chunk_size=configs.base_chunk_size,
chunk_overlap=configs.chunk_overlap,
chunk_idx_name="small_chunk_idx",
)