INNER CODE UNIT · Python

load_chunker

cognitivetech/ollama-ebook-summary · tools-prototype/chunking/main.py:5

def load_chunker(module_name):
    module = importlib.import_module(module_name)
    class_ = getattr(module, module_name.capitalize())
    return class_()

def main():
    input_file = sys.argv[1]
    chunker_name = sys.argv[2]  # e.g., "semantic_chunker"

    chunker = load_chunker(chunker_name)
    output_file = os.path.splitext(input_file)[0] + f'_chunked_{chunker.id}.csv'

    with open(input_file, 'r', encoding='utf-8') as file:
        text = file.read()

    # Preprocess and chunk text
    text = chunker.preprocess_text(text)
    chunks = chunker.chunk_text(text)

View source record →