INNER CODE UNIT · Python

num_tokens_from_string

Azure/logicapps · ai-sample/TokenizeDocFunction/function_app.py:72

def num_tokens_from_string(string: str, encoding_name="cl100k_base") -> int:
    if not string:
        return 0
    # Returns the number of tokens in a text string
    encoding = tiktoken.get_encoding(encoding_name)
    num_tokens = len(encoding.encode(string))
    return num_tokens


def split_document_by_splitter_type(
    documents,
    document_type,
    splitter="RECURSIVE",
    secondary_splitter="RECURSIVE",
    headers_to_split_on=None,
    chunk_size=4000,
    chunk_overlap=200,
    length_function=len,

View source record →