INNER CODE UNIT · Python
num_tokens_from_string
eugeneyan/obsidian-copilot · src/app.py:99
def num_tokens_from_string(string: str, model_name: str) -> int:
"""
Returns the number of tokens in a string based on tiktoken encoding.
Args:
string: String to count tokens for
model_name: Tokenizer model type
Returns:
Number of tokens in the string
"""
encoding = tiktoken.encoding_for_model(model_name)
num_tokens = len(encoding.encode(string))
return num_tokens
def get_chunks_from_hits(
hits: List[dict], model_name: str = "gpt-3.5-turbo", max_tokens: int = 3200