INNER CODE UNIT · Python
estimate_tokens
mrwadams/stride-gpt · apps/web/main.py:503
def estimate_tokens(text, model="gpt-5.2"):
"""
Estimate the number of tokens in a text string.
Uses tiktoken for OpenAI models, or falls back to a character-based approximation.
Args:
text: The text to estimate tokens for
model: The model to use for estimation (default: gpt-5.2)
Returns:
Estimated token count
"""
try:
# Try to use tiktoken for accurate estimation
enc = tiktoken.encoding_for_model(model)
return len(enc.encode(text))
except (ImportError, KeyError, ValueError):
# Fall back to character-based approximation