INNER CODE UNIT · Python
tokenize
Koziev/NLP_Datasets · Samples/sort_samples_by_kenlm.py:20
def tokenize(s):
for word in nltk.word_tokenize(s):
yield word
def is_punct(word):
return len(word)>0 and unicodedata.category(word[0]) == 'Po'
def is_num(s):
if len(s)>0:
if s[0] in ('-', '+'):
return s[1:].isdigit()
return s.isdigit()
else:
return False