INNER CODE UNIT · Python

prepare_word

Koziev/NLP_Datasets · Samples/sort_samples_by_kenlm.py:38

def prepare_word(w):
    if is_num(w):
        return u'_num_'
    else:
        return w.lower()


def prepare4lm(tokens):
    s2 = unicode.join(u' ', [prepare_word(t) for t in tokens if not is_punct(t) and len(t)>0 ])
    return s2

# -------------------------------------------------------------------

model_filepath = '/home/eek/polygon/kenlm/ru.text.arpa'
print('Loading the language model {}...'.format(model_filepath) )
model = kenlm.Model( model_filepath )

for filename in glob.glob('./*.txt'):

View source record →