INNER CODE UNIT · Python

prepare4lm

Koziev/NLP_Datasets · Samples/sort_samples_by_kenlm.py:45

def prepare4lm(tokens):
    s2 = unicode.join(u' ', [prepare_word(t) for t in tokens if not is_punct(t) and len(t)>0 ])
    return s2

# -------------------------------------------------------------------

model_filepath = '/home/eek/polygon/kenlm/ru.text.arpa'
print('Loading the language model {}...'.format(model_filepath) )
model = kenlm.Model( model_filepath )

for filename in glob.glob('./*.txt'):
    print(u'Processing {}'.format(filename))

    sent_set = set()
    sent_list = []
    with codecs.open(filename, 'r', 'utf-8') as rdr:
        for line in rdr:
            line2 = prepare4lm( tokenize(line.strip()) )

View source record →