INNER CODE UNIT · Python

readFile

taishi-i/nagisa · nagisa/mecab_system_eval.py:13

def readFile(filename):
    sent = []
    data = []
    with open(filename, 'r', encoding='utf_8_sig') as f:
        for line in f:
            line = line.rstrip()
            if line == "EOS":
                data.append(sent)
                sent = []
            else:
                surface, csv_form = line.split("\t")
                csv_form = line
                if PY_3 is True:
                    surface  = surface.encode("UTF-8")
                    csv_form = csv_form.encode("UTF-8")
                sent.append([surface, csv_form])
    return data

View source record →