INNER CODE UNIT · Python

get_txt_filepaths_from_zip

IlyaGusev/rulm · data_processing/convert_math.py:18

def get_txt_filepaths_from_zip(archive):
    txt_files = []
    for file_path in archive.namelist():
        if file_path.endswith('.txt') and file_path.startswith("whole_corpus/train"):
            print(file_path)
            txt_files.append(file_path)
    return txt_files


def parse_single_txt(path_to_txt, zip_file):
    with zip_file.open(path_to_txt) as f:
        lines = f.readlines()
    lines = [line.decode("utf-8").strip() for line in lines]
    lines = [line for line in lines if line]
    assert len(lines) % 2 == 0, "\n".join(lines)

    examples = []
    for i in range(len(lines) // 2):

View source record →