INNER CODE UNIT · Python

answer

IlyaGusev/rulm · data_processing/convert_math.py:37

        answer = lines[2 * i + 1]
        assert len(answer) < len(question), f"{answer} vs {question}"
        example =  question + " Ответ: " + answer
        examples.append(example)
    return examples


if __name__ == "__main__":
    zip_file = zipfile.ZipFile(input_path)
    txt_paths = get_txt_filepaths_from_zip(zip_file)
    archive = PlainArchive(output_path)
    all_examples = []
    for path in tqdm(txt_paths, total=len(txt_paths)):
        examples = parse_single_txt(path, zip_file)
        all_examples.extend(examples)
    random.shuffle(all_examples)
    for batch_num, batch in enumerate(gen_batch(all_examples, 1000)):
        text = "\n".join(batch)

View source record →