INNER CODE UNIT · Python
process_text
RQLuo/MixTeX-Latex-OCR · mixtex_data_gen/gen.py:42
def process_text(input_file, output_file, formulas):
with open(input_file, 'r', encoding='utf-8') as f:
text = f.read()
text = text.replace('\n', '')
sentences = re.split(r'[.,]', text)
output = ''
with open(output_file, 'w', encoding='utf-8') as f:
for sentence in sentences:
for char in sentence:
output += char
if random.random() < 0.02:
formula = random.sample(formulas, 1)[0]
if len(formula) < 50:
try:
output += ' \\(' + re.sub(r'\\tag\{.*?\}', '', formula) + '\\) '
except:
print(formula)