INNER CODE UNIT · Python

remove_symbols

RQLuo/MixTeX-Latex-OCR · mixtex_data_gen/gen.py:71

def remove_symbols(text):
    pattern = re.compile(r'[^\u4e00-\u9fa5a-zA-Z，。！？、： ；“”‘’\'\"\(\)\[\]\{\}\<\>\.\,\?\!\:\;\-]')
    cleaned_text = re.sub(pattern, '', text)
    return cleaned_text

# Function to format words with LaTeX commands and randomly insert formulas and numbers
# Input:
#   words (list): List of words to format
#   formulas (list): List of LaTeX formulas to insert
#   lines (list): List of lines from the original text for random insertion
# Output:
#   output (str): The formatted LaTeX string
def format_text_with_latex(words, formulas, lines):
    output = ''
    count = 0
    for char in tqdm(words):
        count += 1
        if len(char) >= 2 and random.random() < 0.01:

View source record →