INNER CODE UNIT · Python
clean_text
lefterisloukas/edgar-crawler · extract_items.py:246
def clean_text(text: str) -> str:
"""
Clean the text by removing unnecessary blocks of text and substituting special characters.
Args:
text (str): The raw text string.
Returns:
str: The normalized, clean text.
"""
# Replace special characters with their corresponding substitutions
text = re.sub(r"[\xa0]", " ", text)
text = re.sub(r"[\u200b]", " ", text)
text = re.sub(r"[\x91]", "‘", text)
text = re.sub(r"[\x92]", "’", text)
text = re.sub(r"[\x93]", "“", text)
text = re.sub(r"[\x94]", "”", text)
text = re.sub(r"[\x95]", "•", text)