INNER CODE UNIT · Python
_split_text
AMontgomerie/question_generator · questiongenerator.py:140
def _split_text(self, text: str) -> List[str]:
"""Splits the text into sentences, and attempts to split or truncate long sentences."""
MAX_SENTENCE_LEN = 128
sentences = re.findall(".*?[.!\?]", text)
cut_sentences = []
for sentence in sentences:
if len(sentence) > MAX_SENTENCE_LEN:
cut_sentences.extend(re.split("[,;:)]", sentence))
# remove useless post-quote sentence fragments
cut_sentences = [s for s in sentences if len(s.split(" ")) > 5]
sentences = sentences + cut_sentences
return list(set([s.strip(" ") for s in sentences]))
def _split_into_segments(self, text: str) -> List[str]:
"""Splits a long text into segments short enough to be input into the transformer network.