INNER CODE UNIT · Python
__call__
liyucheng09/Selective_Context · app/app.py:225
def __call__(self, text: str, reduce_ratio: float = 0.35, reduce_level :str = 'phrase') -> List[str]:
context = self.beautify_context(text)
self.mask_ratio = reduce_ratio
sents = re.split(self.sent_tokenize_pattern, context)
sents = [sent.strip() for sent in sents if sent.strip()]
# You want the reduce happen at sentence level, phrase level, or token level?
assert reduce_level in ['sent', 'phrase', 'token'], f"reduce_level should be one of ['sent', 'phrase', 'token'], got {reduce_level}"
sent_lus, phrase_lus, token_lus = self._lexical_unit(sents)
lexical_level = {
'sent': sent_lus,
'phrase': phrase_lus,
'token': token_lus
}
# context is the reduced context, masked_sents denotes what context has been filtered out