INNER CODE UNIT · Python
mask_a_sent
liyucheng09/Selective_Context · app/app.py:217
def mask_a_sent(self, sent, level):
if level == 'phrase':
return self.phrase_mask_token
elif level == 'sent':
return self.sent_mask_token
elif level == 'token':
return ''
def __call__(self, text: str, reduce_ratio: float = 0.35, reduce_level :str = 'phrase') -> List[str]:
context = self.beautify_context(text)
self.mask_ratio = reduce_ratio
sents = re.split(self.sent_tokenize_pattern, context)
sents = [sent.strip() for sent in sents if sent.strip()]
# You want the reduce happen at sentence level, phrase level, or token level?
assert reduce_level in ['sent', 'phrase', 'token'], f"reduce_level should be one of ['sent', 'phrase', 'token'], got {reduce_level}"