INNER CODE UNIT · Python
beautify_context
liyucheng09/Selective_Context · app/app.py:188
def beautify_context(self, context: str) -> str:
context = re.sub(r"\s+", " ", context)
return context
def self_info_mask(self, sents: List[str], self_info: List[float], mask_level):
# mask_level: mask sentences, phrases, or tokens
sents_after_mask = []
masked_sents = []
self.ppl_threshold = np.nanpercentile(self_info, self.mask_ratio * 100)
# if title is not None:
# with open(os.path.join(self.path, title+'_prob_token.tsv'), 'w', encoding='utf-8') as f:
# for token, info in zip(tokens, self_info):
# f.write(f"{token}\t{info}\n")
# with open(os.path.join(self.path, title+'_prob_sent.tsv'), 'w', encoding='utf-8') as f:
# for sent, info in zip(sents, sent_self_info):
# f.write(f"{sent}\n{info}\n\n")