INNER CODE UNIT · Python
__init__
liyucheng09/Selective_Context · app/app.py:37
def __init__(self, model_type = 'gpt2', lang = 'en'):
self.model_type = model_type
self.lang = lang
# this means we calculate self-information sentence by sentence
self.sent_level_self_info = True
self._prepare_phrase_tokenizer()
self.sent_tokenize_pattern = r"(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?)\s"
self.phrase_mask_token = ''
self.sent_mask_token = "<deleted>"
self._prepare_model()
def _prepare_phrase_tokenizer(self):
# we use space to tokenize sentence into phrases
# for English, we should use `spacy.load("en_core_web_sm").add_pipe('merge_noun_chunks')`