INNER CODE UNIT · Python

SelectiveContext

liyucheng09/Selective_Context · app/app.py:35

class SelectiveContext:

    def __init__(self, model_type = 'gpt2', lang = 'en'):

        self.model_type = model_type
        self.lang = lang

        # this means we calculate self-information sentence by sentence
        self.sent_level_self_info = True

        self._prepare_phrase_tokenizer()
        self.sent_tokenize_pattern = r"(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?)\s"
        self.phrase_mask_token = ''
        self.sent_mask_token = "<deleted>"

        self._prepare_model()
    
    def _prepare_phrase_tokenizer(self):

View source record →