INNER CODE UNIT · Python

add_to_tail

liyucheng09/Selective_Context · app/app.py:32

    def add_to_tail(self, token, self_info):
        return LexicalUnits(self.unit_type, self.text + [token], self.self_info + [self_info])

class SelectiveContext:

    def __init__(self, model_type = 'gpt2', lang = 'en'):

        self.model_type = model_type
        self.lang = lang

        # this means we calculate self-information sentence by sentence
        self.sent_level_self_info = True

        self._prepare_phrase_tokenizer()
        self.sent_tokenize_pattern = r"(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?)\s"
        self.phrase_mask_token = ''
        self.sent_mask_token = "<deleted>"

View source record →