INNER CODE UNIT · Python

add_to_head

liyucheng09/Selective_Context · app/app.py:29

    def add_to_head(self, token, self_info):
        return LexicalUnits(self.unit_type, [token] + self.text, [self_info] + self.self_info)
    
    def add_to_tail(self, token, self_info):
        return LexicalUnits(self.unit_type, self.text + [token], self.self_info + [self_info])

class SelectiveContext:

    def __init__(self, model_type = 'gpt2', lang = 'en'):

        self.model_type = model_type
        self.lang = lang

        # this means we calculate self-information sentence by sentence
        self.sent_level_self_info = True

        self._prepare_phrase_tokenizer()
        self.sent_tokenize_pattern = r"(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?)\s"

View source record →