INNER CODE UNIT · Python

load_model

graviraja/100-Days-of-NLP · applications/classification/toxic_comment_classification/app/app.py:15

def load_model():
    model.load_state_dict(torch.load(os.path.join("model", 'model.pt'), map_location=torch.device('cpu')))

bpe_model = youtokentome.BPE(model=os.path.join("model", "bpe.model"))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def bpe_tokenizer(sentence):
    encoded_ids = bpe_model.encode(sentence.lower(), output_type=youtokentome.OutputType.ID)
    return encoded_ids


def preprocess(text):
    # -- Converting to lower case
    text = text.lower()
    
    # replacing english abbreviations with full forms
    text = re.sub(r"what's", "what is ", text)
    text = re.sub(r"\'s", " ", text)

View source record →