INNER CODE UNIT · Python

bpe_tokenizer

graviraja/100-Days-of-NLP · applications/generation/utterance_generation/app/app.py:19

def bpe_tokenizer(sentence):
    encoded_ids = bpe_model.encode(sentence.lower(), output_type=youtokentome.OutputType.ID, bos=True, eos=True)
    return encoded_ids


def generate_utterance_greedy(sentence, bpe_model, model, device, max_len=50):
    model.eval()

    if isinstance(sentence, str):
        tokens = bpe_tokenizer(sentence)
    else:
        tokens = [int(token) for token in sentence]

    src_indexes = tokens
 
    # convert to tensor format
    # since the inference is done on single sentence, batch size is 1
    src_tensor = torch.LongTensor(src_indexes).unsqueeze(0).to(device)

View source record →