INNER CODE UNIT · Python
word
graviraja/100-Days-of-NLP · applications/classification/ner_tagging/app/app.py:92
word = word + [chars_vocab.PAD] * pad_length
# convert the chars into numerical format
char_ids = []
for each_char in word:
char_ids.append(chars_vocab[each_char])
char_seq.append(char_ids)
# numericalize
token_ids = [words_vocab[tok] for tok in tokens]
# seq length
sent_length = [len(token_ids)]
# create tensors
sent_tensor = torch.LongTensor(token_ids)
sent_tensor = sent_tensor.unsqueeze(0)
# sent_tensor => [1, seq_len]