INNER CODE UNIT · Python
TextTokenizer
IBM/transition-amr-parser · src/ibm_neural_aligner/main.py:338
class TextTokenizer(object):
r"""
Constructs a tokenizer for text sentences.
Args:
ids_to_tokens (:obj:`List[str]`):
List of tokens.
"""
def __init__(self, ids_to_tokens):
assert isinstance(ids_to_tokens, list)
assert len(set(ids_to_tokens)) == len(ids_to_tokens)
self.ids_to_tokens = ids_to_tokens
self.vocab = {tok: idx for idx, tok in enumerate(ids_to_tokens)}
print(f'text tokenizer w/ size = {len(self.vocab)}')
def tokenize(self, tokens):
input_ids = [self.vocab[tok] for tok in tokens]
return {'original_input': tokens, 'input_ids': input_ids}