INNER CODE UNIT · Python
tokenize
IBM/transition-amr-parser · src/ibm_neural_aligner/main.py:353
def tokenize(self, tokens):
input_ids = [self.vocab[tok] for tok in tokens]
return {'original_input': tokens, 'input_ids': input_ids}
class AMRTokenizer(object):
r"""
Constructs a tokenizer for AMR objects.
Args:
ids_to_tokens (:obj:`List[str]`):
List of tokens.
"""
def __init__(self, ids_to_tokens):
assert isinstance(ids_to_tokens, list)
assert len(set(ids_to_tokens)) == len(ids_to_tokens)
self.ids_to_tokens = ids_to_tokens