INNER CODE UNIT · Python
tokenize
IBM/transition-amr-parser · src/ibm_neural_aligner/main.py:428
def tokenize(self, amr):
tokens, token_types, node_ids = AMRTokenizer.get_linearized_parse(amr)
input_ids = [self.vocab[tok] for tok in tokens]
tokenizer_output = {}
tokenizer_output['original_input'] = tokens
tokenizer_output['input_ids'] = input_ids
tokenizer_output['token_type_ids'] = token_types
tokenizer_output['node_mask'] = [False if x < 0 else True for x in token_types]
tokenizer_output['node_ids'] = node_ids
return tokenizer_output
class Dataset(object):
def __init__(self, corpus, text_tokenizer=None, amr_tokenizer=None):
self.corpus = corpus
self.text_tokenizer = text_tokenizer
self.amr_tokenizer = amr_tokenizer
self.cached = {}