INNER CODE UNIT · Python

__init__

IBM/transition-amr-parser · src/ibm_neural_aligner/main.py:346

    def __init__(self, ids_to_tokens):
        assert isinstance(ids_to_tokens, list)
        assert len(set(ids_to_tokens)) == len(ids_to_tokens)
        self.ids_to_tokens = ids_to_tokens
        self.vocab = {tok: idx for idx, tok in enumerate(ids_to_tokens)}
        print(f'text tokenizer w/ size = {len(self.vocab)}')

    def tokenize(self, tokens):
        input_ids = [self.vocab[tok] for tok in tokens]
        return {'original_input': tokens, 'input_ids': input_ids}


class AMRTokenizer(object):
    r"""
    Constructs a tokenizer for AMR objects.

    Args:
        ids_to_tokens (:obj:`List[str]`):

View source record →