INNER CODE UNIT · Python

AMRTokenizer

IBM/transition-amr-parser · src/ibm_neural_aligner/main.py:358

class AMRTokenizer(object):
    r"""
    Constructs a tokenizer for AMR objects.

    Args:
        ids_to_tokens (:obj:`List[str]`):
            List of tokens.
    """

    def __init__(self, ids_to_tokens):
        assert isinstance(ids_to_tokens, list)
        assert len(set(ids_to_tokens)) == len(ids_to_tokens)
        self.ids_to_tokens = ids_to_tokens
        self.vocab = {tok: idx for idx, tok in enumerate(ids_to_tokens)}
        print(f'amr tokenizer w/ size = {len(self.vocab)}')

    @staticmethod
    def get_linearized_parse(amr):

View source record →