INNER CODE UNIT · Python
__init__
yya518/FinBERT · archive/datasets.py:11
def __init__(self, x_y_list, vocab_path, max_seq_length=256, vocab = 'base-cased', transform=None):
self.max_seq_length = max_seq_length
self.x_y_list = x_y_list
self.vocab = vocab
if self.vocab == 'base-cased':
self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False, do_basic_tokenize=True)
elif self.vocab == 'finance-cased':
self.tokenizer = BertTokenizer(vocab_file = vocab_path, do_lower_case = False, do_basic_tokenize = True)
elif self.vocab == 'base-uncased':
self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, do_basic_tokenize=True)
elif self.vocab == 'finance-uncased':
self.tokenizer = BertTokenizer(vocab_file = vocab_path, do_lower_case = True, do_basic_tokenize = True)
def __getitem__(self,index):
tokenized_review = self.tokenizer.tokenize(self.x_y_list[0][index])
if len(tokenized_review) > self.max_seq_length:
tokenized_review = tokenized_review[:self.max_seq_length]