INNER CODE UNIT · Python
__getitem__
yya518/FinBERT · archive/datasets.py:24
def __getitem__(self,index):
tokenized_review = self.tokenizer.tokenize(self.x_y_list[0][index])
if len(tokenized_review) > self.max_seq_length:
tokenized_review = tokenized_review[:self.max_seq_length]
ids_review = self.tokenizer.convert_tokens_to_ids(tokenized_review)
mask_input = [1]*len(ids_review)
padding = [0] * (self.max_seq_length - len(ids_review))
ids_review += padding
mask_input += padding
input_type = [0]*self.max_seq_length
assert len(ids_review) == self.max_seq_length
assert len(mask_input) == self.max_seq_length