INNER CODE UNIT · Python
__init__
barissayil/SentimentAnalysis · dataset.py:14
def __init__(self, filename, maxlen, tokenizer):
# Store the contents of the file in pandas dataframe.
self.df = pd.read_csv(filename, delimiter="\t")
# Initialize tokenizer for the desired transformer model.
self.tokenizer = tokenizer
# Maximum length of tokens list to keep all the sequences of fixed size.
self.maxlen = maxlen
def __len__(self):
# Return length of dataframe.
return len(self.df)
def __getitem__(self, index):
# Select sentence and label at specified index from data frame.
sentence = self.df.loc[index, "sentence"]
label = self.df.loc[index, "label"]
# Preprocess text to be suitable for transformer
tokens = self.tokenizer.tokenize(sentence)