INNER CODE UNIT · Python
SSTDataset
barissayil/SentimentAnalysis · dataset.py:6
class SSTDataset(Dataset):
"""
Stanford Sentiment Treebank V1.0
Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank
Richard Socher, Alex Perelygin, Jean Wu, Jason Chuang, Christopher Manning, Andrew Ng and Christopher Potts
Conference on Empirical Methods in Natural Language Processing (EMNLP 2013)
"""
def __init__(self, filename, maxlen, tokenizer):
# Store the contents of the file in pandas dataframe.
self.df = pd.read_csv(filename, delimiter="\t")
# Initialize tokenizer for the desired transformer model.
self.tokenizer = tokenizer
# Maximum length of tokens list to keep all the sequences of fixed size.
self.maxlen = maxlen
def __len__(self):
# Return length of dataframe.