INNER CODE UNIT · Python
__init__
yxuansu/SimCTG · dialogue_generation/dataclass.py:8
def __init__(self, model_name, train_path, dev_path, test_path, min_len, max_len, eos_token, pad_token):
'''
model_name: gpt2
train_path: training data path
dev_path: validation data path
test_path: test data path
min_len: minimum length for training sequences
max_len: maximum length for training sequences
'''
#from transformers import GPT2TokenizerFast
from transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
self.min_len, self.max_len = min_len, max_len
if eos_token == 'endoftext':
print ('Use English GPT')
self.eos_token = self.tokenizer.eos_token
else:
self.eos_token = eos_token