INNER CODE UNIT · Python

financialPhraseBankDataset

yya518/FinBERT · archive/datasets.py:66

def financialPhraseBankDataset(dir_):
    fb_path = os.path.join(dir_, 'FinancialPhraseBank-v1.0')
    data_50 = os.path.join(fb_path, 'Sentences_50Agree.txt')
    sent_50 = []
    rand_idx = 45
    
    with open(data_50, 'rb') as fi:
        for l in fi:
            l = l.decode('utf-8', 'replace')
            sent_50.append(l.strip())
    
    x_y_list_50 = [sent.split("@") for sent in sent_50]
    x50, y50 = transform_labels(x_y_list_50)
    
    data = [x50, y50]
    X_train, X_test, y_train, y_test = train_test_split(data[0], data[1], test_size=0.1, random_state=rand_idx, stratify=data[1])

    y_train = pd.get_dummies(y_train).values.tolist()

View source record →