INNER CODE UNIT · Python
new_text
CogStack/OpenGPT · opengpt/dataset_utils.py:33
new_text = tokenizer.decode(tokens[i*max_len:(i+1)*max_len])
new_data_row = [row[c] if c != 'text' else new_text for c in cols]
new_data_row.append(len(tokens[i*max_len:(i+1)*max_len]))
new_data_row.append(f'part_{i}')
new_data.append(new_data_row)
# Save
new_df = pd.DataFrame(new_data[1:], columns=new_data[0])
new_df.to_csv(os.path.join(base_path, name, 'data_split_by_length.csv'), index=False)
logging.warning(f'{dataset["name"]}: length before vs after: {len(df)} vs {len(new_df)}\n')
def create_dataset_no_input(config):
r''' This does not require an input dataset to generate a new dataset, only a prompt is needed
'''
prompt_db = json.load(open(config.path.prompt_db, 'rb'))
raw_data_columns = ['id', 'raw_output', 'prompt_hash']
raw_data = pd.DataFrame(None, columns=raw_data_columns)