INNER CODE UNIT · Python
prompt_text
CogStack/OpenGPT · opengpt/dataset_utils.py:75
prompt_text = prompt_text_template.format(**parameters)
try:
out = teacher(prompt_text, config)
new_data = pd.DataFrame([[len(raw_data), out, prompt['hash']]], columns=raw_data_columns)
raw_data = pd.concat([raw_data, new_data], ignore_index=True)
if len(raw_data) % config.data_generation_checkpoint_every == 0:
logging.warning("Checkpointing the generated dataset.")
raw_data.to_csv(raw_data_path, index=False)
except Exception as e:
logging.exception(e)
logging.warning(f"Skipping example for prompt: {prompt['hash']}\n")
if raw_data is not None and len(raw_data) > 0:
raw_data.to_csv(raw_data_path, index=False)
return raw_data