INNER CODE UNIT · Python
create_dataset_no_input
CogStack/OpenGPT · opengpt/dataset_utils.py:45
def create_dataset_no_input(config):
r''' This does not require an input dataset to generate a new dataset, only a prompt is needed
'''
prompt_db = json.load(open(config.path.prompt_db, 'rb'))
raw_data_columns = ['id', 'raw_output', 'prompt_hash']
raw_data = pd.DataFrame(None, columns=raw_data_columns)
raw_data_path = os.path.join(config.base_path, config.name, f"raw_generated_data_for_{config.name}.csv")
if os.path.exists(raw_data_path):
raw_data = pd.read_csv(raw_data_path)
logging.warning(f"Loading an existing openai generated dataset found at: {raw_data_path}" +
f"There are already {len(raw_data)} rows in the that dataset, the generation will continue from where last left off. " +
f"The script will also do all examples that were not done in the previous run.")
teacher = getattr(teachers, f'ask_{config.teacher.name}')
for prompt_config in config.prompts:
prompts = [prompt for prompt in prompt_db if prompt['hash'] in prompt_config['hashes']] # There must be one