INNER CODE UNIT · Python
create_dataset
CogStack/OpenGPT · opengpt/dataset_utils.py:95
def create_dataset(config):
prompt_db = json.load(open(config.path.prompt_db, 'rb'))
raw_data_columns = ['id', 'raw_output', 'dataset', 'language', 'run', 'prompt_hash', 'prompt_text_hash', 'context']
raw_data = pd.DataFrame(None, columns=raw_data_columns)
prepared_data = None
raw_data_path = os.path.join(config.base_path, config.name, f"raw_generated_data_for_{config.name}.csv")
prepared_data_path = os.path.join(config.base_path, config.name, f"prepared_generated_data_for_{config.name}.csv")
if os.path.exists(raw_data_path) and os.path.exists(prepared_data_path):
raw_data = pd.read_csv(raw_data_path)
prepared_data = pd.read_csv(prepared_data_path)
logging.warning(f"Loading an existing openai generated dataset found at: \n{raw_data_path}\n and\n{prepared_data_path}\n" +
f"There are already {len(raw_data)} rows in the that dataset, the generation will continue from where last left off. " +
f"The script will also do all examples that were not done in the previous run.\n" +
"***Take care that if prompt_config['random_prompt'] is set to true, it can produce unwanted results.\n\n")
cnt = 0
for prompt_config in config.prompts:
prompts = [prompt for prompt in prompt_db if prompt['hash'] in prompt_config['hashes']] # There must be one