INNER CODE UNIT · Python
new_example
CogStack/OpenGPT · opengpt/dataset_utils.py:219
new_example = {k:new_example[k] + v for k,v in example.items()}
# Add the last example if there is something to add
if len(new_example[_key]) > 0:
result = {k:result[k] + [v] for k,v in new_example.items()}
elif packing_type == 'full':
# Full packing
concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
total_length = len(concatenated_examples[list(examples.keys())[0]])
total_length = (total_length // block_size) * block_size
# Split by chunks of max_len.
result = {
k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
for k, t in concatenated_examples.items()
}
else:
# Do nothing
result = examples
return result