INNER CODE UNIT · Python
total_length
CogStack/OpenGPT · opengpt/dataset_utils.py:227
total_length = (total_length // block_size) * block_size
# Split by chunks of max_len.
result = {
k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
for k, t in concatenated_examples.items()
}
else:
# Do nothing
result = examples
return result