INNER CODE UNIT · Python
group_texts
HIT-SCIR/Chinese-Mixtral-8x7B · data/preprocess_datasets.py:15
def group_texts(examples):
# Concatenate all texts.
concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
total_length = len(concatenated_examples[list(examples.keys())[0]])
# We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
# customize this part to your needs.
if total_length >= sequence_length:
total_length = (total_length // sequence_length) * sequence_length
# Split by chunks of block_size.
result = {
k: [t[i: i + sequence_length] for i in range(0, total_length, sequence_length)]
for k, t in concatenated_examples.items()
}
result["labels"] = result["input_ids"].copy()
return result
num_workers = multiprocessing.cpu_count()