INNER CODE UNIT · Python
preprocess_function
microsoft/muzic · musecoco/1-text2attribute_model/main.py:330
def preprocess_function(examples, attributes):
# Tokenize the texts
result = tokenizer(examples['text'],
padding=padding,
max_length=max_seq_length,
truncation=True)
if 'labels' in examples:
for idx in range(len(examples['labels'])):
att_value = OrderedDict()
for order, att in enumerate(attributes):
att_value[att] = examples['labels'][idx][order].index(1)
examples['labels'][idx] = deepcopy(att_value)
result['labels']= examples['labels']
return result
with training_args.main_process_first(desc="dataset map pre-processing"):
raw_datasets = raw_datasets.map(
partial(preprocess_function, attributes=attributes),