INNER CODE UNIT · Python

process_batch

Koziev/NLP_Datasets · Conversations/Data/tinkoff_model_dialogues_scoring.py:16

def process_batch(dialogues):
    tx = []
    input_2_idialog = []
    idialog2result = collections.defaultdict(list)

    for idialog, lines in enumerate(dialogues):
        for i, reply in enumerate(lines):
            if i > 0:
                context = lines[:i]
                reply = lines[i]
                t = '[CLS]{}[RESPONSE_TOKEN]{}'.format('[SEP]'.join(context), reply)
                tx.append(t)
                input_2_idialog.append((idialog, i))
            else:
                idialog2result[idialog].append((reply, 1.0, 1.0))

    inputs = tokenizer(tx, max_length=512, truncation=True, padding=True, add_special_tokens=False, return_tensors='pt').to(device)

View source record →