INNER CODE UNIT · Python
batch_tgt_input_tensor
yxuansu/SimCTG · dialogue_generation/dataclass.py:95
batch_tgt_input_tensor = batch_tgt_tensor[:, :-1].clone()
batch_tgt_output_tensor = batch_tgt_tensor[:, 1:].clone()
return batch_tgt_input_tensor, batch_tgt_output_tensor
def parse_batch(self, batch_id_list):
batch_input, batch_labels = self.process_output(batch_id_list)
batch_labels[batch_labels[:, :] == self.pad_token_id] = -100
return batch_input, batch_labels
def get_next_train_batch(self, batch_size):
batch_idx_list = random.sample(self.train_idx_list, batch_size)
batch_id_list, batch_token_list = [], []
for idx in batch_idx_list:
batch_id_list.append(self.train_token_id_list[idx])
batch_token_list.append(self.train_token_list[idx])
batch_input_tensor, batch_labels = self.parse_batch(batch_id_list)
return batch_input_tensor, batch_labels, batch_token_list