INNER CODE UNIT · Python

process_one_text

yxuansu/SimCTG · dialogue_generation/dataclass.py:68

    def process_one_text(self, text, res_token_list, res_token_id_list):
        text_list = text.strip('\n').strip('\t').split('\t')
        if len(text_list) <= 1:
            return
        text = ''
        for one_text in text_list:
            text += one_text + self.eos_token
        tokens = self.tokenizer.tokenize(text)[-self.max_len:]
        token_ids = self.tokenizer.encode(text, add_special_tokens=False)
        if len(token_ids) <= self.min_len:
            return
        token_ids = token_ids[-self.max_len:]
        assert len(token_ids) <= self.max_len
        res_token_list.append(tokens)
        res_token_id_list.append(token_ids)
        return

    def pad_batch(self, batch_id_list):

View source record →