INNER CODE UNIT · Python

process_one_file

yxuansu/SimCTG · dialogue_generation/dataclass.py:52

    def process_one_file(self, path):
        print ('Processing {}'.format(path))
        res_token_list, res_token_id_list = [], []
        with open(path, 'r', encoding = 'utf8') as i:
            lines = i.readlines()
        n = len(lines)
        p = progressbar.ProgressBar(n)
        p.start()
        for i in range(n):
            p.update(i)
            text = lines[i].strip('\n')
            self.process_one_text(text, res_token_list, res_token_id_list)
        p.finish()
        print ('{} processed!'.format(path))
        return res_token_list, res_token_id_list

    def process_one_text(self, text, res_token_list, res_token_id_list):
        text_list = text.strip('\n').strip('\t').split('\t')

View source record →