INNER CODE UNIT · Python
main
Guitaricet/relora · pretokenize.py:38
def main(args):
print("In main")
logger.info("*" * 40)
logger.info(f"Starting script with the arguments")
for k, v in vars(args).items():
logger.info(f"{k:30} {v}")
logger.info("*" * 40)
_tokenizer_name_for_save = args.tokenizer.replace("/", "_")
save_path = os.path.join(args.save_dir, f"{args.dataset}_{_tokenizer_name_for_save}_{args.sequence_length}")
if args.dataset_config is not None:
save_path = os.path.join(args.save_dir, f"{args.dataset}_{args.dataset_config}_{_tokenizer_name_for_save}_{args.sequence_length}")
if os.path.exists(save_path):
raise ValueError(f"Path {save_path} already exists")
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)
logger.info(f"Loaidng the dataset in streaming mode: {args.take is not None}")