INNER CODE UNIT · Python
parse_args
Guitaricet/relora · pretokenize.py:22
def parse_args(args=None):
parser = argparse.ArgumentParser()
parser.add_argument("--tokenizer", type=str, required=True, help="HuggingFace tokenizer name")
parser.add_argument("--dataset", type=str, required=True, help="HuggingFace dataset name. E.g., wikitext")
parser.add_argument("--dataset_config", type=str, default=None, help="HuggingFace dataset config name. E.g., wikitext-2-v1")
parser.add_argument("--text_field", type=str, default="text", help="Name of the text field in the dataset")
parser.add_argument("--sequence_length", type=int, default=2048, help="Sequence length")
parser.add_argument("--num_cpu", type=int, default=multiprocessing.cpu_count(), help="Number of CPU cores")
parser.add_argument("--save_dir", type=str, required=True, help="Directory to save the pre-tokenized dataset")
parser.add_argument("--take", type=int, default=None, help="Number of examples to take from the dataset")
args = parser.parse_args(args)
return args
def main(args):
print("In main")