INNER CODE UNIT · Python
get_args
KodCode-AI/kodcode · pipeline/step1.3_process_and_sanitize.py:17
def get_args():
parser = argparse.ArgumentParser(description="Instruction Extraction and Sanitization Manager.")
# Input Parameters
parser.add_argument("--input_file", type=str, required=True, help="Path to the input JSONL file.")
parser.add_argument("--language", type=str, default="auto", help="Language of the code.")
# Similarity Parameters
parser.add_argument("--sentence_model", type=str, default="sentence-transformers/all-mpnet-base-v2", help="SentenceTransformer model for encoding instructions.")
parser.add_argument("--encoding_batch_size", type=int, default=512, help="Batch size for encoding sentences.")
parser.add_argument("--distance_threshold", type=float, default=0.1, help="Cosine similarity threshold for filtering similar instructions.")
parser.add_argument("--search_space_size", type=int, default=5000, help="Number of nearest neighbors to search for similarity.")
parser.add_argument("--search_batch_size", type=int, default=4096, help="Batch size for searching similarity.")
# Device Configuration
parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", help="Device to run the model on ('cuda' or 'cpu').")
return parser.parse_args()