INNER CODE UNIT · Python
sanitize_instructions
KodCode-AI/kodcode · pipeline/step1.3_process_and_sanitize.py:132
def sanitize_instructions(input_file, output_file, sentence_model, encoding_batch_size, distance_threshold, search_space_size, search_batch_size, device):
"""
Sanitizes instructions by removing duplicates based on L2 distance.
"""
################
# Step 1 - Load and Prepare the Dataset
################
print(f"Loading dataset from {input_file}...")
dataset = load_dataset("json", data_files=input_file, split='train')
instructions = dataset["instruction"]
print(f"Number of instructions: {len(instructions)}")
################
# Step 2 - Encode Instructions
################
print("Loading SentenceTransformer model...")
model = SentenceTransformer(sentence_model)
model.to(device)