INNER CODE UNIT · Python
end
KodCode-AI/kodcode · pipeline/step1.3_process_and_sanitize.py:174
end = min(i + batch_size, len(embeddings))
batch = embeddings[i:end]
scores, indices = faiss_index.search(batch, k)
similar_indices.append(indices)
similar_scores.append(scores)
similar_indices = np.vstack(similar_indices)
similar_scores = np.vstack(similar_scores)
################
# Step 5 - Apply Threshold and Update Dataset
################
print("Applying similarity threshold and updating dataset...")
row_ids = [item.get("row_id", None) for item in dataset["metadata"]] if "metadata" in dataset.features else [None] * len(instructions)
min_distances = {}
duplicate_counts = {}
min_similar_row_ids = {}