INNER CODE UNIT · Python
k
KodCode-AI/kodcode · pipeline/step1.3_process_and_sanitize.py:169
k = search_space_size + 1 # +1 to include the instruction itself
similar_indices = []
similar_scores = []
for i in tqdm(range(0, len(embeddings), batch_size), desc="Searching Batches"):
end = min(i + batch_size, len(embeddings))
batch = embeddings[i:end]
scores, indices = faiss_index.search(batch, k)
similar_indices.append(indices)
similar_scores.append(scores)
similar_indices = np.vstack(similar_indices)
similar_scores = np.vstack(similar_scores)
################
# Step 5 - Apply Threshold and Update Dataset
################
print("Applying similarity threshold and updating dataset...")