INNER CODE UNIT · Python
num_batches
KodCode-AI/kodcode · pipeline/completion_open_model.py:188
num_batches = (len(processed_dataset) - last_checkpoint_idx + BATCH_SIZE - 1) // BATCH_SIZE
print(f"Remaining number of batches: {num_batches}")
else:
last_checkpoint_idx = 0
# Calculate total number of batches
num_batches = (len(processed_dataset) + BATCH_SIZE - 1) // BATCH_SIZE
print(f"Total number of batches: {num_batches}")
for i in tqdm(range(num_batches)):
start_idx = i * BATCH_SIZE + last_checkpoint_idx
end_idx = min((i + 1) * BATCH_SIZE + last_checkpoint_idx, len(processed_dataset))
batch = processed_dataset[start_idx:end_idx]
if args.engine == "together":
batch = process_batch_with_api(batch)
else:
batch = process_batch(batch, llm, params, tokenizer)