INNER CODE UNIT · Python

start_idx

KodCode-AI/kodcode · pipeline/completion_open_model.py:198

        start_idx = i * BATCH_SIZE + last_checkpoint_idx
        end_idx = min((i + 1) * BATCH_SIZE + last_checkpoint_idx, len(processed_dataset))
        batch = processed_dataset[start_idx:end_idx]
        if args.engine == "together":
            batch = process_batch_with_api(batch)
        else:
            batch = process_batch(batch, llm, params, tokenizer)
        
        processed_dataset[start_idx:end_idx] = batch
        # Overwrite the same checkpoint file after serveral batches
        if i % CHECKPOINT_EVERY == 0:
            save_dataset(processed_dataset[:end_idx], checkpoint_file)
            print(f"Dataset checkpoint saved after batch {i + 1}.")

    return processed_dataset

# Main function to control workflow
def main():

View source record →