INNER CODE UNIT · Rust

Some

deepjavalibrary/djl · extensions/tokenizers/rust/src/lib.rs:885

    if let Some(truncation_params) = tokenizer.get_truncation_mut() {
        truncation_params.strategy = res_strategy.unwrap();
        truncation_params.stride = truncation_stride as usize;
        truncation_params.max_length = truncation_max_length as usize;
    } else {
        let truncation_params = TruncationParams {
            strategy: res_strategy.unwrap(),
            stride: truncation_stride as usize,
            max_length: truncation_max_length as usize,
            ..Default::default()
        };
        let _ = tokenizer.with_truncation(Some(truncation_params));
    }
}

#[no_mangle]
pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_disableTruncation(
    _: JNIEnv,

View source record →