INNER CODE UNIT · Rust

tokenizer

bminixhofer/nlprule · python/src/lib.rs:306

    fn tokenizer(&self) -> &Tokenizer {
        &self.tokenizer
    }
}

#[pymethods]
impl PyTokenizer {
    #[text_signature = "(code, sentence_splitter=None)"]
    #[staticmethod]
    fn load(lang_code: &str) -> PyResult<Self> {
        let bytes = get_resource(lang_code, "tokenizer.bin.gz")?;

        let tokenizer: Tokenizer = bincode::deserialize_from(bytes)
            .map_err(|x| PyValueError::new_err(format!("{}", x)))?;
        Ok(PyTokenizer { tokenizer })
    }

    #[new]

View source record →