INNER CODE UNIT · Rust

load

bminixhofer/nlprule · python/src/lib.rs:315

    fn load(lang_code: &str) -> PyResult<Self> {
        let bytes = get_resource(lang_code, "tokenizer.bin.gz")?;

        let tokenizer: Tokenizer = bincode::deserialize_from(bytes)
            .map_err(|x| PyValueError::new_err(format!("{}", x)))?;
        Ok(PyTokenizer { tokenizer })
    }

    #[new]
    fn new(path: Option<&str>) -> PyResult<Self> {
        let tokenizer = if let Some(path) = path {
            Tokenizer::new(path)
                .map_err(|x| PyValueError::new_err(format!("error creating Tokenizer: {}", x)))?
        } else {
            Tokenizer::default()
        };

        Ok(PyTokenizer { tokenizer })

View source record →