INNER CODE UNIT · Rust
tagger
bminixhofer/nlprule · python/src/lib.rs:340
fn tagger(&self) -> PyTagger {
self.tokenizer.tagger().clone().into()
}
/// Applies the full tokenization pipeline to the given text.
/// This includes POS tagging, lemmatization, chunking and sentencization.
///
/// Arguments:
/// text_or_texts (Union[str, List[str]]): The text(s) to tokenize.
///
/// Returns:
/// tokens (Union[List[List[Token]], List[List[List[Token]]]]):
/// The analyzed tokens. A list of lists of tokens. The outer list corresponds to a sentence. Batched if the input is batched.
#[text_signature = "(text_or_texts)"]
fn pipe(&self, py: Python, text_or_texts: PyObject) -> PyResult<PyObject> {
text_guard(py, text_or_texts, |text| {
let tokens = self
.tokenizer