Struct Tokenizer

pub struct Tokenizer { /* private fields */ }

Implementations

impl Tokenizer

pub fn build(special_tokens: Vec<String>) -> Self

pub fn tokenize(&self, raw_str: &str) -> Vec<String>

Trait Implementations

impl Clone for Tokenizer

fn clone(&self) -> Tokenizer

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Tokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl HasPyGilRef for Tokenizer

type AsRefTarget = PyCell<Tokenizer>

Utility type to make Py::as_ref work.

impl IntoPy<Py<PyAny>> for Tokenizer

fn into_py(self, py: Python<'_>) -> PyObject

Performs the conversion.

impl PyClass for Tokenizer

type Frozen = False

Whether the pyclass is frozen.

impl PyClassImpl for Tokenizer

const IS_BASETYPE: bool = false

#[pyclass(subclass)]

const IS_SUBCLASS: bool = false

#[pyclass(extends=…)]

const IS_MAPPING: bool = false

#[pyclass(mapping)]

const IS_SEQUENCE: bool = false

#[pyclass(sequence)]

type BaseType = PyAny

Base class

type ThreadChecker = SendablePyClass<Tokenizer>

This handles following two situations:

type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::MutableChild

Immutable or mutable

type Dict = PyClassDummySlot

Specify this class has #[pyclass(dict)] or not.

type WeakRef = PyClassDummySlot

Specify this class has #[pyclass(weakref)] or not.

type BaseNativeType = PyAny

The closest native ancestor. This is PyAny by default, and when you declare #[pyclass(extends=PyDict)], it’s PyDict.

fn items_iter() -> PyClassItemsIter

fn doc(py: Python<'_>) -> PyResult<&'static CStr>

Rendered class doc

fn lazy_type_object() -> &'static LazyTypeObject<Self>

fn dict_offset() -> Option<isize>

fn weaklist_offset() -> Option<isize>

impl PyClassNewTextSignature<Tokenizer> for PyClassImplCollector<Tokenizer>

fn new_text_signature(self) -> Option<&'static str>

impl<'a, 'py> PyFunctionArgument<'a, 'py> for &'a Tokenizer

type Holder = Option<PyRef<'py, Tokenizer>>

fn extract( obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder, ) -> PyResult<Self>

impl<'a, 'py> PyFunctionArgument<'a, 'py> for &'a mut Tokenizer

type Holder = Option<PyRefMut<'py, Tokenizer>>

fn extract( obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder, ) -> PyResult<Self>

impl PyMethods<Tokenizer> for PyClassImplCollector<Tokenizer>

fn py_methods(self) -> &'static PyClassItems

impl PyTypeInfo for Tokenizer

const NAME: &'static str = "Tokenizer"

Class name.

const MODULE: Option<&'static str> = ::core::option::Option::None

Module name, if any.

fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject

Returns the PyTypeObject instance for this type.

fn type_object(py: Python<'_>) -> &PyType

👎Deprecated since 0.21.0: PyTypeInfo::type_object will be replaced by PyTypeInfo::type_object_bound in a future PyO3 version

Returns the safe abstraction over the type object.

fn type_object_bound(py: Python<'_>) -> Bound<'_, PyType>

Returns the safe abstraction over the type object.

fn is_type_of(object: &PyAny) -> bool

👎Deprecated since 0.21.0: PyTypeInfo::is_type_of will be replaced by PyTypeInfo::is_type_of_bound in a future PyO3 version

Checks if object is an instance of this type or a subclass of this type.

fn is_type_of_bound(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type or a subclass of this type.

fn is_exact_type_of(object: &PyAny) -> bool

👎Deprecated since 0.21.0: PyTypeInfo::is_exact_type_of will be replaced by PyTypeInfo::is_exact_type_of_bound in a future PyO3 version

Checks if object is an instance of this type.

fn is_exact_type_of_bound(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type.

impl DerefToPyAny for Tokenizer