Trait tokenizers::utils::parallelism::MaybeParallelSlice

pub trait MaybeParallelSlice<'data, T>
where
    T: Sync,
{
    // Required methods
    fn maybe_par_chunks(
        &self,
        chunk_size: usize
    ) -> CondIterator<Chunks<'_, T>, Chunks<'_, T>>;
    fn maybe_par_chunks_cond(
        &self,
        cond: bool,
        chunk_size: usize
    ) -> CondIterator<Chunks<'_, T>, Chunks<'_, T>>;
}

Allows to convert into chunks that can be executed either parallelly or serially.

Required Methods

fn maybe_par_chunks( &self, chunk_size: usize ) -> CondIterator<Chunks<'_, T>, Chunks<'_, T>>

Create a CondIterator, that will be executed either in parallel or serially, based solely on the TOKENIZERS_PARALLELISM environment variable

fn maybe_par_chunks_cond( &self, cond: bool, chunk_size: usize ) -> CondIterator<Chunks<'_, T>, Chunks<'_, T>>

Create a CondIterator, that will be executed either in parallel or serially, based on both the TOKENIZERS_PARALLELISM environment variable and the provided bool. Both must be true to run with parallelism activated.

Implementations on Foreign Types

impl<T> MaybeParallelSlice<'_, T> for [T]

where T: Sync,

fn maybe_par_chunks( &self, chunk_size: usize ) -> CondIterator<Chunks<'_, T>, Chunks<'_, T>>

fn maybe_par_chunks_cond( &self, cond: bool, chunk_size: usize ) -> CondIterator<Chunks<'_, T>, Chunks<'_, T>>

Implementors