wordchipper 0.9.2

HPC Rust LLM Tokenizer Library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

//! Token Decoder Options
//!
//! Options for building a [`TokenDecoder`].

use crate::{
    TokenDecoder,
    TokenType,
    UnifiedTokenVocab,
    alloc::sync::Arc,
    decoders::SlabIndexDecoder,
};

/// Options for configuring a [`TokenDecoder`].
// TODO: serialize/deserialize?
#[derive(Debug, Clone, Copy, PartialEq, Default)]
pub struct TokenDecoderOptions {
    /// Should the decoder be threaded?
    ///
    /// Enabling parallelism will request a threaded implementation.
    pub parallel: bool,
}

impl TokenDecoderOptions {
    /// Gets the configured parallelism value.
    ///
    /// Enabling parallelism will request a threaded implementation.
    pub fn parallel(&self) -> bool {
        self.parallel
    }

    /// Sets the configured parallelism value.
    ///
    /// Enabling parallelism will request a threaded implementation.
    pub fn set_parallel(
        &mut self,
        parallel: bool,
    ) {
        self.parallel = parallel;
    }

    /// Sets the configured parallelism value.
    ///
    /// Enabling parallelism will request a threaded implementation.
    pub fn with_parallel(
        mut self,
        parallel: bool,
    ) -> Self {
        self.set_parallel(parallel);
        self
    }

    /// Build a [`TokenDecoder`] for the given vocab.
    pub fn build<T: TokenType>(
        &self,
        vocab: Arc<UnifiedTokenVocab<T>>,
    ) -> Arc<dyn TokenDecoder<T>> {
        #[allow(unused_mut)]
        let mut dec: Arc<dyn TokenDecoder<T>> = Arc::new(SlabIndexDecoder::from_vocab(vocab));

        #[cfg(feature = "parallel")]
        if self.parallel {
            use crate::support::concurrency::rayon::ParallelRayonDecoder;
            dec = Arc::new(ParallelRayonDecoder::new(dec));
        }

        dec
    }
}