ferrum-models 0.7.2

Model architectures (LLaMA, Qwen, BERT) for Ferrum inference
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

//! Tokenizer 占位实现

use std::sync::Arc;

use ferrum_types::Result;

#[derive(Debug, Default)]
pub struct TokenizerFactory;

impl TokenizerFactory {
    pub fn new() -> Self {
        Self
    }

    pub async fn create(&self, _source: &std::path::Path) -> Result<TokenizerHandle> {
        Ok(TokenizerHandle(Arc::new(PlaceholderTokenizer)))
    }
}

#[derive(Clone, Debug)]
pub struct TokenizerHandle(pub Arc<PlaceholderTokenizer>);

#[derive(Debug, Default)]
pub struct PlaceholderTokenizer;

impl PlaceholderTokenizer {
    pub fn vocab_size(&self) -> usize {
        0
    }
}