Skip to main content

ferrum_models/
tokenizer.rs

1//! Tokenizer 占位实现
2
3use std::sync::Arc;
4
5use ferrum_types::Result;
6
7#[derive(Debug, Default)]
8pub struct TokenizerFactory;
9
10impl TokenizerFactory {
11    pub fn new() -> Self {
12        Self
13    }
14
15    pub async fn create(&self, _source: &std::path::Path) -> Result<TokenizerHandle> {
16        Ok(TokenizerHandle(Arc::new(PlaceholderTokenizer)))
17    }
18}
19
20#[derive(Clone, Debug)]
21pub struct TokenizerHandle(pub Arc<PlaceholderTokenizer>);
22
23#[derive(Debug, Default)]
24pub struct PlaceholderTokenizer;
25
26impl PlaceholderTokenizer {
27    pub fn vocab_size(&self) -> usize {
28        0
29    }
30}