kernelx_core/capabilities/
tokenize.rs

1use crate::{capabilities::HasCapability, models::ModelConfig, Result};
2use async_trait::async_trait;
3
4#[async_trait]
5pub trait Tokenize: HasCapability {
6    async fn tokenize(&self, texts: Vec<&str>) -> Result<Vec<usize>> {
7        self.tokenize_impl(self.model_id(), texts, self.config())
8            .await
9    }
10
11    #[doc(hidden)]
12    async fn tokenize_impl(
13        &self,
14        model: &str,
15        texts: Vec<&str>,
16        config: &ModelConfig,
17    ) -> Result<Vec<usize>>;
18}