xai_rust/models/
tokenizer.rs

1//! Tokenizer API types.
2
3use serde::{Deserialize, Serialize};
4
5/// Request to tokenize text.
6#[derive(Debug, Clone, Serialize)]
7pub struct TokenizeRequest {
8    /// The model to use for tokenization.
9    pub model: String,
10    /// The text to tokenize.
11    pub text: String,
12}
13
14impl TokenizeRequest {
15    /// Create a new tokenize request.
16    pub fn new(model: impl Into<String>, text: impl Into<String>) -> Self {
17        Self {
18            model: model.into(),
19            text: text.into(),
20        }
21    }
22}
23
24/// Response from tokenizing text.
25#[derive(Debug, Clone, Deserialize)]
26pub struct TokenizeResponse {
27    /// The token IDs.
28    pub tokens: Vec<u32>,
29    /// Number of tokens.
30    #[serde(default)]
31    pub token_count: u32,
32}
33
34impl TokenizeResponse {
35    /// Get the number of tokens.
36    pub fn count(&self) -> usize {
37        if self.token_count > 0 {
38            self.token_count as usize
39        } else {
40            self.tokens.len()
41        }
42    }
43}
xai_rust/models/tokenizer.rs

xai_rust/models/
tokenizer.rs
