xai_openapi/
tokenize.rs

1//! Tokenization API types for `/v1/tokenize-text` endpoint.
2
3use serde::{Deserialize, Serialize};
4
5use crate::prelude::*;
6
7/// Request body for tokenization.
8#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
9pub struct TokenizeRequest {
10    /// The model to tokenize with.
11    #[serde(skip_serializing_if = "Option::is_none")]
12    pub model: Option<String>,
13
14    /// The text content to be tokenized.
15    #[serde(skip_serializing_if = "Option::is_none")]
16    pub text: Option<String>,
17
18    /// Optional user identifier.
19    #[serde(skip_serializing_if = "Option::is_none")]
20    pub user: Option<String>,
21}
22
23/// Response body for tokenization.
24#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
25pub struct TokenizeResponse {
26    /// A list of tokens.
27    pub token_ids: Vec<TokenizeResponseToken>,
28}
29
30/// A token from the tokenization response.
31#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
32pub struct TokenizeResponseToken {
33    /// The integer representation of the token for the model.
34    pub token_id: u32,
35
36    /// The string of the token.
37    pub string_token: String,
38
39    /// The bytes that constituted the token.
40    pub token_bytes: Vec<u32>,
41}
xai_openapi/tokenize.rs

xai_openapi/
tokenize.rs
