xai-openapi 0.1.1

Rust types for the xAI API (Grok models)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

//! Tokenization API types for `/v1/tokenize-text` endpoint.

use serde::{Deserialize, Serialize};

use crate::prelude::*;

/// Request body for tokenization.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct TokenizeRequest {
    /// The model to tokenize with.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub model: Option<String>,

    /// The text content to be tokenized.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub text: Option<String>,

    /// Optional user identifier.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub user: Option<String>,
}

/// Response body for tokenization.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct TokenizeResponse {
    /// A list of tokens.
    pub token_ids: Vec<TokenizeResponseToken>,
}

/// A token from the tokenization response.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct TokenizeResponseToken {
    /// The integer representation of the token for the model.
    pub token_id: u32,

    /// The string of the token.
    pub string_token: String,

    /// The bytes that constituted the token.
    pub token_bytes: Vec<u32>,
}