cohere_rust/api/
tokenize.rs

1use serde::{Deserialize, Serialize};
2
3use super::GenerateModel;
4
5#[derive(Serialize, Debug)]
6pub struct TokenizeRequest<'input> {
7    /// The string to be tokenized
8    pub text: &'input str,
9    /// optional - The model to use for tokenization. Custom models can also be supplied with their full ID.
10    #[serde(skip_serializing_if = "Option::is_none")]
11    pub model: Option<GenerateModel>,
12}
13
14#[derive(Deserialize, Debug)]
15pub struct TokenizeResponse {
16    /// The tokens
17    pub tokens: Vec<u64>,
18    /// String representations of the tokens
19    pub token_strings: Vec<String>,
20}