Struct TokenizerApi 

pub struct TokenizerApi { /* private fields */ }

Tokenizer API client.

Implementations

impl TokenizerApi

pub async fn tokenize( &self, request: TokenizeRequest, ) -> Result<TokenizeResponse>

Tokenize text using a specific model.

Example

use xai_rust::{XaiClient, TokenizeRequest};

let client = XaiClient::from_env()?;

let request = TokenizeRequest::new("grok-4", "Hello, world!");
let response = client.tokenizer().tokenize(request).await?;

println!("Token count: {}", response.count());
println!("Tokens: {:?}", response.tokens);

pub async fn tokenize_text( &self, model: impl Into<String>, text: impl Into<String>, ) -> Result<TokenizeResponse>

Tokenize text using a model name and text string.

Example

use xai_rust::XaiClient;

let client = XaiClient::from_env()?;

let response = client.tokenizer()
    .tokenize_text("grok-4", "Hello, world!")
    .await?;

println!("Tokens: {:?}", response.tokens);

pub async fn count_tokens( &self, model: impl Into<String>, text: impl Into<String>, ) -> Result<usize>

Count tokens in text using a specific model.

Example

use xai_rust::XaiClient;

let client = XaiClient::from_env()?;

let count = client.tokenizer()
    .count_tokens("grok-4", "Hello, world!")
    .await?;

println!("Token count: {}", count);

Trait Implementations

impl Clone for TokenizerApi

fn clone(&self) -> TokenizerApi

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenizerApi

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.