aleph_alpha_api/
tokenization.rs

1use serde::{Deserialize, Serialize};
2
3#[derive(Serialize)]
4pub struct TokenizationRequest {
5    /// Name of the model tasked with completing the prompt. E.g. `luminous-base`.
6    pub model: String,
7    /// String to tokenize.
8    pub prompt: String,
9    /// Set this value to `true` to return text-tokens.
10    pub tokens: bool,
11    /// Set this value to `true to return numeric token-ids.
12    pub token_ids: bool,
13}
14
15#[derive(Deserialize)]
16pub struct TokenizationResponse {
17    pub tokens: Option<Vec<String>>,
18    pub token_ids: Option<Vec<u32>>,
19}
20
21#[derive(Serialize, Debug)]
22pub struct DetokenizationRequest {
23    /// Name of the model tasked with completing the prompt. E.g. `luminous-base"`.
24    pub model: String,
25    /// List of ids to detokenize.
26    pub token_ids: Vec<u32>,
27}
28
29#[derive(Deserialize, Debug)]
30pub struct DetokenizationResponse {
31    pub result: String,
32}
aleph_alpha_api/tokenization.rs

aleph_alpha_api/
tokenization.rs
