llamaedge/params.rs
1//! Parameters for the chat completion API.
2
3#[cfg(feature = "audio")]
4use endpoints::audio::transcription::TimestampGranularity;
5use endpoints::chat::{ChatResponseFormat, Tool, ToolChoice};
6#[cfg(feature = "image")]
7use endpoints::{
8 files::FileObject,
9 images::{SamplingMethod, Scheduler},
10};
11#[cfg(feature = "image")]
12use std::path::PathBuf;
13
14#[cfg(feature = "image")]
15pub type ImageResponseFormat = endpoints::images::ResponseFormat;
16
17/// Parameters for the chat completion API.
18#[derive(Debug, Clone)]
19pub struct ChatParams {
20 /// The model to use for generating completions.
21 pub model: Option<String>,
22 /// Adjust the randomness of the generated text. Between 0.0 and 2.0. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.
23 ///
24 /// We generally recommend altering this or top_p but not both.
25 /// Defaults to 1.0.
26 pub temperature: Option<f64>,
27 /// Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P. The value should be between 0.0 and 1.0.
28 ///
29 /// Top-p sampling, also known as nucleus sampling, is another text generation method that selects the next token from a subset of tokens that together have a cumulative probability of at least p. This method provides a balance between diversity and quality by considering both the probabilities of tokens and the number of tokens to sample from. A higher value for top_p (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text.
30 ///
31 /// We generally recommend altering this or temperature but not both.
32 /// Defaults to 1.0.
33 pub top_p: Option<f64>,
34 /// How many chat completion choices to generate for each input message.
35 /// Defaults to 1.
36 pub n_choice: Option<u64>,
37 /// A list of tokens at which to stop generation. If None, no stop tokens are used. Up to 4 sequences where the API will stop generating further tokens.
38 /// Defaults to None
39 pub stop: Option<Vec<String>>,
40 /// **Deprecated** Use `max_completion_tokens` instead.
41 ///
42 /// The maximum number of tokens to generate. The value should be no less than 1.
43 /// Defaults to 1024.
44 pub max_tokens: Option<u64>,
45 /// An upper bound for the number of tokens that can be generated for a completion. Defaults to -1, which means no upper bound.
46 pub max_completion_tokens: Option<i32>,
47 /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.
48 /// Defaults to 0.0.
49 pub presence_penalty: Option<f64>,
50 /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.
51 /// Defaults to 0.0.
52 pub frequency_penalty: Option<f64>,
53 /// A unique identifier representing your end-user.
54 pub user: Option<String>,
55 /// Format that the model must output
56 pub response_format: Option<ChatResponseFormat>,
57 /// A list of tools the model may call.
58 ///
59 /// Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for.
60 pub tools: Option<Vec<Tool>>,
61 /// Controls which (if any) function is called by the model.
62 pub tool_choice: Option<ToolChoice>,
63}
64impl Default for ChatParams {
65 fn default() -> Self {
66 Self {
67 model: None,
68 temperature: Some(1.0),
69 top_p: Some(1.0),
70 n_choice: Some(1),
71 stop: None,
72 max_tokens: Some(1024),
73 max_completion_tokens: Some(-1),
74 presence_penalty: Some(0.0),
75 frequency_penalty: Some(0.0),
76 user: None,
77 response_format: None,
78 tools: None,
79 tool_choice: None,
80 }
81 }
82}
83
84/// Parameters for the RAG chat completion API.
85#[cfg(feature = "rag")]
86#[derive(Debug, Clone)]
87pub struct RagChatParams {
88 /// The model to use for generating completions.
89 pub model: Option<String>,
90 /// Adjust the randomness of the generated text. Between 0.0 and 2.0. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.
91 ///
92 /// We generally recommend altering this or top_p but not both.
93 /// Defaults to 0.8.
94 pub temperature: f64,
95 /// Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P. The value should be between 0.0 and 1.0.
96 ///
97 /// Top-p sampling, also known as nucleus sampling, is another text generation method that selects the next token from a subset of tokens that together have a cumulative probability of at least p. This method provides a balance between diversity and quality by considering both the probabilities of tokens and the number of tokens to sample from. A higher value for top_p (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text.
98 ///
99 /// We generally recommend altering this or temperature but not both.
100 /// Defaults to 0.9. To disable top-p sampling, set it to 1.0.
101 pub top_p: f64,
102 /// How many chat completion choices to generate for each input message.
103 /// Defaults to 1.
104 pub n_choice: u64,
105 /// A list of tokens at which to stop generation. If None, no stop tokens are used. Up to 4 sequences where the API will stop generating further tokens.
106 /// Defaults to None
107 pub stop: Option<Vec<String>>,
108 /// **Deprecated** Use `max_completion_tokens` instead.
109 ///
110 /// The maximum number of tokens to generate. The value should be no less than 1.
111 /// Defaults to 1024.
112 pub max_tokens: u64,
113 /// An upper bound for the number of tokens that can be generated for a completion. Defaults to -1, which means no upper bound.
114 pub max_completion_tokens: i32,
115 /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.
116 /// Defaults to 0.0.
117 pub presence_penalty: f64,
118 /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.
119 /// Defaults to 0.0.
120 pub frequency_penalty: f64,
121 /// A unique identifier representing your end-user.
122 pub user: Option<String>,
123 /// Format that the model must output
124 pub response_format: Option<ChatResponseFormat>,
125 /// A list of tools the model may call.
126 ///
127 /// Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for.
128 pub tools: Option<Vec<Tool>>,
129 /// Controls which (if any) function is called by the model.
130 pub tool_choice: Option<ToolChoice>,
131 /// Number of user messages to use for context retrieval. Defaults to 1.
132 pub context_window: u64,
133 /// The configuration for the VectorDB server.
134 pub vdb_config: Option<RagVdbConfig>,
135}
136#[cfg(feature = "rag")]
137impl Default for RagChatParams {
138 fn default() -> Self {
139 Self {
140 model: None,
141 temperature: 0.8,
142 top_p: 0.9,
143 n_choice: 1,
144 stop: None,
145 max_tokens: 1024,
146 max_completion_tokens: -1,
147 presence_penalty: 0.0,
148 frequency_penalty: 0.0,
149 user: None,
150 response_format: None,
151 tools: None,
152 tool_choice: None,
153 context_window: 1,
154 vdb_config: None,
155 }
156 }
157}
158
159/// The configuration for the VectorDB server.
160#[cfg(feature = "rag")]
161#[derive(Debug, Clone)]
162pub struct RagVdbConfig {
163 /// The URL of the VectorDB server.
164 pub server_url: String,
165 /// The names of the collections in VectorDB.
166 pub collection_name: Vec<String>,
167 /// Max number of retrieved results. The number of the values must be the same as the number of `collection_name`.
168 pub limit: Vec<u64>,
169 /// The score threshold for the retrieved results. The number of the values must be the same as the number of `collection_name`.
170 pub score_threshold: Vec<f32>,
171 /// The API key for the VectorDB server.
172 pub api_key: Option<String>,
173}
174
175/// Parameters for the transcription API.
176#[cfg(feature = "audio")]
177#[derive(Debug, Clone)]
178pub struct TranscriptionParams {
179 /// ID of the model to use.
180 pub model: Option<String>,
181 /// An optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language.
182 pub prompt: Option<String>,
183 /// The format of the transcript output, in one of these options: `json`, `text`, `srt`, `verbose_json`, or `vtt`.
184 pub response_format: String,
185 /// The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use [log probability](https://en.wikipedia.org/wiki/Log_probability) to automatically increase the temperature until certain thresholds are hit. Defaults to 0.0.
186 pub temperature: f64,
187 /// The timestamp granularities to populate for this transcription.
188 /// `response_format` must be set `verbose_json` to use timestamp granularities. Either or both of these options are supported: `word`, or `segment`.
189 pub timestamp_granularities: Option<Vec<TimestampGranularity>>,
190
191 /// Automatically detect the spoken language in the provided audio input. Defaults to false.
192 pub detect_language: bool,
193 /// Time offset in milliseconds. Defaults to 0.
194 pub offset_time: u64,
195 /// Length of audio (in seconds) to be processed starting from the point defined by the `offset_time` field (or from the beginning by default). Defaults to 0.
196 pub duration: u64,
197 /// Maximum amount of text context (in tokens) that the model uses when processing long audio inputs incrementally. Defaults to -1.
198 pub max_context: i32,
199 /// Maximum number of tokens that the model can generate in a single transcription segment (or chunk). Defaults to 0.
200 pub max_len: u64,
201 /// Split audio chunks on word rather than on token. Defaults to false.
202 pub split_on_word: bool,
203 /// Use the new computation context. Defaults to false.
204 pub use_new_context: bool,
205}
206#[cfg(feature = "audio")]
207impl Default for TranscriptionParams {
208 fn default() -> Self {
209 Self {
210 model: None,
211 prompt: None,
212 response_format: "json".to_string(),
213 temperature: 0.0,
214 timestamp_granularities: Some(vec![TimestampGranularity::Segment]),
215 detect_language: false,
216 offset_time: 0,
217 duration: 0,
218 max_context: -1,
219 max_len: 0,
220 split_on_word: false,
221 use_new_context: false,
222 }
223 }
224}
225
226/// Parameters for the translation API.
227#[cfg(feature = "audio")]
228#[derive(Debug, Clone)]
229pub struct TranslationParams {
230 /// ID of the model to use.
231 pub model: Option<String>,
232 /// An optional text to guide the model's style or continue a previous audio segment. The prompt should be in English.
233 pub prompt: Option<String>,
234 /// The format of the transcript output, in one of these options: `json`, `text`, `srt`, `verbose_json`, or `vtt`. Defaults to `json`.
235 pub response_format: String,
236 /// The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use [log probability](https://en.wikipedia.org/wiki/Log_probability) to automatically increase the temperature until certain thresholds are hit. Defaults to 0.0.
237 pub temperature: f64,
238 /// automatically detect the spoken language in the provided audio input. Defaults to false.
239 pub detect_language: bool,
240 /// Time offset in milliseconds. Defaults to 0.
241 pub offset_time: u64,
242 /// Length of audio (in seconds) to be processed starting from the point defined by the `offset_time` field (or from the beginning by default). Defaults to 0.
243 pub duration: u64,
244 /// Maximum amount of text context (in tokens) that the model uses when processing long audio inputs incrementally. Defaults to -1.
245 pub max_context: i32,
246 /// Maximum number of tokens that the model can generate in a single transcription segment (or chunk). Defaults to 0.
247 pub max_len: u64,
248 /// Split audio chunks on word rather than on token. Defaults to false.
249 pub split_on_word: bool,
250 /// Use the new computation context. Defaults to false.
251 pub use_new_context: bool,
252}
253#[cfg(feature = "audio")]
254impl Default for TranslationParams {
255 fn default() -> Self {
256 Self {
257 model: None,
258 prompt: None,
259 response_format: "json".to_string(),
260 temperature: 0.0,
261 detect_language: false,
262 offset_time: 0,
263 duration: 0,
264 max_context: -1,
265 max_len: 0,
266 split_on_word: false,
267 use_new_context: false,
268 }
269 }
270}
271
272/// Parameters for the embeddings API.
273#[derive(Debug, Clone)]
274pub struct EmbeddingsParams {
275 /// ID of the model to use.
276 pub model: Option<String>,
277 /// The format to return the embeddings in. Can be either float or base64.
278 /// Defaults to float.
279 pub encoding_format: String,
280 /// A unique identifier representing your end-user.
281 pub user: Option<String>,
282 /// The URL of the VectorDB server.
283 #[cfg(feature = "rag")]
284 pub vdb_server_url: Option<String>,
285 /// The name of the collection in VectorDB.
286 #[cfg(feature = "rag")]
287 pub vdb_collection_name: Option<String>,
288 /// The API key for the VectorDB server.
289 #[cfg(feature = "rag")]
290 pub vdb_api_key: Option<String>,
291}
292impl Default for EmbeddingsParams {
293 fn default() -> Self {
294 Self {
295 model: None,
296 encoding_format: "float".to_string(),
297 user: None,
298 #[cfg(feature = "rag")]
299 vdb_server_url: None,
300 #[cfg(feature = "rag")]
301 vdb_collection_name: None,
302 #[cfg(feature = "rag")]
303 vdb_api_key: None,
304 }
305 }
306}
307
308/// Parameters for the image generation API.
309#[cfg(feature = "image")]
310#[derive(Debug, Clone)]
311pub struct ImageCreateParams {
312 /// Negative prompt for the image generation.
313 pub negative_prompt: Option<String>,
314 /// Name of the model to use for image generation.
315 pub model: String,
316 /// Number of images to generate. Defaults to `1`.
317 pub n: u64,
318 /// The format in which the generated images are returned. Must be one of `url` or `b64_json`. Defaults to `Url`.
319 pub response_format: ImageResponseFormat,
320 /// A unique identifier representing your end-user, which can help monitor and detect abuse.
321 pub user: Option<String>,
322 /// Unconditional guidance scale. Defaults to `7.0`.
323 pub cfg_scale: f32,
324 /// Sampling method. Defaults to "euler_a".
325 pub sample_method: SamplingMethod,
326 /// Number of sample steps. Defaults to `20`.
327 pub steps: usize,
328 /// Image height, in pixel space. Defaults to `512`.
329 pub height: usize,
330 /// Image width, in pixel space. Defaults to `512`.
331 pub width: usize,
332 /// Strength to apply Control Net. Defaults to `0.9`.
333 pub control_strength: f32,
334 /// The image to control the generation.
335 pub control_image: Option<FileObject>,
336 /// RNG seed. Negative value means to use random seed. Defaults to `42`.
337 pub seed: i32,
338 /// Strength for noising/unnoising. Defaults to `0.75`.
339 pub strength: f32,
340 /// Denoiser sigma scheduler. Possible values are `discrete`, `karras`, `exponential`, `ays`, `gits`. Defaults to `discrete`.
341 pub scheduler: Scheduler,
342 /// Apply canny preprocessor. Defaults to `false`.
343 pub apply_canny_preprocessor: bool,
344 /// Strength for keeping input identity. Defaults to `0.2`.
345 pub style_ratio: f32,
346}
347#[cfg(feature = "image")]
348impl Default for ImageCreateParams {
349 fn default() -> Self {
350 Self {
351 negative_prompt: None,
352 model: "".to_string(),
353 n: 1,
354 response_format: ImageResponseFormat::Url,
355 user: None,
356 cfg_scale: 7.0,
357 sample_method: SamplingMethod::EulerA,
358 steps: 20,
359 height: 512,
360 width: 512,
361 control_strength: 0.9,
362 control_image: None,
363 seed: 42,
364 strength: 0.75,
365 scheduler: Scheduler::Discrete,
366 apply_canny_preprocessor: false,
367 style_ratio: 0.2,
368 }
369 }
370}
371
372/// Parameters for the image edit API.
373#[cfg(feature = "image")]
374#[derive(Debug, Clone)]
375pub struct ImageEditParams {
376 /// Negative prompt for the image generation.
377 pub negative_prompt: Option<String>,
378 /// An additional image whose fully transparent areas (e.g. where alpha is zero) indicate where `image` should be edited. Must have the same dimensions as the input image.
379 pub mask: Option<PathBuf>,
380 /// The model to use for image generation.
381 pub model: String,
382 /// The number of images to generate. Defaults to `1`.
383 pub n: u64,
384 /// The format in which the generated images are returned. Must be one of `url` or `b64_json`. Defaults to `url`.
385 pub response_format: ImageResponseFormat,
386 /// A unique identifier representing your end-user, which can help monitor and detect abuse.
387 pub user: Option<String>,
388 /// Unconditional guidance scale. Defaults to `7.0`.
389 pub cfg_scale: f32,
390 /// Sampling method. Defaults to "euler_a".
391 pub sample_method: SamplingMethod,
392 /// Number of sample steps. Defaults to `20`.
393 pub steps: usize,
394 /// Image height, in pixel space. Defaults to `512`.
395 pub height: usize,
396 /// Image width, in pixel space. Defaults to `512`.
397 pub width: usize,
398 /// strength to apply Control Net. Defaults to `0.9`.
399 pub control_strength: f32,
400 /// The image to control the generation.
401 pub control_image: Option<PathBuf>,
402 /// RNG seed. Negative value means to use random seed. Defaults to `42`.
403 pub seed: i32,
404 /// Strength for noising/unnoising. Defaults to `0.75`.
405 pub strength: f32,
406 /// Denoiser sigma scheduler. Possible values are `discrete`, `karras`, `exponential`, `ays`, `gits`. Defaults to `discrete`.
407 pub scheduler: Scheduler,
408 /// Apply canny preprocessor. Defaults to `false`.
409 pub apply_canny_preprocessor: bool,
410 /// Strength for keeping input identity. Defaults to `0.2`.
411 pub style_ratio: f32,
412}
413#[cfg(feature = "image")]
414impl Default for ImageEditParams {
415 fn default() -> Self {
416 Self {
417 negative_prompt: None,
418 mask: None,
419 model: "".to_string(),
420 n: 1,
421 response_format: ImageResponseFormat::Url,
422 user: None,
423 cfg_scale: 7.0,
424 sample_method: SamplingMethod::EulerA,
425 steps: 20,
426 height: 512,
427 width: 512,
428 control_strength: 0.9,
429 control_image: None,
430 seed: 42,
431 strength: 0.75,
432 scheduler: Scheduler::Discrete,
433 apply_canny_preprocessor: false,
434 style_ratio: 0.2,
435 }
436 }
437}