bitrouter_core/models/language/
call_options.rs

1//! Options for calling a language model, including prompt, generation parameters, tools, and provider-specific options
2
3use http::HeaderMap;
4use tokio_util::sync::CancellationToken;
5
6use crate::models::shared::{provider::ProviderOptions, types::JsonSchema};
7
8use super::{
9    prompt::LanguageModelPrompt, tool::LanguageModelTool, tool_choice::LanguageModelToolChoice,
10};
11
12/// Options for calling a language model
13#[derive(Debug, Clone)]
14pub struct LanguageModelCallOptions {
15    /// The prompt to send to the language model, which is a sequence of messages from the system, user, assistant, and tools
16    pub prompt: LanguageModelPrompt,
17    /// Whether to stream the response as it's generated, or return it all at once when complete
18    pub stream: Option<bool>,
19    /// The maximum number of tokens to generate in the response
20    pub max_output_tokens: Option<u32>,
21    /// Sampling temperature to use, between 0 and 1
22    pub temperature: Option<f32>,
23    /// Top-p (nucleus) sampling probability
24    pub top_p: Option<f32>,
25    /// Top-k sampling
26    pub top_k: Option<u32>,
27    /// Stop sequences to end generation when encountered
28    pub stop_sequences: Option<Vec<String>>,
29    /// Presence penalty to penalize new tokens based on whether they appear in the prompt
30    pub presence_penalty: Option<f32>,
31    /// Frequency penalty to penalize new tokens based on their frequency in the prompt
32    pub frequency_penalty: Option<f32>,
33    /// The format of the response
34    pub response_format: Option<LanguageModelResponseFormat>,
35    /// Seed for random number generation
36    pub seed: Option<u64>,
37    /// Tools available to the language model
38    pub tools: Option<Vec<LanguageModelTool>>,
39    /// The tool choice strategy to use when the model calls a tool
40    pub tool_choice: Option<LanguageModelToolChoice>,
41    /// Whether to include raw chunks in the response
42    pub include_raw_chunks: Option<bool>,
43    /// Signal to abort the request
44    pub abort_signal: Option<CancellationToken>,
45    /// Custom headers to include in the request
46    pub headers: Option<HeaderMap>,
47
48    /// Provider-specific options that can be used to pass additional information to the provider or control provider-specific behavior
49    pub provider_options: Option<ProviderOptions>,
50}
51
52#[derive(Debug, Clone)]
53pub enum LanguageModelResponseFormat {
54    /// The response should be returned as text
55    Text,
56    /// Structured JSON response, with optional schema for validation
57    Json {
58        /// Optional JSON schema to validate the output against
59        schema: Option<JsonSchema>,
60        /// The name of the output
61        name: Option<String>,
62        /// Description of the object that should be generated
63        description: Option<String>,
64    },
65}
bitrouter_core/models/language/call_options.rs

bitrouter_core/models/language/
call_options.rs
