Struct LlmOptions

pub struct LlmOptions {
    pub system_message: Option<String>,
    pub tools_sse_urls: Option<Vec<String>>,
    pub temperature: Option<f32>,
    pub top_p: Option<f32>,
}

Fields

system_message: Option<String>

tools_sse_urls: Option<Vec<String>>

temperature: Option<f32>

top_p: Option<f32>

Implementations

impl LlmOptions

pub fn with_system_message(self, system_message: String) -> Self

Examples found in repository

examples/llm.rs (line 21)

fn main() {
    // large model
    let mut llm = BlocklessLlm::new(Models::Mistral7BInstructV03(None)).unwrap();

    // small model
    let mut llm_small = BlocklessLlm::new(Models::Llama321BInstruct(None)).unwrap();

    let prompt = r#"
    You are a helpful assistant.
    First time I ask, you name will be lucy.
    Second time I ask, you name will be bob.
    "#;
    llm.set_options(LlmOptions::default().with_system_message(prompt.to_string()))
        .unwrap();

    let response = llm.chat_request("What is your name?").unwrap();
    println!("llm Response: {}", response);

    let prompt_smol = r#"
    You are a helpful assistant.
    First time I ask, you name will be daisy.
    Second time I ask, you name will be hector.
    "#;
    llm_small
        .set_options(LlmOptions::default().with_system_message(prompt_smol.to_string()))
        .unwrap();

    let response = llm_small.chat_request("What is your name?").unwrap();
    println!("llm_small Response: {}", response);

    let response = llm_small.chat_request("What is your name?").unwrap();
    println!("llm_small Response: {}", response);

    // test if same instance is used in host/runtime
    let response = llm.chat_request("What is your name?").unwrap();
    println!("llm Response: {}", response);
}

pub fn with_tools_sse_urls(self, tools_sse_urls: Vec<String>) -> Self

Examples found in repository

examples/llm-mcp.rs (lines 15-18)

fn main() {
    // large model
    let mut llm = BlocklessLlm::new(Models::Custom(
        "Llama-3.1-8B-Instruct-q4f16_1-MLC".to_string(),
    ))
    .unwrap();

    // Assume we have two tools running on different ports
    // 1. http://localhost:3001/sse - add
    // 2. http://localhost:3002/sse - multiply
    llm.set_options(LlmOptions::default().with_tools_sse_urls(vec![
        "http://localhost:3001/sse".to_string(),
        "http://localhost:3002/sse".to_string(),
    ]))
    .unwrap();

    let response = llm
        .chat_request("Add the following numbers: 1215, 2213")
        .unwrap();
    println!("llm Response: {}", response);

    let response = llm.chat_request("Multiply 1215 by 2213").unwrap();
    println!("llm Response: {}", response);
}

Trait Implementations

impl Clone for LlmOptions

fn clone(&self) -> LlmOptions

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LlmOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LlmOptions

fn default() -> LlmOptions

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for LlmOptions

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Display for LlmOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl PartialEq for LlmOptions

fn eq(&self, other: &LlmOptions) -> bool

Tests for self and other values to be equal, and is used by ==.

const fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for LlmOptions

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl TryFrom<Vec<u8>> for LlmOptions

type Error = LlmErrorKind

The type returned in the event of a conversion error.

fn try_from(bytes: Vec<u8>) -> Result<Self, Self::Error>

Performs the conversion.

impl StructuralPartialEq for LlmOptions