Struct BlocklessLlm

pub struct BlocklessLlm { /* private fields */ }

Implementations

impl BlocklessLlm

pub fn new(model: Models) -> Result<Self, LlmErrorKind>

Examples found in repository

examples/llm-mcp.rs (lines 7-9)

fn main() {
    // large model
    let mut llm = BlocklessLlm::new(Models::Custom(
        "Llama-3.1-8B-Instruct-q4f16_1-MLC".to_string(),
    ))
    .unwrap();

    // Assume we have two tools running on different ports
    // 1. http://localhost:3001/sse - add
    // 2. http://localhost:3002/sse - multiply
    llm.set_options(LlmOptions::default().with_tools_sse_urls(vec![
        "http://localhost:3001/sse".to_string(),
        "http://localhost:3002/sse".to_string(),
    ]))
    .unwrap();

    let response = llm
        .chat_request("Add the following numbers: 1215, 2213")
        .unwrap();
    println!("llm Response: {}", response);

    let response = llm.chat_request("Multiply 1215 by 2213").unwrap();
    println!("llm Response: {}", response);
}

examples/llm.rs (line 11)

fn main() {
    // large model
    let mut llm = BlocklessLlm::new(Models::Mistral7BInstructV03(None)).unwrap();

    // small model
    let mut llm_small = BlocklessLlm::new(Models::Llama321BInstruct(None)).unwrap();

    let prompt = r#"
    You are a helpful assistant.
    First time I ask, you name will be lucy.
    Second time I ask, you name will be bob.
    "#;
    llm.set_options(LlmOptions::default().with_system_message(prompt.to_string()))
        .unwrap();

    let response = llm.chat_request("What is your name?").unwrap();
    println!("llm Response: {}", response);

    let prompt_smol = r#"
    You are a helpful assistant.
    First time I ask, you name will be daisy.
    Second time I ask, you name will be hector.
    "#;
    llm_small
        .set_options(LlmOptions::default().with_system_message(prompt_smol.to_string()))
        .unwrap();

    let response = llm_small.chat_request("What is your name?").unwrap();
    println!("llm_small Response: {}", response);

    let response = llm_small.chat_request("What is your name?").unwrap();
    println!("llm_small Response: {}", response);

    // test if same instance is used in host/runtime
    let response = llm.chat_request("What is your name?").unwrap();
    println!("llm Response: {}", response);
}

pub fn handle(&self) -> u32

pub fn get_model(&self) -> Result<String, LlmErrorKind>

pub fn set_model(&mut self, model_name: &str) -> Result<(), LlmErrorKind>

pub fn get_options(&self) -> Result<LlmOptions, LlmErrorKind>

pub fn set_options(&mut self, options: LlmOptions) -> Result<(), LlmErrorKind>

Examples found in repository

examples/llm-mcp.rs (lines 15-18)

fn main() {
    // large model
    let mut llm = BlocklessLlm::new(Models::Custom(
        "Llama-3.1-8B-Instruct-q4f16_1-MLC".to_string(),
    ))
    .unwrap();

    // Assume we have two tools running on different ports
    // 1. http://localhost:3001/sse - add
    // 2. http://localhost:3002/sse - multiply
    llm.set_options(LlmOptions::default().with_tools_sse_urls(vec![
        "http://localhost:3001/sse".to_string(),
        "http://localhost:3002/sse".to_string(),
    ]))
    .unwrap();

    let response = llm
        .chat_request("Add the following numbers: 1215, 2213")
        .unwrap();
    println!("llm Response: {}", response);

    let response = llm.chat_request("Multiply 1215 by 2213").unwrap();
    println!("llm Response: {}", response);
}

examples/llm.rs (line 21)

fn main() {
    // large model
    let mut llm = BlocklessLlm::new(Models::Mistral7BInstructV03(None)).unwrap();

    // small model
    let mut llm_small = BlocklessLlm::new(Models::Llama321BInstruct(None)).unwrap();

    let prompt = r#"
    You are a helpful assistant.
    First time I ask, you name will be lucy.
    Second time I ask, you name will be bob.
    "#;
    llm.set_options(LlmOptions::default().with_system_message(prompt.to_string()))
        .unwrap();

    let response = llm.chat_request("What is your name?").unwrap();
    println!("llm Response: {}", response);

    let prompt_smol = r#"
    You are a helpful assistant.
    First time I ask, you name will be daisy.
    Second time I ask, you name will be hector.
    "#;
    llm_small
        .set_options(LlmOptions::default().with_system_message(prompt_smol.to_string()))
        .unwrap();

    let response = llm_small.chat_request("What is your name?").unwrap();
    println!("llm_small Response: {}", response);

    let response = llm_small.chat_request("What is your name?").unwrap();
    println!("llm_small Response: {}", response);

    // test if same instance is used in host/runtime
    let response = llm.chat_request("What is your name?").unwrap();
    println!("llm Response: {}", response);
}

pub fn chat_request(&self, prompt: &str) -> Result<String, LlmErrorKind>

Examples found in repository

examples/llm-mcp.rs (line 22)

fn main() {
    // large model
    let mut llm = BlocklessLlm::new(Models::Custom(
        "Llama-3.1-8B-Instruct-q4f16_1-MLC".to_string(),
    ))
    .unwrap();

    // Assume we have two tools running on different ports
    // 1. http://localhost:3001/sse - add
    // 2. http://localhost:3002/sse - multiply
    llm.set_options(LlmOptions::default().with_tools_sse_urls(vec![
        "http://localhost:3001/sse".to_string(),
        "http://localhost:3002/sse".to_string(),
    ]))
    .unwrap();

    let response = llm
        .chat_request("Add the following numbers: 1215, 2213")
        .unwrap();
    println!("llm Response: {}", response);

    let response = llm.chat_request("Multiply 1215 by 2213").unwrap();
    println!("llm Response: {}", response);
}

examples/llm.rs (line 24)

fn main() {
    // large model
    let mut llm = BlocklessLlm::new(Models::Mistral7BInstructV03(None)).unwrap();

    // small model
    let mut llm_small = BlocklessLlm::new(Models::Llama321BInstruct(None)).unwrap();

    let prompt = r#"
    You are a helpful assistant.
    First time I ask, you name will be lucy.
    Second time I ask, you name will be bob.
    "#;
    llm.set_options(LlmOptions::default().with_system_message(prompt.to_string()))
        .unwrap();

    let response = llm.chat_request("What is your name?").unwrap();
    println!("llm Response: {}", response);

    let prompt_smol = r#"
    You are a helpful assistant.
    First time I ask, you name will be daisy.
    Second time I ask, you name will be hector.
    "#;
    llm_small
        .set_options(LlmOptions::default().with_system_message(prompt_smol.to_string()))
        .unwrap();

    let response = llm_small.chat_request("What is your name?").unwrap();
    println!("llm_small Response: {}", response);

    let response = llm_small.chat_request("What is your name?").unwrap();
    println!("llm_small Response: {}", response);

    // test if same instance is used in host/runtime
    let response = llm.chat_request("What is your name?").unwrap();
    println!("llm Response: {}", response);
}

Trait Implementations

impl Clone for BlocklessLlm

fn clone(&self) -> BlocklessLlm

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BlocklessLlm

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for BlocklessLlm

fn default() -> BlocklessLlm

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for BlocklessLlm

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Drop for BlocklessLlm

fn drop(&mut self)

Executes the destructor for this type.

impl Serialize for BlocklessLlm

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.