Struct LLMClient 

pub struct LLMClient { /* private fields */ }

A client for interacting with an LLM.

Implementations

impl LLMClient

pub async fn new(provider_type: LLMProviderType) -> Result<Self>

Creates a new LLMClient.

Examples found in repository

examples/direct_llm_usage.rs (line 64)

async fn simple_call() -> helios_engine::Result<()> {
    // Create a configuration for the LLM.
    let llm_config = LLMConfig {
        model_name: "gpt-3.5-turbo".to_string(),
        base_url: "https://api.openai.com/v1".to_string(),
        api_key: std::env::var("OPENAI_API_KEY")
            .unwrap_or_else(|_| "your-api-key-here".to_string()),
        temperature: 0.7,
        max_tokens: 2048,
    };

    // Create a new LLM client.
    let client = LLMClient::new(helios_engine::llm::LLMProviderType::Remote(llm_config)).await?;

    // Prepare the messages to send to the LLM.
    let messages = vec![
        ChatMessage::system("You are a helpful assistant that gives concise answers."),
        ChatMessage::user("What is the capital of France? Answer in one sentence."),
    ];

    // Make the call to the LLM.
    println!("Sending request...");
    match client.chat(messages, None, None, None, None).await {
        Ok(response) => {
            println!("✓ Response: {}", response.content);
        }
        Err(e) => {
            println!("✗ Error: {}", e);
            println!("  (Make sure to set OPENAI_API_KEY environment variable)");
        }
    }

    Ok(())
}

/// Demonstrates a multi-turn conversation with context.
async fn conversation_with_context() -> helios_engine::Result<()> {
    // Create a configuration for the LLM.
    let llm_config = LLMConfig {
        model_name: "gpt-3.5-turbo".to_string(),
        base_url: "https://api.openai.com/v1".to_string(),
        api_key: std::env::var("OPENAI_API_KEY")
            .unwrap_or_else(|_| "your-api-key-here".to_string()),
        temperature: 0.7,
        max_tokens: 2048,
    };

    // Create a new LLM client.
    let client = LLMClient::new(helios_engine::llm::LLMProviderType::Remote(llm_config)).await?;

    // Use a `ChatSession` to manage the conversation history.
    let mut session = ChatSession::new()
        .with_system_prompt("You are a helpful math tutor. Give brief, clear explanations.");

    // --- First turn ---
    println!("Turn 1:");
    session.add_user_message("What is 15 * 23?");
    print!("  User: What is 15 * 23?\n  ");

    match client
        .chat(session.get_messages(), None, None, None, None)
        .await
    {
        Ok(response) => {
            session.add_assistant_message(&response.content);
            println!("Assistant: {}", response.content);
        }
        Err(e) => {
            println!("Error: {}", e);
            return Ok(());
        }
    }

    // --- Second turn (with context from the first turn) ---
    println!("\nTurn 2:");
    session.add_user_message("Now divide that by 5.");
    print!("  User: Now divide that by 5.\n  ");

    match client
        .chat(session.get_messages(), None, None, None, None)
        .await
    {
        Ok(response) => {
            session.add_assistant_message(&response.content);
            println!("Assistant: {}", response.content);
        }
        Err(e) => {
            println!("Error: {}", e);
        }
    }

    println!("\n💡 Notice how the assistant remembered the result from the first calculation!");

    Ok(())
}

/// Provides information about using different LLM providers.
fn different_providers_info() {
    println!("You can use Helios with various LLM providers:\n");

    println!("🔵 OpenAI:");
    println!("   LLMConfig {{");
    println!("       model_name: \"gpt-4\".to_string(),");
    println!("       base_url: \"https://api.openai.com/v1\".to_string(),");
    println!("       api_key: env::var(\"OPENAI_API_KEY\").unwrap(),");
    println!("       temperature: 0.7,");
    println!("       max_tokens: 2048,");
    println!("   }}\n");

    println!("🟢 Local LM Studio:");
    println!("   LLMConfig {{");
    println!("       model_name: \"local-model\".to_string(),");
    println!("       base_url: \"http://localhost:1234/v1\".to_string(),");
    println!("       api_key: \"not-needed\".to_string(),");
    println!("       temperature: 0.7,");
    println!("       max_tokens: 2048,");
    println!("   }}\n");

    println!("🦙 Ollama:");
    println!("   LLMConfig {{");
    println!("       model_name: \"llama2\".to_string(),");
    println!("       base_url: \"http://localhost:11434/v1\".to_string(),");
    println!("       api_key: \"not-needed\".to_string(),");
    println!("       temperature: 0.7,");
    println!("       max_tokens: 2048,");
    println!("   }}\n");

    println!("🔷 Azure OpenAI:");
    println!("   LLMConfig {{");
    println!("       model_name: \"gpt-35-turbo\".to_string(),");
    println!("       base_url: \"https://your-resource.openai.azure.com/...\".to_string(),");
    println!("       api_key: env::var(\"AZURE_OPENAI_KEY\").unwrap(),");
    println!("       temperature: 0.7,");
    println!("       max_tokens: 2048,");
    println!("   }}\n");
}

/// Starts an interactive chat session with the LLM.
async fn interactive_chat() -> helios_engine::Result<()> {
    // Create a configuration for the LLM.
    let llm_config = LLMConfig {
        model_name: "gpt-3.5-turbo".to_string(),
        base_url: "https://api.openai.com/v1".to_string(),
        api_key: std::env::var("OPENAI_API_KEY")
            .unwrap_or_else(|_| "your-api-key-here".to_string()),
        temperature: 0.7,
        max_tokens: 2048,
    };

    // Create a new LLM client.
    let client = LLMClient::new(helios_engine::llm::LLMProviderType::Remote(llm_config)).await?;
    let mut session =
        ChatSession::new().with_system_prompt("You are a friendly and helpful AI assistant.");

    println!("Chat started! Type 'exit' or 'quit' to end the conversation.\n");

    loop {
        print!("You: ");
        io::stdout().flush()?;

        let mut input = String::new();
        io::stdin().read_line(&mut input)?;
        let input = input.trim();

        if input.is_empty() {
            continue;
        }

        if input == "exit" || input == "quit" {
            println!("\n👋 Goodbye!");
            break;
        }

        // Handle special commands.
        if input == "clear" {
            session.clear();
            println!("🧹 Conversation cleared!\n");
            continue;
        }

        if input == "history" {
            println!("\n📜 Conversation history:");
            for (i, msg) in session.messages.iter().enumerate() {
                println!("  {}. {:?}: {}", i + 1, msg.role, msg.content);
            }
            println!();
            continue;
        }

        session.add_user_message(input);

        print!("Assistant: ");
        io::stdout().flush()?;

        match client
            .chat(session.get_messages(), None, None, None, None)
            .await
        {
            Ok(response) => {
                session.add_assistant_message(&response.content);
                println!("{}\n", response.content);
            }
            Err(e) => {
                println!("\n❌ Error: {}", e);
                println!("   (Make sure OPENAI_API_KEY is set correctly)\n");
                // Remove the last user message since it failed.
                session.messages.pop();
            }
        }
    }

    Ok(())
}

examples/streaming_chat.rs (line 29)

async fn main() -> helios_engine::Result<()> {
    println!("🚀 Helios Engine - Streaming Example");
    println!("=====================================\n");

    // Set up the LLM configuration.
    let llm_config = LLMConfig {
        model_name: "gpt-3.5-turbo".to_string(),
        base_url: "https://api.openai.com/v1".to_string(),
        api_key: std::env::var("OPENAI_API_KEY")
            .unwrap_or_else(|_| "your-api-key-here".to_string()),
        temperature: 0.7,
        max_tokens: 2048,
    };

    // Create a new LLM client.
    let client = LLMClient::new(helios_engine::llm::LLMProviderType::Remote(llm_config)).await?;

    // --- Example 1: Simple streaming response ---
    println!("Example 1: Simple Streaming Response");
    println!("======================================\n");

    let messages = vec![
        ChatMessage::system("You are a helpful assistant."),
        ChatMessage::user("Write a short poem about coding."),
    ];

    print!("Assistant: ");
    io::stdout().flush()?;

    // Stream the response from the model, printing each chunk as it arrives.
    let response = client
        .chat_stream(messages, None, None, None, None, |chunk| {
            print!("{}", chunk);
            io::stdout().flush().unwrap();
        })
        .await?;

    println!("\n\n");

    // --- Example 2: Interactive streaming chat ---
    println!("Example 2: Interactive Streaming Chat");
    println!("======================================\n");

    let mut session = ChatSession::new().with_system_prompt("You are a helpful coding assistant.");

    let questions = vec![
        "What is Rust?",
        "What are its main benefits?",
        "Show me a simple example.",
    ];

    for question in questions {
        println!("User: {}", question);
        session.add_user_message(question);

        print!("Assistant: ");
        io::stdout().flush()?;

        // Stream the response, maintaining the conversation context.
        let response = client
            .chat_stream(session.get_messages(), None, None, None, None, |chunk| {
                print!("{}", chunk);
                io::stdout().flush().unwrap();
            })
            .await?;

        session.add_assistant_message(&response.content);
        println!("\n");
    }

    // --- Example 3: Streaming with thinking tags ---
    println!("\nExample 3: Streaming with Thinking Tags");
    println!("=========================================\n");
    println!("When using models that support thinking tags (like o1),");
    println!("you can detect and display them during streaming.\n");

    /// A helper struct to track and display thinking tags in streamed responses.
    struct ThinkingTracker {
        in_thinking: bool,
        thinking_buffer: String,
    }

    impl ThinkingTracker {
        /// Creates a new `ThinkingTracker`.
        fn new() -> Self {
            Self {
                in_thinking: false,
                thinking_buffer: String::new(),
            }
        }

        /// Processes a chunk of a streamed response and returns the processed output.
        fn process_chunk(&mut self, chunk: &str) -> String {
            let mut output = String::new();
            let mut chars = chunk.chars().peekable();

            while let Some(c) = chars.next() {
                if c == '<' {
                    let remaining: String = chars.clone().collect();
                    if remaining.starts_with("thinking>") {
                        self.in_thinking = true;
                        self.thinking_buffer.clear();
                        output.push_str("\n💭 [Thinking");
                        for _ in 0..9 {
                            chars.next();
                        }
                        continue;
                    } else if remaining.starts_with("/thinking>") {
                        self.in_thinking = false;
                        output.push_str("]\n");
                        for _ in 0..10 {
                            chars.next();
                        }
                        continue;
                    }
                }

                if self.in_thinking {
                    self.thinking_buffer.push(c);
                    if self.thinking_buffer.len() % 3 == 0 {
                        output.push('.');
                    }
                } else {
                    output.push(c);
                }
            }

            output
        }
    }

    let messages = vec![ChatMessage::user(
        "Solve this problem: What is 15 * 234 + 89?",
    )];

    let mut tracker = ThinkingTracker::new();
    print!("Assistant: ");
    io::stdout().flush()?;

    // Stream the response, processing thinking tags as they arrive.
    let _response = client
        .chat_stream(messages, None, None, None, None, |chunk| {
            let output = tracker.process_chunk(chunk);
            print!("{}", output);
            io::stdout().flush().unwrap();
        })
        .await?;

    println!("\n\n✅ Streaming examples completed!");
    println!("\nKey benefits of streaming:");
    println!("  • Real-time response display");
    println!("  • Better user experience for long responses");
    println!("  • Ability to show thinking/reasoning process");
    println!("  • Early cancellation possible (future feature)");

    Ok(())
}

pub fn provider_type(&self) -> &LLMProviderType

Returns the type of the LLM provider.

impl LLMClient

pub async fn chat( &self, messages: Vec<ChatMessage>, tools: Option<Vec<ToolDefinition>>, temperature: Option<f32>, max_tokens: Option<u32>, stop: Option<Vec<String>>, ) -> Result<ChatMessage>

Sends a chat request to the LLM.

Examples found in repository

examples/direct_llm_usage.rs (line 74)

async fn simple_call() -> helios_engine::Result<()> {
    // Create a configuration for the LLM.
    let llm_config = LLMConfig {
        model_name: "gpt-3.5-turbo".to_string(),
        base_url: "https://api.openai.com/v1".to_string(),
        api_key: std::env::var("OPENAI_API_KEY")
            .unwrap_or_else(|_| "your-api-key-here".to_string()),
        temperature: 0.7,
        max_tokens: 2048,
    };

    // Create a new LLM client.
    let client = LLMClient::new(helios_engine::llm::LLMProviderType::Remote(llm_config)).await?;

    // Prepare the messages to send to the LLM.
    let messages = vec![
        ChatMessage::system("You are a helpful assistant that gives concise answers."),
        ChatMessage::user("What is the capital of France? Answer in one sentence."),
    ];

    // Make the call to the LLM.
    println!("Sending request...");
    match client.chat(messages, None, None, None, None).await {
        Ok(response) => {
            println!("✓ Response: {}", response.content);
        }
        Err(e) => {
            println!("✗ Error: {}", e);
            println!("  (Make sure to set OPENAI_API_KEY environment variable)");
        }
    }

    Ok(())
}

/// Demonstrates a multi-turn conversation with context.
async fn conversation_with_context() -> helios_engine::Result<()> {
    // Create a configuration for the LLM.
    let llm_config = LLMConfig {
        model_name: "gpt-3.5-turbo".to_string(),
        base_url: "https://api.openai.com/v1".to_string(),
        api_key: std::env::var("OPENAI_API_KEY")
            .unwrap_or_else(|_| "your-api-key-here".to_string()),
        temperature: 0.7,
        max_tokens: 2048,
    };

    // Create a new LLM client.
    let client = LLMClient::new(helios_engine::llm::LLMProviderType::Remote(llm_config)).await?;

    // Use a `ChatSession` to manage the conversation history.
    let mut session = ChatSession::new()
        .with_system_prompt("You are a helpful math tutor. Give brief, clear explanations.");

    // --- First turn ---
    println!("Turn 1:");
    session.add_user_message("What is 15 * 23?");
    print!("  User: What is 15 * 23?\n  ");

    match client
        .chat(session.get_messages(), None, None, None, None)
        .await
    {
        Ok(response) => {
            session.add_assistant_message(&response.content);
            println!("Assistant: {}", response.content);
        }
        Err(e) => {
            println!("Error: {}", e);
            return Ok(());
        }
    }

    // --- Second turn (with context from the first turn) ---
    println!("\nTurn 2:");
    session.add_user_message("Now divide that by 5.");
    print!("  User: Now divide that by 5.\n  ");

    match client
        .chat(session.get_messages(), None, None, None, None)
        .await
    {
        Ok(response) => {
            session.add_assistant_message(&response.content);
            println!("Assistant: {}", response.content);
        }
        Err(e) => {
            println!("Error: {}", e);
        }
    }

    println!("\n💡 Notice how the assistant remembered the result from the first calculation!");

    Ok(())
}

/// Provides information about using different LLM providers.
fn different_providers_info() {
    println!("You can use Helios with various LLM providers:\n");

    println!("🔵 OpenAI:");
    println!("   LLMConfig {{");
    println!("       model_name: \"gpt-4\".to_string(),");
    println!("       base_url: \"https://api.openai.com/v1\".to_string(),");
    println!("       api_key: env::var(\"OPENAI_API_KEY\").unwrap(),");
    println!("       temperature: 0.7,");
    println!("       max_tokens: 2048,");
    println!("   }}\n");

    println!("🟢 Local LM Studio:");
    println!("   LLMConfig {{");
    println!("       model_name: \"local-model\".to_string(),");
    println!("       base_url: \"http://localhost:1234/v1\".to_string(),");
    println!("       api_key: \"not-needed\".to_string(),");
    println!("       temperature: 0.7,");
    println!("       max_tokens: 2048,");
    println!("   }}\n");

    println!("🦙 Ollama:");
    println!("   LLMConfig {{");
    println!("       model_name: \"llama2\".to_string(),");
    println!("       base_url: \"http://localhost:11434/v1\".to_string(),");
    println!("       api_key: \"not-needed\".to_string(),");
    println!("       temperature: 0.7,");
    println!("       max_tokens: 2048,");
    println!("   }}\n");

    println!("🔷 Azure OpenAI:");
    println!("   LLMConfig {{");
    println!("       model_name: \"gpt-35-turbo\".to_string(),");
    println!("       base_url: \"https://your-resource.openai.azure.com/...\".to_string(),");
    println!("       api_key: env::var(\"AZURE_OPENAI_KEY\").unwrap(),");
    println!("       temperature: 0.7,");
    println!("       max_tokens: 2048,");
    println!("   }}\n");
}

/// Starts an interactive chat session with the LLM.
async fn interactive_chat() -> helios_engine::Result<()> {
    // Create a configuration for the LLM.
    let llm_config = LLMConfig {
        model_name: "gpt-3.5-turbo".to_string(),
        base_url: "https://api.openai.com/v1".to_string(),
        api_key: std::env::var("OPENAI_API_KEY")
            .unwrap_or_else(|_| "your-api-key-here".to_string()),
        temperature: 0.7,
        max_tokens: 2048,
    };

    // Create a new LLM client.
    let client = LLMClient::new(helios_engine::llm::LLMProviderType::Remote(llm_config)).await?;
    let mut session =
        ChatSession::new().with_system_prompt("You are a friendly and helpful AI assistant.");

    println!("Chat started! Type 'exit' or 'quit' to end the conversation.\n");

    loop {
        print!("You: ");
        io::stdout().flush()?;

        let mut input = String::new();
        io::stdin().read_line(&mut input)?;
        let input = input.trim();

        if input.is_empty() {
            continue;
        }

        if input == "exit" || input == "quit" {
            println!("\n👋 Goodbye!");
            break;
        }

        // Handle special commands.
        if input == "clear" {
            session.clear();
            println!("🧹 Conversation cleared!\n");
            continue;
        }

        if input == "history" {
            println!("\n📜 Conversation history:");
            for (i, msg) in session.messages.iter().enumerate() {
                println!("  {}. {:?}: {}", i + 1, msg.role, msg.content);
            }
            println!();
            continue;
        }

        session.add_user_message(input);

        print!("Assistant: ");
        io::stdout().flush()?;

        match client
            .chat(session.get_messages(), None, None, None, None)
            .await
        {
            Ok(response) => {
                session.add_assistant_message(&response.content);
                println!("{}\n", response.content);
            }
            Err(e) => {
                println!("\n❌ Error: {}", e);
                println!("   (Make sure OPENAI_API_KEY is set correctly)\n");
                // Remove the last user message since it failed.
                session.messages.pop();
            }
        }
    }

    Ok(())
}

pub async fn chat_stream<F>( &self, messages: Vec<ChatMessage>, tools: Option<Vec<ToolDefinition>>, temperature: Option<f32>, max_tokens: Option<u32>, stop: Option<Vec<String>>, on_chunk: F, ) -> Result<ChatMessage>

where F: FnMut(&str) + Send,

Sends a streaming chat request to the LLM.

Examples found in repository

examples/streaming_chat.rs (lines 45-48)

async fn main() -> helios_engine::Result<()> {
    println!("🚀 Helios Engine - Streaming Example");
    println!("=====================================\n");

    // Set up the LLM configuration.
    let llm_config = LLMConfig {
        model_name: "gpt-3.5-turbo".to_string(),
        base_url: "https://api.openai.com/v1".to_string(),
        api_key: std::env::var("OPENAI_API_KEY")
            .unwrap_or_else(|_| "your-api-key-here".to_string()),
        temperature: 0.7,
        max_tokens: 2048,
    };

    // Create a new LLM client.
    let client = LLMClient::new(helios_engine::llm::LLMProviderType::Remote(llm_config)).await?;

    // --- Example 1: Simple streaming response ---
    println!("Example 1: Simple Streaming Response");
    println!("======================================\n");

    let messages = vec![
        ChatMessage::system("You are a helpful assistant."),
        ChatMessage::user("Write a short poem about coding."),
    ];

    print!("Assistant: ");
    io::stdout().flush()?;

    // Stream the response from the model, printing each chunk as it arrives.
    let response = client
        .chat_stream(messages, None, None, None, None, |chunk| {
            print!("{}", chunk);
            io::stdout().flush().unwrap();
        })
        .await?;

    println!("\n\n");

    // --- Example 2: Interactive streaming chat ---
    println!("Example 2: Interactive Streaming Chat");
    println!("======================================\n");

    let mut session = ChatSession::new().with_system_prompt("You are a helpful coding assistant.");

    let questions = vec![
        "What is Rust?",
        "What are its main benefits?",
        "Show me a simple example.",
    ];

    for question in questions {
        println!("User: {}", question);
        session.add_user_message(question);

        print!("Assistant: ");
        io::stdout().flush()?;

        // Stream the response, maintaining the conversation context.
        let response = client
            .chat_stream(session.get_messages(), None, None, None, None, |chunk| {
                print!("{}", chunk);
                io::stdout().flush().unwrap();
            })
            .await?;

        session.add_assistant_message(&response.content);
        println!("\n");
    }

    // --- Example 3: Streaming with thinking tags ---
    println!("\nExample 3: Streaming with Thinking Tags");
    println!("=========================================\n");
    println!("When using models that support thinking tags (like o1),");
    println!("you can detect and display them during streaming.\n");

    /// A helper struct to track and display thinking tags in streamed responses.
    struct ThinkingTracker {
        in_thinking: bool,
        thinking_buffer: String,
    }

    impl ThinkingTracker {
        /// Creates a new `ThinkingTracker`.
        fn new() -> Self {
            Self {
                in_thinking: false,
                thinking_buffer: String::new(),
            }
        }

        /// Processes a chunk of a streamed response and returns the processed output.
        fn process_chunk(&mut self, chunk: &str) -> String {
            let mut output = String::new();
            let mut chars = chunk.chars().peekable();

            while let Some(c) = chars.next() {
                if c == '<' {
                    let remaining: String = chars.clone().collect();
                    if remaining.starts_with("thinking>") {
                        self.in_thinking = true;
                        self.thinking_buffer.clear();
                        output.push_str("\n💭 [Thinking");
                        for _ in 0..9 {
                            chars.next();
                        }
                        continue;
                    } else if remaining.starts_with("/thinking>") {
                        self.in_thinking = false;
                        output.push_str("]\n");
                        for _ in 0..10 {
                            chars.next();
                        }
                        continue;
                    }
                }

                if self.in_thinking {
                    self.thinking_buffer.push(c);
                    if self.thinking_buffer.len() % 3 == 0 {
                        output.push('.');
                    }
                } else {
                    output.push(c);
                }
            }

            output
        }
    }

    let messages = vec![ChatMessage::user(
        "Solve this problem: What is 15 * 234 + 89?",
    )];

    let mut tracker = ThinkingTracker::new();
    print!("Assistant: ");
    io::stdout().flush()?;

    // Stream the response, processing thinking tags as they arrive.
    let _response = client
        .chat_stream(messages, None, None, None, None, |chunk| {
            let output = tracker.process_chunk(chunk);
            print!("{}", output);
            io::stdout().flush().unwrap();
        })
        .await?;

    println!("\n\n✅ Streaming examples completed!");
    println!("\nKey benefits of streaming:");
    println!("  • Real-time response display");
    println!("  • Better user experience for long responses");
    println!("  • Ability to show thinking/reasoning process");
    println!("  • Early cancellation possible (future feature)");

    Ok(())
}

Trait Implementations

impl LLMProvider for LLMClient

fn as_any(&self) -> &dyn Any

Returns the provider as an Any type.

fn generate<'life0, 'async_trait>( &'life0 self, request: LLMRequest, ) -> Pin<Box<dyn Future<Output = Result<LLMResponse>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Generates a response from the LLM.