Struct Session

Source

pub struct Session { /* private fields */ }

Expand description

LiteRT-LM Session - represents a conversation context

A session maintains the conversation history and can generate text responses to prompts.

Implementations§

Source §

impl Session

Source

pub fn generate(&self, prompt: &str) -> Result<String>

Generate text from a prompt

§Arguments

prompt - The input text prompt

§Returns

The generated text response

§Example

use litert_lm::{Engine, Backend};

let engine = Engine::new("model.tflite", Backend::Cpu)?;
let session = engine.create_session()?;
let response = session.generate("What is 2+2?")?;
println!("Response: {}", response);

Examples found in repository ?

examples/batch_inference.rs (line 37)

3fn main() -> Result<(), Box<dyn std::error::Error>> {
4    // Get model path from command line argument
5    let args: Vec<String> = std::env::args().collect();
6    if args.len() < 2 {
7        eprintln!("Usage: {} <model_path>", args[0]);
8        eprintln!("Example: {} model.tflite", args[0]);
9        std::process::exit(1);
10    }
11    let model_path = &args[1];
12
13    println!("Loading model from: {}", model_path);
14
15    // Create engine
16    let engine = Engine::new(model_path, Backend::Cpu)?;
17    println!("Engine created successfully!\n");
18
19    // Test prompts
20    let prompts = vec![
21        "What is the capital of France?",
22        "Explain quantum computing in simple terms.",
23        "Write a haiku about programming.",
24        "What is 2 + 2?",
25    ];
26
27    println!("Running batch inference...\n");
28    println!("========================================");
29
30    // Process each prompt in a separate session
31    for (i, prompt) in prompts.iter().enumerate() {
32        println!("\n[{}] Prompt: {}", i + 1, prompt);
33
34        // Create a new session for each prompt
35        let session = engine.create_session()?;
36
37        match session.generate(prompt) {
38            Ok(response) => {
39                println!("Response: {}", response);
40            }
41            Err(e) => {
42                eprintln!("Error: {}", e);
43            }
44        }
45
46        println!("----------------------------------------");
47    }
48
49    println!("\nBatch inference complete!");
50
51    Ok(())
52}

More examples

Hide additional examples

examples/simple_chat.rs (line 50)

4fn main() -> Result<(), Box<dyn std::error::Error>> {
5    // Get model path from command line argument
6    let args: Vec<String> = std::env::args().collect();
7    if args.len() < 2 {
8        eprintln!("Usage: {} <model_path>", args[0]);
9        eprintln!("Example: {} model.tflite", args[0]);
10        std::process::exit(1);
11    }
12    let model_path = &args[1];
13
14    println!("Loading model from: {}", model_path);
15
16    // Create engine with CPU backend
17    let engine = Engine::new(model_path, Backend::Cpu)?;
18    println!("Engine created successfully!");
19
20    // Create a session (conversation)
21    let session = engine.create_session()?;
22    println!("Session created successfully!");
23    println!();
24    println!("You can now chat with the model. Type 'quit' or 'exit' to stop.");
25    println!("========================================");
26    println!();
27
28    // Interactive chat loop
29    loop {
30        print!("You: ");
31        io::stdout().flush()?;
32
33        let mut input = String::new();
34        io::stdin().read_line(&mut input)?;
35        let input = input.trim();
36
37        if input.is_empty() {
38            continue;
39        }
40
41        if input.eq_ignore_ascii_case("quit") || input.eq_ignore_ascii_case("exit") {
42            println!("Goodbye!");
43            break;
44        }
45
46        // Generate response
47        print!("Assistant: ");
48        io::stdout().flush()?;
49
50        match session.generate(input) {
51            Ok(response) => {
52                println!("{}", response);
53                println!();
54            }
55            Err(e) => {
56                eprintln!("Error generating response: {}", e);
57                println!();
58            }
59        }
60    }
61
62    Ok(())
63}