Struct Engine 

pub struct Engine { /* private fields */ }

LiteRT-LM Engine - the main entry point for loading models

The Engine loads a model file and prepares it for inference. Create sessions from the engine to perform text generation.

Implementations

impl Engine

pub fn new(model_path: &str, backend: Backend) -> Result<Self>

Create a new Engine from a model file

Arguments

• model_path - Path to the .tflite model file
• backend - Backend to use (Cpu or Gpu)

Example

use litert_lm::{Engine, Backend};

let engine = Engine::new("model.tflite", Backend::Cpu)?;

Examples found in repository

examples/batch_inference.rs (line 16)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    // Get model path from command line argument
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 2 {
        eprintln!("Usage: {} <model_path>", args[0]);
        eprintln!("Example: {} model.tflite", args[0]);
        std::process::exit(1);
    }
    let model_path = &args[1];

    println!("Loading model from: {}", model_path);

    // Create engine
    let engine = Engine::new(model_path, Backend::Cpu)?;
    println!("Engine created successfully!\n");

    // Test prompts
    let prompts = vec![
        "What is the capital of France?",
        "Explain quantum computing in simple terms.",
        "Write a haiku about programming.",
        "What is 2 + 2?",
    ];

    println!("Running batch inference...\n");
    println!("========================================");

    // Process each prompt in a separate session
    for (i, prompt) in prompts.iter().enumerate() {
        println!("\n[{}] Prompt: {}", i + 1, prompt);

        // Create a new session for each prompt
        let session = engine.create_session()?;

        match session.generate(prompt) {
            Ok(response) => {
                println!("Response: {}", response);
            }
            Err(e) => {
                eprintln!("Error: {}", e);
            }
        }

        println!("----------------------------------------");
    }

    println!("\nBatch inference complete!");

    Ok(())
}

examples/simple_chat.rs (line 17)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    // Get model path from command line argument
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 2 {
        eprintln!("Usage: {} <model_path>", args[0]);
        eprintln!("Example: {} model.tflite", args[0]);
        std::process::exit(1);
    }
    let model_path = &args[1];

    println!("Loading model from: {}", model_path);

    // Create engine with CPU backend
    let engine = Engine::new(model_path, Backend::Cpu)?;
    println!("Engine created successfully!");

    // Create a session (conversation)
    let session = engine.create_session()?;
    println!("Session created successfully!");
    println!();
    println!("You can now chat with the model. Type 'quit' or 'exit' to stop.");
    println!("========================================");
    println!();

    // Interactive chat loop
    loop {
        print!("You: ");
        io::stdout().flush()?;

        let mut input = String::new();
        io::stdin().read_line(&mut input)?;
        let input = input.trim();

        if input.is_empty() {
            continue;
        }

        if input.eq_ignore_ascii_case("quit") || input.eq_ignore_ascii_case("exit") {
            println!("Goodbye!");
            break;
        }

        // Generate response
        print!("Assistant: ");
        io::stdout().flush()?;

        match session.generate(input) {
            Ok(response) => {
                println!("{}", response);
                println!();
            }
            Err(e) => {
                eprintln!("Error generating response: {}", e);
                println!();
            }
        }
    }

    Ok(())
}

pub fn create_session(&self) -> Result<Session>

Create a new session for text generation

Sessions maintain conversation history and state. You can create multiple sessions from the same engine.

Example

use litert_lm::{Engine, Backend};

let engine = Engine::new("model.tflite", Backend::Cpu)?;
let session = engine.create_session()?;

Examples found in repository

examples/batch_inference.rs (line 35)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    // Get model path from command line argument
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 2 {
        eprintln!("Usage: {} <model_path>", args[0]);
        eprintln!("Example: {} model.tflite", args[0]);
        std::process::exit(1);
    }
    let model_path = &args[1];

    println!("Loading model from: {}", model_path);

    // Create engine
    let engine = Engine::new(model_path, Backend::Cpu)?;
    println!("Engine created successfully!\n");

    // Test prompts
    let prompts = vec![
        "What is the capital of France?",
        "Explain quantum computing in simple terms.",
        "Write a haiku about programming.",
        "What is 2 + 2?",
    ];

    println!("Running batch inference...\n");
    println!("========================================");

    // Process each prompt in a separate session
    for (i, prompt) in prompts.iter().enumerate() {
        println!("\n[{}] Prompt: {}", i + 1, prompt);

        // Create a new session for each prompt
        let session = engine.create_session()?;

        match session.generate(prompt) {
            Ok(response) => {
                println!("Response: {}", response);
            }
            Err(e) => {
                eprintln!("Error: {}", e);
            }
        }

        println!("----------------------------------------");
    }

    println!("\nBatch inference complete!");

    Ok(())
}

examples/simple_chat.rs (line 21)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    // Get model path from command line argument
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 2 {
        eprintln!("Usage: {} <model_path>", args[0]);
        eprintln!("Example: {} model.tflite", args[0]);
        std::process::exit(1);
    }
    let model_path = &args[1];

    println!("Loading model from: {}", model_path);

    // Create engine with CPU backend
    let engine = Engine::new(model_path, Backend::Cpu)?;
    println!("Engine created successfully!");

    // Create a session (conversation)
    let session = engine.create_session()?;
    println!("Session created successfully!");
    println!();
    println!("You can now chat with the model. Type 'quit' or 'exit' to stop.");
    println!("========================================");
    println!();

    // Interactive chat loop
    loop {
        print!("You: ");
        io::stdout().flush()?;

        let mut input = String::new();
        io::stdin().read_line(&mut input)?;
        let input = input.trim();

        if input.is_empty() {
            continue;
        }

        if input.eq_ignore_ascii_case("quit") || input.eq_ignore_ascii_case("exit") {
            println!("Goodbye!");
            break;
        }

        // Generate response
        print!("Assistant: ");
        io::stdout().flush()?;

        match session.generate(input) {
            Ok(response) => {
                println!("{}", response);
                println!();
            }
            Err(e) => {
                eprintln!("Error generating response: {}", e);
                println!();
            }
        }
    }

    Ok(())
}

Trait Implementations

impl Drop for Engine

fn drop(&mut self)

Executes the destructor for this type.

fn pin_drop(self: Pin<&mut Self>)

🔬This is a nightly-only experimental API. (pin_ergonomics)

Execute the destructor for this type, but different to Drop::drop, it requires self to be pinned.

impl Send for Engine

impl Sync for Engine