mullama 0.3.0

Comprehensive Rust bindings for llama.cpp with memory-safe API and advanced features
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

//! Batch processing example showing how to process multiple sequences efficiently
//!
//! This example demonstrates:
//! 1. Creating batches of tokens
//! 2. Batch API usage patterns
//! 3. Efficient processing concepts

use mullama::{Batch, ContextParams, MullamaError};

fn main() -> Result<(), MullamaError> {
    println!("Mullama batch processing example");

    // In a real implementation, you would load an actual GGUF model file:
    // let model = Model::load("path/to/model.gguf")?;

    // For this example, we'll demonstrate the batch API
    println!("Demonstrating batch API...");

    println!("Creating batch...");
    let _batch = Batch::new(1024, 0, 4); // Max 1024 tokens, 0 embedding size, 4 sequences
    println!(" Batch created with capacity for {} tokens", 1024);

    println!("Creating batch from tokens...");
    let tokens = vec![1, 2, 3, 4, 5];
    let token_batch = Batch::from_tokens(&tokens);
    println!(" Batch created from {} tokens", tokens.len());
    println!("   Batch is empty: {}", token_batch.is_empty());
    println!("   Batch length: {}", token_batch.len());

    // Example of context parameters for batch processing
    println!("Creating context parameters for batch processing...");
    let mut ctx_params = ContextParams::default();
    ctx_params.n_batch = 512; // Set batch size
    ctx_params.n_ctx = 2048; // Set context size
    println!(" Context parameters configured");
    println!("   Batch size: {}", ctx_params.n_batch);
    println!("   Context size: {}", ctx_params.n_ctx);

    // Example of what real batch processing would look like
    println!("Batch processing concepts:");
    println!("  1. Create batches of tokens for efficient processing");
    println!("  2. Use Batch::new(max_tokens, embd, max_seq) for custom batches");
    println!("  3. Use Batch::from_tokens(tokens) for simple token sequences");
    println!("  4. Configure context with appropriate batch size");
    println!("  5. Process batches with context.decode_batch() (when implemented)");

    println!("Batch example completed successfully!");

    Ok(())
}