mullama 0.3.0

Comprehensive Rust bindings for llama.cpp with memory-safe API and advanced features
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

//! Minimal web service example using Mullama's web feature stack.
//!
//! The production router requires a loaded model. This example stays model-free so it
//! can compile and run as a health/configuration service in development.
//!
//! Run with: cargo run --example web_service --features web

use axum::{
    extract::State,
    response::IntoResponse,
    routing::{get, post},
    Json, Router,
};
use mullama::prelude::*;
use serde::{Deserialize, Serialize};
use std::{net::SocketAddr, sync::Arc};
use tokio::sync::RwLock;

#[derive(Clone)]
struct DemoState {
    config: MullamaConfig,
    metrics: Arc<RwLock<DemoMetrics>>,
}

#[derive(Default, Serialize)]
struct DemoMetrics {
    requests: u64,
}

#[derive(Deserialize)]
struct DemoGenerateRequest {
    prompt: String,
    #[serde(default = "default_max_tokens")]
    max_tokens: usize,
}

#[derive(Serialize)]
struct DemoGenerateResponse {
    text: String,
    tokens_requested: usize,
}

fn default_max_tokens() -> usize {
    64
}

#[tokio::main]
async fn main() -> Result<(), MullamaError> {
    println!("Mullama Web Service Example");
    println!("===========================");

    #[cfg(feature = "web")]
    {
        let state = DemoState {
            config: MullamaConfig::default(),
            metrics: Arc::new(RwLock::new(DemoMetrics::default())),
        };

        let app: Router = Router::new()
            .route("/health", get(health))
            .route("/config", get(config))
            .route("/metrics", get(metrics))
            .route("/generate", post(generate))
            .with_state(state);

        println!("Configured endpoints:");
        println!("  GET  /health");
        println!("  GET  /config");
        println!("  GET  /metrics");
        println!("  POST /generate");

        let addr: SocketAddr = "127.0.0.1:3000"
            .parse()
            .map_err(|e| MullamaError::ConfigError(format!("Invalid listen address: {}", e)))?;
        println!("To run the service, bind a listener and serve this router at http://{}", addr);

        let _app = app;
    }

    #[cfg(not(feature = "web"))]
    {
        println!("This example requires the web feature.");
        println!("Run with: cargo run --example web_service --features web");
    }

    Ok(())
}

async fn health() -> impl IntoResponse {
    Json(serde_json::json!({
        "status": "ok",
        "service": "mullama-demo"
    }))
}

async fn config(State(state): State<DemoState>) -> impl IntoResponse {
    Json(serde_json::json!({
        "context_size": state.config.context.n_ctx,
        "temperature": state.config.sampling.temperature,
        "top_k": state.config.sampling.top_k
    }))
}

async fn metrics(State(state): State<DemoState>) -> impl IntoResponse {
    let metrics = state.metrics.read().await;
    Json(serde_json::json!({
        "requests": metrics.requests
    }))
}

async fn generate(
    State(state): State<DemoState>,
    Json(request): Json<DemoGenerateRequest>,
) -> impl IntoResponse {
    let mut metrics = state.metrics.write().await;
    metrics.requests += 1;

    Json(DemoGenerateResponse {
        text: format!("Model-free demo response for: {}", request.prompt),
        tokens_requested: request.max_tokens,
    })
}