modelc 0.1.4

Compile model weight files to standalone executable binaries
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

use std::convert::Infallible;
use std::sync::Arc;

use axum::{Json, extract::State, response::sse::{Event, Sse}};
use tokio_stream::wrappers::ReceiverStream;

use super::{AppState, ChatRequest, ChatResponse, CompleteRequest, CompleteResponse, InferRequest, InferResponse, Message, ModelInfo, StreamChunk};
use super::infer::{run_inference, run_text_inference};

pub(super) async fn infer(
    State(state): State<Arc<AppState>>,
    Json(req): Json<InferRequest>,
) -> Json<InferResponse> {
    if !req.inputs.is_empty() {
        let start = std::time::Instant::now();
        let outs: Vec<Vec<f32>> = req
            .inputs
            .iter()
            .map(|inp| run_inference(&state, inp))
            .collect();
        if state.profile {
            eprintln!("  batch infer: {} items in {:.3} ms", outs.len(), start.elapsed().as_secs_f64() * 1000.0);
        }
        return Json(InferResponse {
            output: None,
            outputs: Some(outs),
        });
    }

    let start = std::time::Instant::now();
    let output = run_inference(&state, &req.input);
    if state.profile {
        eprintln!("  infer: {:.3} ms", start.elapsed().as_secs_f64() * 1000.0);
    }
    Json(InferResponse {
        output: Some(output),
        outputs: None,
    })
}

pub(super) async fn model_info(State(state): State<Arc<AppState>>) -> Json<ModelInfo> {
    Json(ModelInfo {
        name: state.name.clone(),
        architecture: state.architecture.clone(),
        total_params: state.total_params,
        total_bytes: state.total_bytes,
        tensors: state.tensor_names.clone(),
    })
}

pub(super) async fn chat(
    State(state): State<Arc<AppState>>,
    Json(req): Json<ChatRequest>,
) -> Json<ChatResponse> {
    let prompt = req
        .messages
        .last()
        .map(|m| m.content.clone())
        .unwrap_or_default();
    let output = run_text_inference(&state, &prompt);
    Json(ChatResponse {
        message: Message {
            role: "assistant".to_string(),
            content: output,
        },
    })
}

pub(super) async fn complete(
    State(state): State<Arc<AppState>>,
    Json(req): Json<CompleteRequest>,
) -> Json<CompleteResponse> {
    let output = run_text_inference(&state, &req.prompt);
    Json(CompleteResponse { completion: output })
}

pub(super) async fn chat_stream(
    State(state): State<Arc<AppState>>,
    Json(req): Json<ChatRequest>,
) -> Sse<ReceiverStream<Result<Event, Infallible>>> {
    let prompt = req
        .messages
        .last()
        .map(|m| m.content.clone())
        .unwrap_or_default();
    let output = run_text_inference(&state, &prompt);

    let (tx, rx) = tokio::sync::mpsc::channel::<Result<Event, Infallible>>(4);

    tokio::spawn(async move {
        let _ = tx
            .send(Ok(Event::default().data(
                serde_json::to_string(&StreamChunk {
                    delta: output,
                    done: true,
                })
                .unwrap(),
            )))
            .await;
    });

    Sse::new(ReceiverStream::new(rx))
}