offgrid 0.2.1

Embeddable offline AI core: safe Rust API over llama.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

//! Embeddable offline AI core.
//!
//! Safe Rust API over llama.cpp: load a GGUF model, run chat completions with
//! streaming and cancellation, and compute embeddings.
//!
//! ```no_run
//! use offgrid::{ChatMessage, ChatRequest, ChatSession, Model, ModelParams, SessionParams};
//!
//! let model = Model::load("model.gguf", &ModelParams::default())?;
//! let mut chat = ChatSession::new(model, &SessionParams::default())?;
//! let req = ChatRequest::new(vec![ChatMessage::user("Hello!")]);
//! let resp = chat.complete(&req, |piece| { print!("{piece}"); true })?;
//! println!("\n{:?}", resp.finish_reason);
//! # Ok::<(), offgrid::Error>(())
//! ```

mod backend;
mod batch;
mod chat;
mod embed;
mod error;
mod json_schema;
#[cfg(feature = "dynamic-backends")]
mod loader;
mod media;
mod model;
mod sampling;
mod template;
mod text;
mod tool_parse;
mod tools;

pub use backend::{DeviceInfo, DeviceKind, devices, system_info, version};
pub use chat::{
    CancelToken, ChatMessage, ChatRequest, ChatResponse, ChatSession, FinishReason, JsonSchemaFormat, RequestCancel,
    ResponseFormat, SessionParams, StreamEvent, Usage,
};
pub use embed::{EmbedParams, Embedder};
pub use error::{Error, Result};
pub use media::{Media, MediaKind, media_marker};
pub use model::{GpuLayers, Model, ModelInfo, ModelParams};
pub use sampling::SamplingParams;
pub use tools::{FunctionCall, FunctionDef, Tool, ToolCall, ToolChoice};