joshua 0.1.0

An mmap-based LLM inference engine for the Rust ecosystem — a Rust clone of Cactus
Documentation
//! Joshua — a pure-Rust LLM inference engine.
//!
//! Joshua provides a pure-Rust inference layer built on
//! [candle](https://github.com/huggingface/candle) (HuggingFace's native Rust
//! ML framework) plus an OpenAI-compatible HTTP server built on [`axum`].
//! No C or C++ runtime dependencies are required for CPU inference.
//!
//! # Quick start
//!
//! ```no_run
//! use joshua::{Engine, GenerationOptions};
//!
//! let engine = Engine::new("path/to/model.gguf").unwrap();
//!
//! let messages = vec![joshua::ChatMessage::text("user", "Hello!")];
//!
//! let (text, usage, _, _) = engine.complete(&messages, &GenerationOptions::default()).unwrap();
//! println!("{text}");
//! ```
//!
//! A `tokenizer.json` from the model's HuggingFace repository must be placed
//! alongside the `.gguf` file so the engine can tokenise prompts.

pub mod compression;
pub mod embedding;
pub mod engine;
pub mod error;
pub mod gguf_ext;
pub mod iq2xxs;
pub mod kquant_dot;
pub mod kimi_k3;
pub mod mmap_tensor;
pub mod mxfp4;
pub mod model;
pub mod quantized_deepseek2;
pub mod quantized_deepseek4;
pub mod quant_matmul;
pub mod npu;
pub mod server;
pub mod simd;
pub mod template;
pub mod whisper;
pub mod tools;
pub mod types;

/// Re-exported so consumers of the `tls` feature can configure rustls
/// (e.g. install a different [`rustls::crypto::CryptoProvider`]) without
/// adding their own dependency on a matching rustls version.
#[cfg(feature = "tls")]
pub use rustls;

pub use engine::{
    find_gguf_in_dir, Engine, EngineOptions, HugePages, MlockMode, MmapMode, PageSize,
};
pub use error::{JoshuaError, Result};
pub use types::{
    ChatMessage, EmbeddingRequest, EmbeddingResponse, GenerationOptions, UsageInfo,
};