Skip to main content

auto_causal_lm/
lib.rs

1//! A Rust re-implementation of `transformers.AutoModelForCausalLM` built on Candle.
2//!
3//! It is **model-only by design**: it loads a Hugging Face-style checkpoint
4//! directory containing `config.json` and `.safetensors` files and returns a
5//! [`CausalLM`] trait object. Tokenization, downloading, and generation loops
6//! live in the caller (see the `examples/` directory for pairing with the
7//! `tokenizers` crate).
8//!
9//! # Quick start
10//!
11//! ```no_run
12//! use auto_causal_lm::AutoModelForCausalLM;
13//! use candle_core::{DType, Device};
14//!
15//! let mut model = AutoModelForCausalLM::from_pretrained_with(
16//!     "/path/to/checkpoint",
17//!     Device::Cpu,
18//!     DType::F32,
19//! )?;
20//! # Ok::<(), auto_causal_lm::Error>(())
21//! ```
22//!
23//! # Supported architectures
24//!
25//! - Llama / Mistral / Mixtral / Yi
26//! - Qwen2 / Qwen3
27//! - Phi3
28//! - StableLM
29//! - StarCoder2
30//! - Granite / GraniteMoE
31//! - OLMo
32//! - Gemma / Gemma2 / Gemma3
33//! - Falcon
34//! - Bloom
35//! - GPT-2
36//! - Mamba / Mamba2
37//! - RWKV6
38//! - ChatGLM / GLM4
39//! - DeepSeek-V2 / DeepSeek-V3
40
41pub mod cache;
42pub mod common;
43pub mod error;
44pub mod families;
45pub mod mapper;
46pub mod model;
47pub mod registry;
48
49pub use cache::{Cache, CacheLayout, KV};
50pub use error::{Error, Result};
51pub use mapper::{KVMapper, MapperRegistry, PassthroughMapper};
52pub use model::{CausalLM, CausalLMCacheExt, KVCache, ModelConfig};
53
54use candle_core::{DType, Device};
55use candle_nn::VarBuilder;
56use std::path::Path;
57
58/// Entry point for loading causal language models from a local checkpoint directory.
59pub struct AutoModelForCausalLM;
60
61impl AutoModelForCausalLM {
62    /// Load a model on CPU with F32 weights (backwards-compatible default).
63    pub fn from_pretrained<P: AsRef<Path>>(model_dir: P) -> Result<Box<dyn CausalLM>> {
64        Self::from_pretrained_with(model_dir, Device::Cpu, DType::F32)
65    }
66
67    /// Load a model with explicit device and dtype.
68    pub fn from_pretrained_with<P: AsRef<Path>>(
69        model_dir: P,
70        device: Device,
71        dtype: DType,
72    ) -> Result<Box<dyn CausalLM>> {
73        let model_dir = model_dir.as_ref();
74        let config_path = model_dir.join("config.json");
75        let config_text = std::fs::read_to_string(&config_path)?;
76        let config: serde_json::Value = serde_json::from_str(&config_text)?;
77
78        let architecture = config
79            .get("architectures")
80            .and_then(|v| v.as_array())
81            .and_then(|a| a.first())
82            .and_then(|v| v.as_str())
83            .map(|s| s.to_string())
84            .or_else(|| {
85                config
86                    .get("model_type")
87                    .and_then(|v| v.as_str())
88                    .map(|s| s.to_string())
89            })
90            .ok_or(Error::MissingConfigField("architectures or model_type"))?;
91
92        let vb = load_var_builder(model_dir, dtype, &device)?;
93
94        let loader = registry::dispatch(&architecture)
95            .ok_or(Error::UnsupportedArchitecture(architecture))?;
96        loader(&config_path, vb, &device)
97    }
98}
99
100fn load_var_builder(
101    model_dir: &Path,
102    dtype: DType,
103    device: &Device,
104) -> Result<VarBuilder<'static>> {
105    let paths = registry::resolve_safetensor_paths(model_dir)?;
106    if paths.is_empty() {
107        return Err(Error::MissingWeights(
108            "no safetensors files found".to_string(),
109        ));
110    }
111    // SAFETY: This function memory-maps caller-supplied files. The caller must
112    // ensure the files remain valid and unmodified for the lifetime of the
113    // returned `VarBuilder`.
114    let tensors = unsafe { candle_core::safetensors::MmapedSafetensors::multi(&paths) }?;
115    Ok(VarBuilder::from_backend(
116        Box::new(tensors),
117        dtype,
118        device.clone(),
119    ))
120}