1pub mod cache;
42pub mod common;
43pub mod error;
44pub mod families;
45pub mod mapper;
46pub mod model;
47pub mod registry;
48
49pub use cache::{Cache, CacheLayout, KV};
50pub use error::{Error, Result};
51pub use mapper::{KVMapper, MapperRegistry, PassthroughMapper};
52pub use model::{CausalLM, CausalLMCacheExt, KVCache, ModelConfig};
53
54use candle_core::{DType, Device};
55use candle_nn::VarBuilder;
56use std::path::Path;
57
58pub struct AutoModelForCausalLM;
60
61impl AutoModelForCausalLM {
62 pub fn from_pretrained<P: AsRef<Path>>(model_dir: P) -> Result<Box<dyn CausalLM>> {
64 Self::from_pretrained_with(model_dir, Device::Cpu, DType::F32)
65 }
66
67 pub fn from_pretrained_with<P: AsRef<Path>>(
69 model_dir: P,
70 device: Device,
71 dtype: DType,
72 ) -> Result<Box<dyn CausalLM>> {
73 let model_dir = model_dir.as_ref();
74 let config_path = model_dir.join("config.json");
75 let config_text = std::fs::read_to_string(&config_path)?;
76 let config: serde_json::Value = serde_json::from_str(&config_text)?;
77
78 let architecture = config
79 .get("architectures")
80 .and_then(|v| v.as_array())
81 .and_then(|a| a.first())
82 .and_then(|v| v.as_str())
83 .map(|s| s.to_string())
84 .or_else(|| {
85 config
86 .get("model_type")
87 .and_then(|v| v.as_str())
88 .map(|s| s.to_string())
89 })
90 .ok_or(Error::MissingConfigField("architectures or model_type"))?;
91
92 let vb = load_var_builder(model_dir, dtype, &device)?;
93
94 let loader = registry::dispatch(&architecture)
95 .ok_or(Error::UnsupportedArchitecture(architecture))?;
96 loader(&config_path, vb, &device)
97 }
98}
99
100fn load_var_builder(
101 model_dir: &Path,
102 dtype: DType,
103 device: &Device,
104) -> Result<VarBuilder<'static>> {
105 let paths = registry::resolve_safetensor_paths(model_dir)?;
106 if paths.is_empty() {
107 return Err(Error::MissingWeights(
108 "no safetensors files found".to_string(),
109 ));
110 }
111 let tensors = unsafe { candle_core::safetensors::MmapedSafetensors::multi(&paths) }?;
115 Ok(VarBuilder::from_backend(
116 Box::new(tensors),
117 dtype,
118 device.clone(),
119 ))
120}