use std::sync::Arc;
use bunsen::{
burner::device::DeviceSession,
data::{
builds::{
BuildFactory,
BuildProvider,
BuildSummary,
CATALOG,
ConstructInit,
},
catalog::ResolvedBuild,
formats::LoadPolicy,
resources::ResourceCache,
},
errors::BunsenResult,
kits::{
language::generation::{
gemma::pretrained::{
DEFAULT_GEMMA_MODEL,
GEMMA_KIT,
default_gemma_factory,
},
nanochat::pretrained::{
DEFAULT_NANOCHAT_MODEL,
NANOCHAT_KIT,
default_nanochat_factory,
},
qwen3::pretrained::{
DEFAULT_QWEN3_MODEL,
QWEN3_KIT,
default_qwen3_factory,
},
qwen3_5::pretrained::{
DEFAULT_QWEN3_5_MODEL,
QWEN3_5_KIT,
default_qwen3_5_factory,
},
},
support::construct::LmPolicy,
},
};
use crate::chat::{
ChatBackend,
ChatModel,
GemmaChat,
NanoChatChat,
Qwen3_5Chat,
Qwen3Chat,
no_chat_template,
};
pub fn require_template<H: ConstructInit>(
factory: &BuildFactory<H>,
name: &str,
) -> BunsenResult<()> {
let (provider, _) = factory.lookup(name)?;
if provider != CATALOG {
return Ok(());
}
let resolved = factory.resolve_presets(name, &LmPolicy::default())?;
match resolved.config.generation.chat_template {
Some(_) => Ok(()),
None => Err(no_chat_template(name)),
}
}
pub enum LoadedChat {
Qwen3(Arc<ChatModel<Qwen3Chat>>),
Qwen3_5(Arc<ChatModel<Qwen3_5Chat>>),
Gemma(Arc<ChatModel<GemmaChat>>),
NanoChat(Arc<ChatModel<NanoChatChat>>),
}
pub trait ChatVisitor {
type Output;
fn visit<B: ChatBackend>(
self,
chat: &Arc<ChatModel<B>>,
) -> Self::Output;
}
impl LoadedChat {
pub fn visit<V: ChatVisitor>(
&self,
visitor: V,
) -> V::Output {
match self {
LoadedChat::Qwen3(chat) => visitor.visit(chat),
LoadedChat::Qwen3_5(chat) => visitor.visit(chat),
LoadedChat::Gemma(chat) => visitor.visit(chat),
LoadedChat::NanoChat(chat) => visitor.visit(chat),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ChatModelFamily {
Qwen3,
Qwen3_5,
Gemma,
NanoChat,
}
impl ChatModelFamily {
pub const ALL: [ChatModelFamily; 4] = [
ChatModelFamily::Qwen3,
ChatModelFamily::Qwen3_5,
ChatModelFamily::Gemma,
ChatModelFamily::NanoChat,
];
pub fn name(self) -> &'static str {
match self {
ChatModelFamily::Qwen3 => "qwen3",
ChatModelFamily::Qwen3_5 => "qwen3.5",
ChatModelFamily::Gemma => "gemma",
ChatModelFamily::NanoChat => "nanochat",
}
}
pub fn default_model(self) -> &'static str {
match self {
ChatModelFamily::Qwen3 => DEFAULT_QWEN3_MODEL,
ChatModelFamily::Qwen3_5 => DEFAULT_QWEN3_5_MODEL,
ChatModelFamily::Gemma => DEFAULT_GEMMA_MODEL,
ChatModelFamily::NanoChat => DEFAULT_NANOCHAT_MODEL,
}
}
pub fn cache_kit(self) -> &'static str {
match self {
ChatModelFamily::Qwen3 => QWEN3_KIT,
ChatModelFamily::Qwen3_5 => QWEN3_5_KIT,
ChatModelFamily::Gemma => GEMMA_KIT,
ChatModelFamily::NanoChat => NANOCHAT_KIT,
}
}
pub fn hf_note(self) -> Option<&'static str> {
match self {
ChatModelFamily::Qwen3 => {
Some("hf:org/repo: any Hugging Face repo in transformers' Qwen3ForCausalLM layout")
}
ChatModelFamily::Qwen3_5 => Some(
"hf:org/repo (under `family qwen3.5`): any Hugging Face repo in a \
transformers Qwen3.5's layout",
),
ChatModelFamily::Gemma => Some(
"hf:org/repo (under `family gemma`): any Hugging Face repo in a \
transformers Gemma's layout",
),
ChatModelFamily::NanoChat => None,
}
}
pub fn providers(self) -> BunsenResult<Vec<Arc<dyn BuildProvider>>> {
fn of<H: ConstructInit>(factory: BuildFactory<H>) -> Vec<Arc<dyn BuildProvider>> {
factory.providers().to_vec()
}
Ok(match self {
ChatModelFamily::Qwen3 => of(default_qwen3_factory()?),
ChatModelFamily::Qwen3_5 => of(default_qwen3_5_factory()?),
ChatModelFamily::Gemma => of(default_gemma_factory()?),
ChatModelFamily::NanoChat => of(default_nanochat_factory()?),
})
}
pub fn rows(self) -> BunsenResult<Vec<(String, ResolvedBuild)>> {
fn rows_of<H: ConstructInit>(
factory: &BuildFactory<H>
) -> BunsenResult<Vec<(String, ResolvedBuild)>> {
factory
.ids()
.into_iter()
.map(|id| Ok((id.clone(), factory.lookup(&id)?.1)))
.collect()
}
match self {
ChatModelFamily::Qwen3 => rows_of(&default_qwen3_factory()?),
ChatModelFamily::Qwen3_5 => rows_of(&default_qwen3_5_factory()?),
ChatModelFamily::Gemma => rows_of(&default_gemma_factory()?),
ChatModelFamily::NanoChat => rows_of(&default_nanochat_factory()?),
}
}
pub fn load(
self,
name: &str,
cache: &ResourceCache,
device: &DeviceSession,
policy: LoadPolicy,
) -> BunsenResult<(LoadedChat, BuildSummary)> {
Ok(match self {
ChatModelFamily::Qwen3 => {
let factory = default_qwen3_factory()?;
require_template(&factory, name)?;
let loaded = factory.load(name, cache, device, policy)?;
let chat = ChatModel::from_construct(loaded.handle)?;
(LoadedChat::Qwen3(Arc::new(chat)), loaded.summary)
}
ChatModelFamily::Qwen3_5 => {
let factory = default_qwen3_5_factory()?;
require_template(&factory, name)?;
let loaded = factory.load(name, cache, device, policy)?;
let chat = ChatModel::new(Qwen3_5Chat::from_construct(loaded.handle)?);
(LoadedChat::Qwen3_5(Arc::new(chat)), loaded.summary)
}
ChatModelFamily::Gemma => {
let factory = default_gemma_factory()?;
require_template(&factory, name)?;
let loaded = factory.load(name, cache, device, policy)?;
let chat = ChatModel::new(GemmaChat::from_construct(loaded.handle)?);
(LoadedChat::Gemma(Arc::new(chat)), loaded.summary)
}
ChatModelFamily::NanoChat => {
let loaded = default_nanochat_factory()?.load(name, cache, device, policy)?;
let chat = ChatModel::new(NanoChatChat::from_construct(loaded.handle)?);
(LoadedChat::NanoChat(Arc::new(chat)), loaded.summary)
}
})
}
pub fn fetch(
self,
name: &str,
cache: &ResourceCache,
) -> BunsenResult<()> {
let (id, loaded) = match self {
ChatModelFamily::Qwen3 => {
let model = default_qwen3_factory()?.resolve(name, cache)?;
let map = model.plan(cache)?;
(model.id(), cache.load(&map)?)
}
ChatModelFamily::Qwen3_5 => {
let model = default_qwen3_5_factory()?.resolve(name, cache)?;
let map = model.plan(cache)?;
(model.id(), cache.load(&map)?)
}
ChatModelFamily::Gemma => {
let model = default_gemma_factory()?.resolve(name, cache)?;
let map = model.plan(cache)?;
(model.id(), cache.load(&map)?)
}
ChatModelFamily::NanoChat => {
let model = default_nanochat_factory()?.resolve(name, cache)?;
let map = model.plan(cache)?;
(model.id(), cache.load(&map)?)
}
};
println!("{id}:");
for (key, part) in loaded.iter() {
println!(" {key}: {}", part.path.display());
}
Ok(())
}
}
pub trait ChatModelFamilyFlags: clap::Args + std::fmt::Debug {
fn presets(&self) -> Vec<(&'static str, &'static str)> {
Vec::new()
}
fn hide_thinking(&self) -> bool {
false
}
}
#[derive(clap::Args, Debug)]
pub struct ChatThinkFlags {
#[arg(long, conflicts_with = "no_think")]
think: bool,
#[arg(long)]
no_think: bool,
#[arg(long)]
show_thinking: bool,
}
impl ChatModelFamilyFlags for ChatThinkFlags {
fn presets(&self) -> Vec<(&'static str, &'static str)> {
match (self.think, self.no_think) {
(true, _) => vec![("thinking", "--think")],
(_, true) => vec![("no-thinking", "--no-think")],
_ => Vec::new(),
}
}
fn hide_thinking(&self) -> bool {
!self.show_thinking
}
}
#[derive(clap::Args, Debug)]
pub struct ChatPlainFlags {}
impl ChatModelFamilyFlags for ChatPlainFlags {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_require_template() {
let gemma = default_gemma_factory().unwrap();
for id in gemma.ids() {
require_template(&gemma, &id).unwrap_or_else(|e| panic!("{id}: {e}"));
}
let qwen3 = default_qwen3_factory().unwrap();
for id in qwen3.ids() {
require_template(&qwen3, &id).unwrap_or_else(|e| panic!("{id}: {e}"));
}
let qwen3_5 = default_qwen3_5_factory().unwrap();
for id in qwen3_5.ids() {
require_template(&qwen3_5, &id).unwrap_or_else(|e| panic!("{id}: {e}"));
}
let err =
require_template(&gemma, "gemma-2b{generation: {chat_template: null}}").unwrap_err();
assert!(err.to_string().contains("no chat template"), "{err}");
}
}