use std::{
ops::ControlFlow,
sync::Arc,
time::{
Duration,
Instant,
},
};
use bunsen::{
errors::{
BunsenError,
BunsenResult,
},
kits::{
language::generation::{
driver::{
CachedHistory,
StopStrings,
TextStream,
},
qwen3::Qwen3Construct,
},
support::{
decoding::{
SearchConfig,
decode_cached_observed,
},
model_config::GenerationPolicy,
},
},
ops::transformers::attention::KVCacheStorage,
};
use burn::{
config::Config,
tensor::Device,
};
use crate::chat::{
ChatBackend,
ChatMessage,
Qwen3Chat,
search_config,
};
pub const DEFAULT_CONTEXT: usize = 8192;
#[derive(Debug)]
pub struct ChatModel<B: ChatBackend = Qwen3Chat> {
backend: B,
}
impl ChatModel<Qwen3Chat> {
pub fn from_construct(bundle: Arc<Qwen3Construct>) -> BunsenResult<Self> {
Ok(Self::new(Qwen3Chat::from_construct(bundle)?))
}
}
impl<B: ChatBackend> ChatModel<B> {
pub fn new(backend: B) -> Self {
Self { backend }
}
pub fn backend(&self) -> &B {
&self.backend
}
pub fn generation(&self) -> &GenerationPolicy {
self.backend.generation()
}
pub fn float_dtype(&self) -> burn::tensor::DType {
self.backend.float_dtype()
}
pub fn session(
self: &Arc<Self>,
config: ChatSessionConfig,
device: &Device,
) -> ChatSession<B> {
let context = config
.context
.or(self.backend.trained_context())
.unwrap_or(DEFAULT_CONTEXT);
let state = self.backend.decode_state(context, config.storage, device);
ChatSession {
model: self.clone(),
config,
context,
device: device.clone(),
messages: Vec::new(),
state,
history: CachedHistory::new(),
}
}
}
#[derive(Config, Debug)]
pub struct ChatSessionConfig {
#[config(default = "1024")]
pub max_new_tokens: usize,
#[config(default = "None")]
pub enable_thinking: Option<bool>,
#[config(default = "None")]
pub search: Option<SearchConfig>,
#[config(default = "Vec::new()")]
pub stop_strings: Vec<String>,
#[config(default = "None")]
pub context: Option<usize>,
#[config(default = "KVCacheStorage::Auto")]
pub storage: KVCacheStorage,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum StopReason {
StopToken(i64),
StopString(String),
Length,
}
#[derive(Clone, Debug, PartialEq)]
pub struct ChatReply {
pub text: String,
pub ids: Vec<i64>,
pub stop: StopReason,
pub prompt_tokens: usize,
pub prefilled_tokens: usize,
pub prefill_time: Duration,
pub decode_time: Duration,
}
impl ChatReply {
pub fn split_thinking(&self) -> (Option<&str>, &str) {
match self.text.split_once("</think>") {
Some((thinking, answer)) => (
Some(
thinking
.trim_start()
.trim_start_matches("<think>")
.trim_matches('\n'),
),
answer.trim_start_matches('\n'),
),
None => (None, &self.text),
}
}
}
#[derive(Debug)]
pub struct ChatSession<B: ChatBackend = Qwen3Chat> {
model: Arc<ChatModel<B>>,
config: ChatSessionConfig,
context: usize,
device: Device,
messages: Vec<ChatMessage>,
state: B::State,
history: CachedHistory,
}
impl<B: ChatBackend> ChatSession<B> {
pub fn model(&self) -> &Arc<ChatModel<B>> {
&self.model
}
pub fn messages(&self) -> &[ChatMessage] {
&self.messages
}
pub fn context(&self) -> usize {
self.context
}
pub fn config(&self) -> &ChatSessionConfig {
&self.config
}
pub fn push(
&mut self,
message: ChatMessage,
) {
self.messages.push(message);
}
pub fn send(
&mut self,
user: &str,
on_text: &mut dyn FnMut(&str),
) -> BunsenResult<ChatReply> {
self.messages.push(ChatMessage::user(user));
let reply = self.generate(on_text);
match reply {
Ok(reply) => {
self.messages
.push(ChatMessage::assistant(reply.text.clone()));
Ok(reply)
}
Err(e) => {
self.messages.pop();
Err(e)
}
}
}
fn generate(
&mut self,
on_text: &mut dyn FnMut(&str),
) -> BunsenResult<ChatReply> {
let chat = self.model.clone();
let backend = &chat.backend;
let enable_thinking = self.config.enable_thinking.or_else(|| {
backend
.generation()
.template_kwargs
.get("enable_thinking")
.and_then(serde_json::Value::as_bool)
});
let prompt = backend.render(&self.messages, enable_thinking)?;
let budget = self.config.max_new_tokens;
if prompt.len() + budget > self.context {
return Err(BunsenError::policy(format!(
"the conversation ({} tokens) and a reply ({budget}) overrun the context ({})",
prompt.len(),
self.context
)));
}
let snapshot_at = backend.snapshot_point(&self.messages, enable_thinking)?;
let start = Instant::now();
let todo = backend
.rewind(&mut self.history, &mut self.state, &prompt, &self.device)
.to_vec();
let from = prompt.len() - todo.len();
let first = match snapshot_at.filter(|&at| from <= at && at < prompt.len()) {
Some(at) => {
if at > from {
let _ = backend.prefill(&prompt[from..at], &mut self.state, &self.device);
}
backend.snapshot(&mut self.state);
backend.prefill(&prompt[at..], &mut self.state, &self.device)
}
None => backend.prefill(&todo, &mut self.state, &self.device),
};
self.history.extend(&todo);
let prefill_time = start.elapsed();
let search = self
.config
.search
.clone()
.unwrap_or_else(|| search_config(backend.generation()));
let stops = &backend.generation().stop_ids;
let mut searcher = search.init_search(stops, stops[0]);
let mut filters = backend.filters();
filters.extend(search.sampling_filters());
let start = Instant::now();
let mut stream = TextStream::new(backend.detokenizer());
let mut stop_strings = StopStrings::new(self.config.stop_strings.clone());
let mut shown = String::new();
let mut failed = None;
let out = decode_cached_observed(
backend.model(),
&mut self.state,
vec![Vec::new()],
first,
budget,
searcher.as_mut(),
&filters,
&mut |tokens| {
let Some(&id) = tokens[0].get(stream.ids().len()) else {
return ControlFlow::Continue(());
};
match stream.push(id) {
Ok(delta) => {
let safe = stop_strings.push(&delta);
on_text(&safe);
shown.push_str(&safe);
}
Err(e) => {
failed = Some(e);
return ControlFlow::Break(());
}
}
if stop_strings.stopped().is_some() {
ControlFlow::Break(())
} else {
ControlFlow::Continue(())
}
},
);
if let Some(e) = failed {
return Err(e);
}
let candidate = out
.into_iter()
.next()
.and_then(|group| group.into_iter().next())
.expect("one row, one candidate");
if stop_strings.stopped().is_none() {
let tail = stop_strings.push(&stream.finish()?);
let tail = tail + &stop_strings.finish();
on_text(&tail);
shown.push_str(&tail);
}
let decode_time = start.elapsed();
let stepped = match candidate.stop {
Some(_) => candidate.tokens.len(),
None => candidate.tokens.len().saturating_sub(1),
};
self.history.extend(&candidate.tokens[..stepped]);
let stop = match (stop_strings.stopped(), candidate.stop) {
(Some(s), _) => StopReason::StopString(s.to_string()),
(None, Some(id)) => StopReason::StopToken(id),
(None, None) => StopReason::Length,
};
Ok(ChatReply {
text: shown,
ids: candidate.tokens_with_stop(),
stop,
prompt_tokens: prompt.len(),
prefilled_tokens: todo.len(),
prefill_time,
decode_time,
})
}
}
#[cfg(test)]
mod tests {
use bunsen::{
burner::module::ModuleInit,
kits::{
language::generation::qwen3::{
Qwen3Config,
Qwen3Model,
TOKENIZER,
TOKENIZER_CONFIG,
},
support::tokens::testing::toy_byte_level_json,
},
support::testing::cpu_device,
};
use super::*;
fn toy_chat(dir: &std::path::Path) -> Arc<ChatModel> {
let template = "{%- for m in messages %}{{- '<|im_start|>' + m.role + '\\n' + m.content + '<|im_end|>\\n' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}{%- endif %}";
let files = [
(TOKENIZER, "tokenizer.json", toy_byte_level_json()),
(
TOKENIZER_CONFIG,
"tokenizer_config.json",
serde_json::json!({ "chat_template": template }).to_string(),
),
];
let device = cpu_device();
let model: Qwen3Model = Qwen3Config::new(310, 12, 2, 3, 1, 6, 20).init(&device);
let policy = bunsen::kits::language::generation::qwen3::Qwen3Policy {
generation: GenerationPolicy {
stop_ids: vec![301],
chat_template: Some(
bunsen::kits::support::model_config::ChatTemplateSource::File {
file: TOKENIZER_CONFIG.to_string(),
},
),
..Default::default()
},
};
let mut bundle = Qwen3Construct::new(model)
.with_max_positions(Some(256))
.with_policy(policy, Default::default());
for (key, name, body) in files {
let path = dir.join(name);
std::fs::write(&path, body).unwrap();
bundle = bundle.with_file(key, path);
}
Arc::new(ChatModel::from_construct(Arc::new(bundle)).unwrap())
}
#[test]
fn test_turns_stream_and_reuse_the_cache() {
let dir = std::env::temp_dir().join(format!("bunsen-tool-chat-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let chat = toy_chat(&dir);
let config = ChatSessionConfig::new().with_max_new_tokens(6);
let mut session = chat.session(config, &cpu_device());
assert_eq!(session.context(), 256);
let mut streamed = String::new();
let first = session
.send("hello", &mut |delta| streamed.push_str(delta))
.unwrap();
assert_eq!(streamed, first.text, "the deltas are the reply");
assert_eq!(first.prefilled_tokens, first.prompt_tokens, "a cold cache");
assert!(!first.ids.is_empty() && first.ids.len() <= 6);
assert_eq!(session.messages().len(), 2);
let second = session.send("again", &mut |_| {}).unwrap();
assert!(
second.prefilled_tokens < second.prompt_tokens,
"{} of {} prefilled",
second.prefilled_tokens,
second.prompt_tokens
);
assert_eq!(session.messages().len(), 4);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn test_overrun_is_refused() {
let dir =
std::env::temp_dir().join(format!("bunsen-tool-chat-overrun-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let chat = toy_chat(&dir);
let config = ChatSessionConfig::new()
.with_max_new_tokens(16)
.with_context(Some(20));
let mut session = chat.session(config, &cpu_device());
bunsen::errors::testing::ErrorMatcher::kind(bunsen::errors::BunsenErrorKind::Policy)
.message_contains("overrun")
.assert_err(&session.send("hello hello hello", &mut |_| {}));
assert!(session.messages().is_empty());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn test_split_thinking() {
let reply = |text: &str| ChatReply {
text: text.to_string(),
ids: Vec::new(),
stop: StopReason::Length,
prompt_tokens: 0,
prefilled_tokens: 0,
prefill_time: Duration::ZERO,
decode_time: Duration::ZERO,
};
assert_eq!(
reply("<think>\nhm\n</think>\n\nBlue.").split_thinking(),
(Some("hm"), "Blue.")
);
assert_eq!(reply("Blue.").split_thinking(), (None, "Blue."));
}
}