use std::io::{self, BufRead, Write};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use super::util::{missing_tokenizer_warning, resolve_tokenizer};
#[allow(clippy::too_many_arguments)]
pub(crate) fn run(
model: Option<String>,
max_tokens: usize,
temperature: f32,
top_k: usize,
top_p: f32,
seed: u64,
max_seq_len: usize,
tokenizer: Option<String>,
) -> anyhow::Result<()> {
let model = model
.or_else(|| std::env::var("OXI_MODEL").ok().filter(|s| !s.is_empty()))
.ok_or_else(|| {
anyhow::anyhow!("no model: pass --model <gguf> or set OXI_MODEL (e.g. in .env)")
})?;
let tokenizer = tokenizer.or_else(|| {
std::env::var("OXI_TOKENIZER")
.ok()
.filter(|s| !s.is_empty())
});
let mmap = oxibonsai_core::gguf::reader::mmap_gguf_file(std::path::Path::new(&model))
.map_err(|e| anyhow::anyhow!("failed to open model '{model}': {e}"))?;
let gguf = oxibonsai_core::gguf::reader::GgufFile::parse(&mmap)?;
let params = oxibonsai_runtime::sampling::SamplingParams {
temperature,
top_k,
top_p,
repetition_penalty: 1.1,
..oxibonsai_runtime::sampling::SamplingParams::default()
};
let mut engine =
oxibonsai_runtime::InferenceEngine::from_gguf(&gguf, params, seed, max_seq_len)?;
let tok = {
let lookup = resolve_tokenizer(tokenizer.as_deref(), &model);
if let Some(tok_path) = lookup.found {
Some(oxibonsai_runtime::TokenizerBridge::from_file(&tok_path)?)
} else {
tracing::warn!("{}", missing_tokenizer_warning(&lookup.searched));
None
}
};
println!("OxiBonsai Interactive Chat (type 'quit' or Ctrl-D to exit)");
println!("Tip: press Ctrl-C during generation to interrupt output without exiting.");
println!("---");
let interrupted = Arc::new(AtomicBool::new(false));
{
let flag = Arc::clone(&interrupted);
ctrlc::set_handler(move || {
flag.store(true, Ordering::SeqCst);
})
.map_err(|e| anyhow::anyhow!("failed to install Ctrl-C handler: {e}"))?;
}
let stdin = io::stdin();
loop {
print!("> ");
io::stdout().flush()?;
let mut input = String::new();
match stdin.lock().read_line(&mut input) {
Ok(0) => {
println!();
break;
}
Ok(_) => {}
Err(e) if e.kind() == io::ErrorKind::Interrupted => {
interrupted.store(false, Ordering::SeqCst);
println!();
eprintln!("[Ctrl-C: type 'quit' or press Ctrl-D to exit]");
continue;
}
Err(e) => return Err(e.into()),
}
let input = input.trim();
if input.is_empty() {
interrupted.store(false, Ordering::SeqCst);
continue;
}
if input == "quit" || input == "exit" {
break;
}
if input == "/reset" {
engine.reset();
println!("[context cleared]");
continue;
}
let prompt_tokens = if let Some(tok) = &tok {
tok.encode(input)?
} else {
vec![151644]
};
interrupted.store(false, Ordering::SeqCst);
let start = std::time::Instant::now();
let (tx, rx) = std::sync::mpsc::channel::<u32>();
let interrupted_ref = Arc::clone(&interrupted);
let mut stream_state = tok.as_ref().map(|t| t.new_decode_stream(true));
let output_count = std::thread::scope(|s| -> anyhow::Result<usize> {
let thread_tx = tx.clone();
let engine_ref = &mut engine;
let tokens_ref = &prompt_tokens;
let gen_handle = s.spawn(move || {
engine_ref.generate_streaming_sync(tokens_ref, max_tokens, &thread_tx)
});
drop(tx);
let mut count = 0usize;
for token_id in rx {
if interrupted_ref.load(Ordering::SeqCst) {
break;
}
count += 1;
match (&tok, stream_state.as_mut()) {
(Some(tok), Some(state)) => {
if let Some(text) = tok.step_decode(state, token_id)? {
print!("{text}");
}
}
_ => {
if count == 1 {
print!("Tokens:");
}
print!(" {token_id}");
}
}
let _ = io::stdout().flush();
}
match gen_handle.join() {
Ok(Ok(_)) => {}
Ok(Err(e)) => return Err(e.into()),
Err(_) => return Err(anyhow::anyhow!("generation thread panicked")),
}
Ok(count)
})?;
let elapsed = start.elapsed();
println!();
if interrupted.swap(false, Ordering::SeqCst) {
eprintln!("[interrupted after {} tokens]", output_count);
} else {
let tok_per_sec = if elapsed.as_secs_f64() > 0.0 {
output_count as f64 / elapsed.as_secs_f64()
} else {
0.0
};
eprintln!(
"[{} tokens in {:.2}s, {:.1} tok/s]",
output_count,
elapsed.as_secs_f64(),
tok_per_sec
);
}
}
Ok(())
}