use std::io::{self, IsTerminal, Read, Seek, Write};
use std::path::Path;
use std::sync::OnceLock;
use anyhow::Context;
use base64::Engine as _;
use clap::Args;
use futures::TryStreamExt;
use indicatif::ProgressBar;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncBufReadExt;
use crate::daemon;
mod decide;
#[derive(Args, Debug, Clone)]
pub struct RunArgs {
#[arg(value_name = "MODEL")]
pub model: String,
#[arg(long, short = 'p', value_name = "PROVIDER")]
pub provider: Option<String>,
#[arg(long, value_name = "PROVIDER")]
pub overflow_provider: Option<String>,
#[arg(long, value_name = "MODEL")]
pub overflow_model: Option<String>,
#[arg(long)]
pub think: Option<bool>,
#[arg(long, value_name = "VARIANT", conflicts_with = "think", value_parser = variant_parser())]
pub variant: Option<String>,
#[arg(long)]
pub num_predict: Option<u32>,
#[arg(long)]
pub nowordwrap: bool,
#[arg(long, hide = true, default_value_t = 0)]
pub width: u32,
#[arg(long, hide = true, default_value_t = 0)]
pub height: u32,
#[arg(long, hide = true, default_value_t = 0)]
pub steps: u32,
#[arg(long, hide = true)]
pub seed: Option<u32>,
#[arg(long, hide = true, default_value_t = 0.0)]
pub cfg_scale: f32,
#[arg(long, hide = true, default_value = "")]
pub negative: String,
#[arg(long, hide = true, conflicts_with = "audio")]
pub video: bool,
#[arg(long, hide = true)]
pub audio: bool,
#[arg(long, hide = true, default_value_t = 0.0)]
pub seconds: f32,
#[arg(long, hide = true, value_name = "PATH")]
pub image: Option<std::path::PathBuf>,
#[arg(long, hide = true, value_name = "PATH")]
pub input_video: Option<std::path::PathBuf>,
#[arg(long, hide = true, value_name = "IDX,IDX", default_value = "")]
pub condition_frames: String,
#[arg(long, hide = true)]
pub condition_keep_last: bool,
#[arg(long, hide = true, value_name = "MODE")]
pub action_mode: Option<String>,
#[arg(long, hide = true, default_value = "")]
pub action_domain: String,
#[arg(long, hide = true, value_name = "PATH")]
pub actions: Option<std::path::PathBuf>,
#[arg(long, hide = true, default_value_t = 0)]
pub action_chunk: u32,
#[arg(long, hide = true, default_value_t = 480)]
pub action_tier: u32,
#[arg(long, hide = true, default_value = "ego_view")]
pub action_view: String,
#[arg(long, hide = true, default_value_t = 0.0)]
pub fps: f32,
#[arg(long, hide = true)]
pub flow_shift: Option<f32>,
#[arg(long, value_name = "DURATION", allow_hyphen_values = true)]
pub keepalive: Option<String>,
#[command(flatten)]
pub ask: decide::Asked,
#[arg(long, value_name = "FORMAT", value_parser = ["json"])]
pub format: Option<String>,
#[arg(long)]
pub verbose: bool,
#[arg(
value_name = "PROMPT",
trailing_var_arg = true,
allow_hyphen_values = true
)]
pub prompt: Vec<String>,
}
pub(crate) fn variants() -> impl Iterator<Item = &'static str> {
let levels = crate::chat_template::EFFORT_LEVELS.iter().copied();
["none", "thinking"].into_iter().chain(levels)
}
pub(crate) fn variant_parser() -> clap::builder::PossibleValuesParser {
clap::builder::PossibleValuesParser::new(variants())
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize)]
#[serde(untagged)]
enum Think<'a> {
Switch(bool),
Level(&'a str),
}
impl<'a> Think<'a> {
fn from_args(args: &'a RunArgs) -> Option<Self> {
match args.variant.as_deref() {
Some("none") => Some(Think::Switch(false)),
Some("thinking") => Some(Think::Switch(true)),
Some(level) => Some(Think::Level(level)),
None => args.think.map(Think::Switch),
}
}
}
#[derive(Debug, Clone, Copy)]
struct ChatOptions<'a> {
think: Option<Think<'a>>,
num_predict: Option<u32>,
word_wrap: bool,
api_key: Option<&'a str>,
multimodal: bool,
keep_alive: Option<&'a str>,
}
impl Default for ChatOptions<'_> {
fn default() -> Self {
Self {
think: None,
num_predict: None,
word_wrap: true,
api_key: None,
multimodal: false,
keep_alive: None,
}
}
}
pub fn run(args: &RunArgs) -> anyhow::Result<()> {
let provider = crate::providers::provider_flag(args.provider.as_deref())?;
let overflow = crate::hybrid::overflow_flags(
args.overflow_provider.as_deref(),
args.overflow_model.as_deref(),
provider,
)?;
let prompt = args.prompt.join(" ");
check_keepalive(args.keepalive.as_deref())?;
let route = match provider {
Some(provider) => Route::Provider(provider),
None => Route::Local(crate::shortnames::resolve_ollama_api(&args.model)?),
};
crate::daemon::ensure_server("")?;
let interactive = prompt.is_empty() && io::stdin().is_terminal() && io::stdout().is_terminal();
let (model, api_key, multimodal) = match route {
Route::Provider(provider) => {
decide::reject_questions(args)?;
let (model, key) = provider_model(provider, &args.model)?;
(model, key, false)
}
Route::Local(model) => {
let info = crate::daemon::ensure_model_pulled(&model)?;
if info.decision() {
return decide::run(&model, args, info.multimodal(), interactive);
}
decide::reject_questions(args)?;
if info.image() {
let opts = crate::imagegen::ImageOptions {
media: if args.video {
crate::imagegen::Media::Video
} else if args.audio {
crate::imagegen::Media::Audio
} else {
crate::imagegen::Media::Image
},
width: args.width,
height: args.height,
steps: args.steps,
seed: args.seed,
cfg_scale: args.cfg_scale,
negative: args.negative.clone(),
seconds: args.seconds,
fps: args.fps,
flow_shift: args.flow_shift,
image: args.image.clone(),
input_video: args.input_video.clone(),
condition_frames: args.condition_frames.clone(),
condition_keep_last: args.condition_keep_last,
action: args
.action_mode
.as_ref()
.map(|mode| crate::imagegen::ActionOptions {
mode: mode.clone(),
domain: args.action_domain.clone(),
actions: args.actions.clone(),
chunk: args.action_chunk,
tier: args.action_tier,
view: args.action_view.clone(),
}),
};
return crate::imagegen::run(&model, &prompt, interactive, opts);
}
let multimodal = info.multimodal();
match overflow {
Some((provider, hosted)) => {
let (remote, key) = provider_model(provider, hosted)?;
(
crate::hybrid::pair_with_local(&model, &remote)?,
key,
multimodal,
)
}
None => (model, None, multimodal),
}
}
};
let opts = ChatOptions {
think: Think::from_args(args),
num_predict: args.num_predict,
word_wrap: !args.nowordwrap,
api_key: api_key.as_deref(),
multimodal,
keep_alive: args.keepalive.as_deref(),
};
if interactive {
run_interactive_tty(&mut Chat::new(&model, opts)?)
} else {
let p = if prompt.is_empty() {
let mut s = String::new();
io::stdin().read_line(&mut s)?;
s.trim().to_string()
} else {
prompt
};
if !p.is_empty() {
let rt = tokio::runtime::Runtime::new().context("start tokio runtime")?;
let client = chat_client(opts.api_key)?;
chat_submit(&rt, &client, &model, &mut Vec::new(), p, opts)?;
}
Ok(())
}
}
fn check_keepalive(keepalive: Option<&str>) -> anyhow::Result<()> {
if let Some(keepalive) = keepalive {
anyhow::ensure!(
crate::cmd::serve::parse_keep_alive_str(keepalive).is_some(),
"invalid --keepalive {keepalive:?}: want a duration such as 5m, or seconds"
);
}
Ok(())
}
enum Route<'a> {
Local(String),
Provider(&'a str),
}
fn provider_model(provider: &str, model: &str) -> anyhow::Result<(String, Option<String>)> {
anyhow::ensure!(
crate::daemon::connects_securely(),
"--provider needs a local llmman serve, or one over TLS: LLMMAN_HOST points at {}, \
and the provider key would cross the network in cleartext.\n\
Export the key where that daemon runs, and run llmman there.",
crate::daemon::server()
);
let model = model.trim();
let entry = daemon::provider(provider, Some(model).filter(|m| !m.is_empty()))?;
anyhow::ensure!(
!model.is_empty(),
"--provider {provider} also needs a model\n\n{}",
crate::providers::example_models(&entry.name, &entry.model_ids())
);
entry.warn_unlisted(model);
let key = entry.client_key();
anyhow::ensure!(
key.is_some() || entry.daemon_key_usable() || entry.key_optional,
"no API key for {} — {}",
entry.name,
entry.key_hint()
);
Ok((crate::providers::format_remote_ref(provider, model), key))
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct Msg {
role: String,
content: String,
#[serde(skip_serializing_if = "Option::is_none")]
thinking: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
images: Option<Vec<String>>,
}
impl Msg {
fn text(role: &str, content: String) -> Self {
Self {
role: role.into(),
content,
thinking: None,
images: None,
}
}
}
fn chat_client(api_key: Option<&str>) -> anyhow::Result<Client> {
let mut headers = crate::auth::client_headers()?;
if let Some(key) = api_key {
if headers.contains_key(reqwest::header::AUTHORIZATION) {
let mut value = reqwest::header::HeaderValue::from_str(key)
.context("provider API key is not a valid HTTP header value")?;
value.set_sensitive(true);
headers.insert("x-api-key", value);
} else {
headers.insert(reqwest::header::AUTHORIZATION, crate::auth::bearer(key)?);
}
}
crate::daemon::async_client_builder()?
.default_headers(headers)
.redirect(reqwest::redirect::Policy::none())
.build()
.context("build http client")
}
#[derive(Serialize)]
struct ChatReq<'a> {
model: &'a str,
messages: &'a [Msg],
stream: bool,
#[serde(skip_serializing_if = "Option::is_none")]
keep_alive: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
think: Option<Think<'a>>,
#[serde(skip_serializing_if = "Option::is_none")]
options: Option<ChatReqOptions>,
}
#[derive(Serialize)]
struct ChatReqOptions {
#[serde(skip_serializing_if = "Option::is_none")]
num_predict: Option<u32>,
}
#[derive(Deserialize)]
struct ChatChunk {
#[serde(default)]
message: Option<Msg>,
#[serde(default)]
done: bool,
}
fn normalize_file_path(fp: &str) -> String {
const PAIRS: &[(&str, &str)] = &[
("\\ ", " "),
("\\(", "("),
("\\)", ")"),
("\\[", "["),
("\\]", "]"),
("\\{", "{"),
("\\}", "}"),
("\\$", "$"),
("\\&", "&"),
("\\;", ";"),
("\\'", "'"),
("\\\\", "\\"),
("\\*", "*"),
("\\?", "?"),
("\\~", "~"),
];
let mut out = String::with_capacity(fp.len());
let mut rest = fp;
'outer: while !rest.is_empty() {
for (from, to) in PAIRS {
if let Some(tail) = rest.strip_prefix(from) {
out.push_str(to);
rest = tail;
continue 'outer;
}
}
let ch = rest.chars().next().expect("non-empty");
out.push(ch);
rest = &rest[ch.len_utf8()..];
}
out
}
fn extract_file_names(input: &str) -> Vec<&str> {
static RE: OnceLock<regex::Regex> = OnceLock::new();
let re = RE.get_or_init(|| {
regex::Regex::new(r"(?:[a-zA-Z]:)?(?:\./|/|\\)[\S\\ ]+?\.(?i:jpg|jpeg|png|webp|wav)\b")
.expect("valid regex")
});
re.find_iter(input).map(|m| m.as_str()).collect()
}
fn extract_file_data(input: &str) -> anyhow::Result<(String, Vec<String>)> {
let (text, files) = extract_files(input)?;
let engine = base64::engine::general_purpose::STANDARD;
Ok((text, files.iter().map(|f| engine.encode(&f.data)).collect()))
}
struct MediaFile {
content_type: &'static str,
data: Vec<u8>,
}
fn extract_files(input: &str) -> anyhow::Result<(String, Vec<MediaFile>)> {
let mut out = input.to_string();
let mut files = Vec::new();
for fp in extract_file_names(input) {
let nfp = normalize_file_path(fp);
let data = match get_image_data(&nfp) {
Ok(data) => data,
Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
Err(e) => {
eprintln!("Couldn't process file: {:?}", e.to_string());
return Err(e.into());
}
};
let ext = Path::new(&nfp)
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_lowercase())
.unwrap_or_default();
match ext.as_str() {
"wav" => eprintln!("Added audio '{nfp}'"),
_ => eprintln!("Added image '{nfp}'"),
}
out = out.replace(&format!("'{nfp}'"), "");
out = out.replace(&format!("'{fp}'"), "");
out = out.replace(fp, "");
files.push(MediaFile {
content_type: detect_content_type(&data),
data,
});
}
Ok((out.trim().to_string(), files))
}
fn get_image_data(file_path: &str) -> io::Result<Vec<u8>> {
let mut file = std::fs::File::open(file_path)?;
let mut buf = [0u8; 512];
if file.read(&mut buf)? == 0 {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "EOF"));
}
let content_type = detect_content_type(&buf);
const ALLOWED: &[&str] = &[
"image/jpeg",
"image/jpg",
"image/png",
"image/webp",
"audio/wave",
];
if !ALLOWED.contains(&content_type) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid file type: {content_type}"),
));
}
let size = file.metadata()?.len();
const MAX_SIZE: u64 = 100 * 1024 * 1024;
if size > MAX_SIZE {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"file size exceeds maximum limit (100MB)",
));
}
let mut data = Vec::with_capacity(size as usize);
file.seek(io::SeekFrom::Start(0))?;
file.take(size).read_to_end(&mut data)?;
if data.len() as u64 != size {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF",
));
}
Ok(data)
}
fn detect_content_type(data: &[u8]) -> &'static str {
fn masked(data: &[u8], mask: &[u8], pat: &[u8]) -> bool {
data.len() >= pat.len()
&& data
.iter()
.zip(mask)
.zip(pat)
.all(|((d, m), p)| d & m == *p)
}
if data.starts_with(b"\xFF\xD8\xFF") {
"image/jpeg"
} else if data.starts_with(b"\x89PNG\x0D\x0A\x1A\x0A") {
"image/png"
} else if masked(
data,
b"\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF",
b"RIFF\x00\x00\x00\x00WEBPVP",
) {
"image/webp"
} else if masked(
data,
b"\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF",
b"RIFF\x00\x00\x00\x00WAVE",
) {
"audio/wave"
} else {
"application/octet-stream"
}
}
const COLOR_GREY: &str = "\x1b[38;5;245m";
const COLOR_BOLD: &str = "\x1b[1m";
const COLOR_DEFAULT: &str = "\x1b[0m";
fn thinking_opening_text(plain: bool) -> String {
let text = "Thinking...\n";
if plain {
text.to_string()
} else {
format!("{COLOR_GREY}{COLOR_BOLD}{text}{COLOR_DEFAULT}{COLOR_GREY}")
}
}
fn thinking_closing_text(plain: bool) -> String {
let text = "...done thinking.\n\n";
if plain {
text.to_string()
} else {
format!("{COLOR_GREY}{COLOR_BOLD}{text}{COLOR_DEFAULT}")
}
}
#[derive(Default)]
struct WrapState {
line_length: usize,
word_buffer: String,
}
fn term_width() -> usize {
crate::fmt::terminal_cols(1).map_or(80, usize::from)
}
fn wrap_chunk(content: &str, wrap: bool, width: usize, state: &mut WrapState) -> String {
let mut out = String::new();
if wrap && width >= 10 {
for ch in content.chars() {
if state.line_length + 1 > width - 5 {
if state.word_buffer.chars().count() > width - 10 {
out.push_str(&state.word_buffer);
out.push(ch);
state.word_buffer.clear();
state.line_length = 0;
continue;
}
let a = state.word_buffer.chars().count();
if a > 0 {
out.push_str(&format!("\x1b[{a}D"));
}
out.push_str("\x1b[K\n");
out.push_str(&state.word_buffer);
out.push(ch);
state.line_length = state.word_buffer.chars().count() + 1;
} else {
out.push(ch);
state.line_length += 1;
match ch {
' ' | '\t' => state.word_buffer.clear(),
'\n' | '\r' => {
state.line_length = 0;
state.word_buffer.clear();
}
_ => state.word_buffer.push(ch),
}
}
}
} else {
out.push_str(&state.word_buffer);
out.push_str(content);
state.word_buffer.clear();
}
out
}
fn wrap_write(content: &str, wrap: bool, state: &mut WrapState) {
let out = wrap_chunk(content, wrap, term_width(), state);
print!("{out}");
io::stdout().flush().ok();
}
fn start_spinner() -> Option<ProgressBar> {
if !io::stderr().is_terminal() {
return None;
}
Some(crate::fmt::braille_spinner("{spinner} "))
}
struct SpinnerGuard(Option<ProgressBar>);
impl SpinnerGuard {
fn stop(&mut self) {
if let Some(sp) = self.0.take() {
sp.finish_and_clear();
}
}
}
impl Drop for SpinnerGuard {
fn drop(&mut self) {
self.stop();
}
}
trait Turns {
#[cfg_attr(not(unix), allow(dead_code))]
fn multimodal(&self) -> bool;
fn submit(&mut self, content: String) -> anyhow::Result<()>;
fn clear(&mut self);
}
struct Chat<'a> {
rt: tokio::runtime::Runtime,
client: Client,
model: &'a str,
messages: Vec<Msg>,
opts: ChatOptions<'a>,
}
impl<'a> Chat<'a> {
fn new(model: &'a str, opts: ChatOptions<'a>) -> anyhow::Result<Self> {
Ok(Self {
rt: tokio::runtime::Runtime::new().context("start tokio runtime")?,
client: chat_client(opts.api_key)?,
model,
messages: Vec::new(),
opts,
})
}
}
impl Turns for Chat<'_> {
fn multimodal(&self) -> bool {
self.opts.multimodal
}
fn submit(&mut self, content: String) -> anyhow::Result<()> {
chat_submit(
&self.rt,
&self.client,
self.model,
&mut self.messages,
content,
self.opts,
)
}
fn clear(&mut self) {
self.messages.clear();
eprintln!("Conversation cleared.");
}
}
fn run_interactive_tty(turns: &mut dyn Turns) -> anyhow::Result<()> {
#[cfg(unix)]
{
run_interactive_unix(turns)
}
#[cfg(not(unix))]
{
run_interactive_cooked(turns)
}
}
#[cfg(unix)]
fn run_interactive_unix(turns: &mut dyn Turns) -> anyhow::Result<()> {
use unix_readline::Readline;
let mut rl = Readline::new()?;
let mut multiline: Option<String> = None; let mut paste_sb = String::new();
loop {
let prompt = if multiline.is_some() || !paste_sb.is_empty() {
". "
} else {
"> "
};
let line = match rl.readline(prompt) {
Ok(Some(l)) => l,
Ok(None) => break,
Err(unix_readline::ReadlineError::Interrupted) => {
multiline = None;
paste_sb.clear();
continue;
}
};
if rl.pasting {
paste_sb.push_str(&line);
paste_sb.push('\n');
continue;
}
let line = if !paste_sb.is_empty() {
let mut full = std::mem::take(&mut paste_sb);
full.push_str(&line);
full
} else {
line
};
if let Some(ref mut buf) = multiline {
if let Some(content) = line.strip_suffix("\"\"\"") {
buf.push_str(content);
let full = std::mem::take(buf).trim_end_matches('\n').to_string();
multiline = None;
if !full.trim().is_empty() {
submit_turn(&mut rl, turns, full)?;
}
} else {
buf.push_str(&line);
buf.push('\n');
}
continue;
}
match line.trim() {
"" => continue,
"/bye" | "/exit" => break,
"/clear" => {
turns.clear();
continue;
}
s if s.starts_with('/') => {
let first_word = s.split_whitespace().next().unwrap_or(s);
let is_file = turns.multimodal()
&& extract_file_names(s)
.iter()
.any(|f| f.starts_with(first_word));
if !is_file {
eprintln!("Unknown command '{first_word}'.");
eprintln!("Commands: /bye /clear \"\"\" (multiline)");
continue;
}
}
_ => {}
}
if line.trim_start().starts_with("\"\"\"") {
let inner = line.trim_start().trim_start_matches("\"\"\"");
if let Some(closed) = inner.strip_suffix("\"\"\"") {
let content = closed.to_string();
if !content.trim().is_empty() {
submit_turn(&mut rl, turns, content)?;
}
} else {
multiline = Some(inner.to_string() + "\n");
}
continue;
}
if !line.trim().is_empty() {
submit_turn(&mut rl, turns, line)?;
}
}
Ok(())
}
#[cfg(unix)]
fn submit_turn(
rl: &mut unix_readline::Readline,
turns: &mut dyn Turns,
content: String,
) -> anyhow::Result<()> {
rl.leave_raw();
let result = turns.submit(content);
rl.enter_raw();
result
}
fn chat_submit(
rt: &tokio::runtime::Runtime,
client: &Client,
model: &str,
messages: &mut Vec<Msg>,
content: String,
opts: ChatOptions<'_>,
) -> anyhow::Result<()> {
rt.block_on(chat_submit_async(client, model, messages, content, opts))
}
async fn chat_submit_async(
client: &Client,
model: &str,
messages: &mut Vec<Msg>,
content: String,
opts: ChatOptions<'_>,
) -> anyhow::Result<()> {
let mut user = Msg::text("user", content);
if opts.multimodal {
let (text, images) = extract_file_data(&user.content)?;
user.content = text;
if !images.is_empty() {
user.images = Some(images);
}
}
messages.push(user);
let mut spinner = SpinnerGuard(start_spinner());
let ctrl_c = tokio::signal::ctrl_c();
tokio::pin!(ctrl_c);
let send = client
.post(format!("{}/api/chat", daemon::server()))
.json(&ChatReq {
model,
messages,
stream: true,
keep_alive: opts.keep_alive,
think: opts.think,
options: opts.num_predict.map(|n| ChatReqOptions {
num_predict: Some(n),
}),
})
.send();
let resp = tokio::select! {
r = send => r.context("connect to llmman serve")?,
_ = &mut ctrl_c => return Ok(()),
};
if !resp.status().is_success() {
spinner.stop();
let body = resp.text().await.unwrap_or_default();
anyhow::bail!("{}", daemon::api_error(&body).unwrap_or(body));
}
let byte_stream = resp.bytes_stream().map_err(io::Error::other);
let mut lines =
tokio::io::BufReader::new(tokio_util::io::StreamReader::new(byte_stream)).lines();
let plain_text = !io::stdout().is_terminal();
let mut full = String::new();
let mut thinking_content = String::new();
let mut thinking_open = false;
let mut thinking_closed = false;
let mut wrap = WrapState::default();
let mut interrupted = false;
loop {
let line = tokio::select! {
l = lines.next_line() => l.context("read response stream")?,
_ = &mut ctrl_c => { interrupted = true; break; }
};
let Some(line) = line else { break };
if line.is_empty() {
continue;
}
let Ok(chunk) = serde_json::from_str::<ChatChunk>(&line) else {
continue;
};
spinner.stop();
if let Some(ref msg) = chunk.message {
if let Some(ref t) = msg.thinking {
if !t.is_empty() {
if !thinking_open {
print!("{}", thinking_opening_text(plain_text));
thinking_open = true;
thinking_closed = false;
}
thinking_content.push_str(t);
wrap_write(t, opts.word_wrap, &mut wrap);
}
}
if thinking_open && !thinking_closed && !msg.content.is_empty() {
if !thinking_content.ends_with('\n') {
println!();
}
print!("{}", thinking_closing_text(plain_text));
thinking_open = false;
thinking_closed = true;
wrap = WrapState::default();
}
if !msg.content.is_empty() {
wrap_write(&msg.content, opts.word_wrap, &mut wrap);
full.push_str(&msg.content);
}
}
if chunk.done {
break;
}
}
if interrupted {
return Ok(());
}
println!("\n");
messages.push(Msg::text("assistant", full));
Ok(())
}
#[cfg(unix)]
mod unix_readline {
use std::io::{BufReader, Read, Stdin, Write};
use std::os::unix::io::AsRawFd;
const CHAR_INTERRUPT: u8 = 3; const CHAR_EOF: u8 = 4; const CHAR_CTRL_J: u8 = 10; const CHAR_ENTER: u8 = 13; const CHAR_ESC: u8 = 27;
const CHAR_ESCAPE_EX: u8 = 91; const CHAR_BACKSPACE: u8 = 127;
pub enum ReadlineError {
Interrupted,
}
const CHAR_BRACKETED_PASTE: u8 = 50; const PASTE_START: &[u8; 3] = b"00~";
const PASTE_END: &[u8; 3] = b"01~";
pub struct Readline {
reader: BufReader<Stdin>,
orig: libc::termios,
fd: std::os::unix::io::RawFd,
pub pasting: bool, }
impl Readline {
pub fn new() -> anyhow::Result<Self> {
let stdin = std::io::stdin();
let fd = stdin.as_raw_fd();
let orig = unsafe {
let mut t: libc::termios = std::mem::zeroed();
if libc::tcgetattr(fd, &mut t) < 0 {
anyhow::bail!("tcgetattr failed");
}
t
};
let rl = Self {
reader: BufReader::new(stdin),
orig,
fd,
pasting: false,
};
if unsafe { libc::tcsetattr(fd, libc::TCSANOW, &rl.raw_termios()) } < 0 {
anyhow::bail!("tcsetattr failed");
}
print!("\x1b[?2004h");
std::io::stdout().flush().ok();
Ok(rl)
}
fn raw_termios(&self) -> libc::termios {
let mut raw = self.orig;
raw.c_iflag &= !(libc::IGNBRK
| libc::BRKINT
| libc::PARMRK
| libc::ISTRIP
| libc::INLCR
| libc::IGNCR
| libc::ICRNL
| libc::IXON);
raw.c_lflag &= !(libc::ECHO | libc::ECHONL | libc::ICANON | libc::ISIG | libc::IEXTEN);
raw.c_cflag &= !(libc::CSIZE | libc::PARENB);
raw.c_cflag |= libc::CS8;
raw.c_cc[libc::VMIN] = 1;
raw.c_cc[libc::VTIME] = 0;
raw
}
pub fn enter_raw(&self) {
unsafe {
libc::tcsetattr(self.fd, libc::TCSANOW, &self.raw_termios());
}
}
pub fn leave_raw(&self) {
unsafe {
libc::tcsetattr(self.fd, libc::TCSANOW, &self.orig);
}
}
pub fn readline(&mut self, prompt: &str) -> Result<Option<String>, ReadlineError> {
print!("{prompt}");
std::io::stdout().flush().ok();
let mut buf: Vec<u8> = Vec::new();
let mut pasted_lines: Vec<String> = Vec::new();
let mut draining = false;
let mut stop_draining = false;
let mut esc = false;
let mut esc_ex = false;
loop {
if stop_draining {
draining = false;
stop_draining = false;
}
let mut b = [0u8; 1];
match self.reader.read_exact(&mut b) {
Ok(_) => {}
Err(_) => return Ok(None),
}
let r = b[0];
if !self.reader.buffer().is_empty() {
draining = true;
} else if draining {
stop_draining = true;
}
if esc_ex {
esc_ex = false;
match r {
CHAR_BRACKETED_PASTE => {
let mut code = [0u8; 3];
if self.reader.read_exact(&mut code).is_ok() {
if &code == PASTE_START {
self.pasting = true;
} else if &code == PASTE_END {
self.pasting = false;
}
if !self.reader.buffer().is_empty() {
draining = true;
}
}
}
51 | 53 | 54 => {
let mut tilde = [0u8; 1];
let _ = self.reader.read_exact(&mut tilde);
}
_ => {} }
continue;
} else if esc {
esc = false;
if r == CHAR_ESCAPE_EX {
esc_ex = true;
}
continue;
}
match r {
CHAR_INTERRUPT => {
pasted_lines.clear();
buf.clear();
println!();
return Err(ReadlineError::Interrupted);
}
CHAR_EOF => {
if buf.is_empty() {
println!();
return Ok(None);
}
}
CHAR_ESC => {
esc = true;
}
CHAR_BACKSPACE => {
if !buf.is_empty() {
loop {
match buf.pop() {
None => break,
Some(b) if (b & 0xC0) != 0x80 => break, Some(_) => {} }
}
print!("\x08 \x08");
std::io::stdout().flush().ok();
} else if !pasted_lines.is_empty() {
let prev = pasted_lines.pop().unwrap();
print!("\r\x1b[K\x1b[A\r\x1b[K{prompt}{prev}");
std::io::stdout().flush().ok();
buf = prev.into_bytes();
}
}
CHAR_CTRL_J => {
if !draining {
pasted_lines.push(String::from_utf8_lossy(&buf).to_string());
buf.clear();
println!();
print!(". ");
std::io::stdout().flush().ok();
} else {
return Ok(Some(Self::assemble(&mut buf, &mut pasted_lines)));
}
}
CHAR_ENTER => {
return Ok(Some(Self::assemble(&mut buf, &mut pasted_lines)));
}
c => {
if c >= 32 || c == 9 || c >= 0x80 {
buf.push(c);
let _ = std::io::stdout().write_all(&[c]);
std::io::stdout().flush().ok();
}
}
}
}
}
fn assemble(buf: &mut Vec<u8>, pasted_lines: &mut Vec<String>) -> String {
let last = String::from_utf8_lossy(buf).to_string();
buf.clear();
println!();
if pasted_lines.is_empty() {
last
} else {
let prefix = pasted_lines.join("\n");
pasted_lines.clear();
format!("{prefix}\n{last}")
}
}
}
impl Drop for Readline {
fn drop(&mut self) {
print!("\x1b[?2004l");
std::io::stdout().flush().ok();
unsafe {
libc::tcsetattr(self.fd, libc::TCSANOW, &self.orig);
}
}
}
}
#[allow(dead_code)]
fn run_interactive_cooked(turns: &mut dyn Turns) -> anyhow::Result<()> {
use std::io::BufRead;
let stdin = std::io::stdin();
let mut reader = std::io::BufReader::new(stdin.lock());
loop {
print!("> ");
io::stdout().flush().ok();
let mut line = String::new();
if reader.read_line(&mut line)? == 0 {
break;
}
let line = line
.trim_end_matches('\n')
.trim_end_matches('\r')
.to_string();
match line.trim() {
"" => continue,
"/bye" | "/exit" => break,
"/clear" => {
turns.clear();
continue;
}
_ => {}
}
if !line.trim().is_empty() {
turns.submit(line)?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn chat_options_default_matches_ollamas_own_wordwrap_true_default() {
let opts = ChatOptions::default();
assert_eq!(opts.think, None);
assert_eq!(opts.num_predict, None);
assert!(opts.word_wrap);
assert!(!opts.multimodal);
}
fn parse(argv: &[&str]) -> Result<RunArgs, clap::Error> {
#[derive(clap::Parser)]
struct Cli {
#[command(flatten)]
args: RunArgs,
}
let argv = ["run", "m"].iter().chain(argv);
<Cli as clap::Parser>::try_parse_from(argv).map(|c| c.args)
}
fn think_json(argv: &[&str]) -> serde_json::Value {
serde_json::to_value(Think::from_args(&parse(argv).unwrap())).unwrap()
}
#[test]
fn variant_is_sent_as_ollamas_think() {
assert_eq!(think_json(&[]), serde_json::Value::Null);
assert_eq!(think_json(&["--variant", "none"]), false);
assert_eq!(think_json(&["--variant", "thinking"]), true);
assert_eq!(think_json(&["--variant", "xhigh"]), "xhigh");
assert_eq!(think_json(&["--think", "false"]), false);
}
#[test]
fn ask_flags_come_before_the_prompt() {
let args = parse(&[
"--ask",
"Is it spam?",
"--format",
"json",
"Win",
"a",
"cruise",
])
.unwrap();
assert!(!args.ask.is_empty());
assert_eq!(args.format.as_deref(), Some("json"));
assert_eq!(args.prompt, ["Win", "a", "cruise"]);
let args = parse(&["Win a cruise", "--ask", "Is it spam?"]).unwrap();
assert!(args.ask.is_empty());
assert_eq!(args.prompt, ["Win a cruise", "--ask", "Is it spam?"]);
assert!(parse(&["--format", "xml"]).is_err());
}
#[test]
fn keepalive_rides_on_the_chat_request() {
let body = |keep_alive| {
serde_json::to_value(ChatReq {
model: "m",
messages: &[],
stream: true,
keep_alive,
think: None,
options: None,
})
.unwrap()
};
assert_eq!(body(Some("5m"))["keep_alive"], "5m");
assert!(body(None).get("keep_alive").is_none());
for value in ["1h30m", "-1", "-5m"] {
let args = parse(&["--keepalive", value]).unwrap();
assert_eq!(args.keepalive.as_deref(), Some(value));
}
}
#[test]
fn variant_rejects_unknown_names_and_think() {
assert!(parse(&["--variant", "turbo"]).is_err());
assert!(parse(&["--variant", "high", "--think", "true"]).is_err());
}
#[test]
fn extract_file_names_unix_style_paths() {
let input = " some preamble \n \
./relative\\ path/one.png inbetween1 ./not a valid two.jpg inbetween2 ./1.svg\n\
/unescaped space /three.jpeg inbetween3 /valid\\ path/dir/four.png \"./quoted with spaces/five.JPG\n\
/unescaped space /six.webp inbetween6 /valid\\ path/dir/seven.WEBP";
let res = extract_file_names(input);
assert_eq!(res.len(), 7, "{res:?}");
assert!(res[0].contains("one.png"));
assert!(res[1].contains("two.jpg"));
assert!(res[2].contains("three.jpeg"));
assert!(res[3].contains("four.png"));
assert!(res[4].contains("five.JPG"));
assert!(res[5].contains("six.webp"));
assert!(res[6].contains("seven.WEBP"));
assert!(!res[4].contains('"'));
assert!(!res.contains(&"inbetween1"));
assert!(!res.contains(&"./1.svg"));
}
#[test]
fn extract_file_names_windows_style_paths() {
let input = " some preamble\n \
c:/users/jdoe/one.png inbetween1 c:/program files/someplace/two.jpg inbetween2 \n \
/absolute/nospace/three.jpeg inbetween3 /absolute/with space/four.png inbetween4\n\
./relative\\ path/five.JPG inbetween5 \"./relative with/spaces/six.png inbetween6\n\
d:\\path with\\spaces\\seven.JPEG inbetween7 c:\\users\\jdoe\\eight.png inbetween8 \n \
d:\\program files\\someplace\\nine.png inbetween9 \"E:\\program files\\someplace\\ten.PNG\n\
c:/users/jdoe/eleven.webp inbetween11 c:/program files/someplace/twelve.WebP inbetween12\n\
d:\\path with\\spaces\\thirteen.WEBP some ending\n";
let res = extract_file_names(input);
assert_eq!(res.len(), 13, "{res:?}");
assert!(!res.contains(&"inbetween2"));
assert!(res[0].contains("one.png") && res[0].contains("c:"));
assert!(res[1].contains("two.jpg") && res[1].contains("c:"));
assert!(res[2].contains("three.jpeg"));
assert!(res[3].contains("four.png"));
assert!(res[4].contains("five.JPG"));
assert!(res[5].contains("six.png"));
assert!(res[6].contains("seven.JPEG") && res[6].contains("d:"));
assert!(res[7].contains("eight.png") && res[7].contains("c:"));
assert!(res[8].contains("nine.png") && res[8].contains("d:"));
assert!(res[9].contains("ten.PNG") && res[9].contains("E:"));
assert!(res[10].contains("eleven.webp") && res[10].contains("c:"));
assert!(res[11].contains("twelve.WebP") && res[11].contains("c:"));
assert!(res[12].contains("thirteen.WEBP") && res[12].contains("d:"));
}
#[test]
fn normalize_file_path_undoes_shell_escapes() {
assert_eq!(
normalize_file_path("/My\\ Photos/a\\(1\\)\\[x\\]\\{y\\}\\$\\&\\;\\'\\*\\?\\~.png"),
"/My Photos/a(1)[x]{y}$&;'*?~.png"
);
assert_eq!(normalize_file_path("a\\\\b"), "a\\b");
assert_eq!(normalize_file_path("/plain/path.jpg"), "/plain/path.jpg");
}
#[test]
fn detect_content_type_matches_gos_sniffer_for_the_allowed_types() {
assert_eq!(
detect_content_type(b"\xFF\xD8\xFF\xE0\x00\x10JFIF"),
"image/jpeg"
);
assert_eq!(
detect_content_type(b"\x89PNG\x0D\x0A\x1A\x0A\x00\x00\x00\x0DIHDR"),
"image/png"
);
assert_eq!(
detect_content_type(b"RIFF\x24\x00\x00\x00WEBPVP8 "),
"image/webp"
);
assert_eq!(
detect_content_type(b"RIFF\x58\x02\x00\x00WAVEfmt "),
"audio/wave"
);
assert_eq!(detect_content_type(b"GIF89a"), "application/octet-stream");
assert_eq!(detect_content_type(b"hello"), "application/octet-stream");
assert_eq!(detect_content_type(b"RIFF"), "application/octet-stream");
}
fn temp_file(name: &str, data: &[u8]) -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!(
"llmman-run-img-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
let fp = dir.join(name);
std::fs::write(&fp, data).unwrap();
fp
}
fn jpeg_bytes() -> Vec<u8> {
let mut data = vec![0u8; 600];
data[..22].copy_from_slice(&[
0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, b'J', b'F', b'I', b'F', 0x00, 0x01, 0x01, 0x01,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xd9,
]);
data
}
#[test]
fn extract_file_data_removes_quoted_filepath() {
let fp = temp_file("img.jpg", &jpeg_bytes());
let input = format!("before '{}' after", fp.display());
let (cleaned, imgs) = extract_file_data(&input).unwrap();
assert_eq!(imgs.len(), 1);
assert_eq!(cleaned, "before after");
}
#[test]
fn extract_file_data_wav() {
let mut data = vec![0u8; 600];
data[..44].copy_from_slice(&[
b'R', b'I', b'F', b'F', 0x58, 0x02, 0x00, 0x00, b'W', b'A', b'V', b'E', b'f', b'm', b't', b' ', 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x80, 0x3e, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x02, 0x00, 0x10, 0x00, b'd', b'a', b't', b'a', 0x34, 0x02, 0x00, 0x00, ]);
let fp = temp_file("sample.wav", &data);
let input = format!("before {} after", fp.display());
let (cleaned, imgs) = extract_file_data(&input).unwrap();
assert_eq!(imgs.len(), 1);
assert_eq!(cleaned, "before after");
}
#[test]
fn extract_file_data_encodes_the_file_as_standard_base64() {
let bytes = jpeg_bytes();
let fp = temp_file("img.jpg", &bytes);
let (_, imgs) = extract_file_data(&format!("look: {}", fp.display())).unwrap();
assert_eq!(
imgs,
vec![base64::engine::general_purpose::STANDARD.encode(&bytes)]
);
}
#[test]
fn extract_file_data_leaves_a_nonexistent_path_in_the_prompt() {
let input = "what is in /definitely/not/here.png then";
let (cleaned, imgs) = extract_file_data(input).unwrap();
assert!(imgs.is_empty());
assert_eq!(cleaned, input);
}
#[test]
fn extract_file_data_rejects_an_existing_file_of_the_wrong_type() {
let fp = temp_file("fake.png", b"this is not a png at all, just text");
let err = extract_file_data(&format!("see {}", fp.display())).unwrap_err();
assert!(err.to_string().contains("invalid file type"), "{err:#}");
}
#[test]
fn msg_images_is_omitted_from_json_when_none() {
let json = serde_json::to_string(&Msg::text("user", "hi".into())).unwrap();
assert_eq!(json, r#"{"role":"user","content":"hi"}"#);
let mut with = Msg::text("user", "hi".into());
with.images = Some(vec!["QUJD".into()]);
let json = serde_json::to_string(&with).unwrap();
assert_eq!(json, r#"{"role":"user","content":"hi","images":["QUJD"]}"#);
}
#[test]
fn thinking_opening_text_plain_has_no_ansi_codes() {
assert_eq!(thinking_opening_text(true), "Thinking...\n");
}
#[test]
fn thinking_opening_text_colored_wraps_in_grey_bold_then_grey_again() {
let text = thinking_opening_text(false);
assert!(text.starts_with(&format!("{COLOR_GREY}{COLOR_BOLD}")));
assert!(text.contains("Thinking...\n"));
assert!(text.ends_with(&format!("{COLOR_DEFAULT}{COLOR_GREY}")));
}
#[test]
fn thinking_closing_text_plain_has_no_ansi_codes() {
assert_eq!(thinking_closing_text(true), "...done thinking.\n\n");
}
#[test]
fn thinking_closing_text_colored_ends_in_a_full_reset() {
let text = thinking_closing_text(false);
assert!(text.starts_with(&format!("{COLOR_GREY}{COLOR_BOLD}")));
assert!(text.contains("...done thinking.\n\n"));
assert!(text.ends_with(COLOR_DEFAULT));
assert!(!text.ends_with(&format!("{COLOR_DEFAULT}{COLOR_GREY}")));
}
#[test]
fn wrap_chunk_disabled_passes_content_through_unchanged() {
let mut state = WrapState::default();
let out = wrap_chunk(
"hello there, this line is not wrapped at all",
false,
20,
&mut state,
);
assert_eq!(out, "hello there, this line is not wrapped at all");
}
#[test]
fn wrap_chunk_too_narrow_terminal_passes_content_through_unchanged() {
let mut state = WrapState::default();
let out = wrap_chunk("hello there", true, 9, &mut state);
assert_eq!(out, "hello there");
}
#[test]
fn wrap_chunk_breaks_before_a_word_that_would_overflow_the_line() {
let mut state = WrapState::default();
let out = wrap_chunk("one two three four five", true, 20, &mut state);
assert!(out.contains("\x1b[K\n"));
let after_break = out.split("\x1b[K\n").nth(1).unwrap();
assert!(after_break.starts_with("four"));
}
#[test]
fn wrap_chunk_state_persists_across_calls_like_a_single_stream() {
let mut state_streamed = WrapState::default();
let mut streamed = String::new();
for word in ["one ", "two ", "three ", "four ", "five"] {
streamed.push_str(&wrap_chunk(word, true, 20, &mut state_streamed));
}
let mut state_whole = WrapState::default();
let whole = wrap_chunk("one two three four five", true, 20, &mut state_whole);
assert_eq!(streamed, whole);
}
#[test]
fn wrap_chunk_an_unbroken_word_longer_than_the_line_flushes_without_backtracking() {
let mut state = WrapState::default();
let out = wrap_chunk("supercalifragilisticexpialidocious", true, 20, &mut state);
assert_eq!(
out,
"supercalifragilsupercalifragilisticexpialidocisticexpialidocious"
);
}
}