use std::sync::atomic::Ordering;
use std::sync::mpsc::channel;
use std::thread;
use std::time::Instant;
use modelbridge::{Cancel, Chunk, Client, Provider, Request};
const PATIENCE: usize = 280;
fn main() -> Result<(), String> {
let provider = std::env::args().nth(1).unwrap_or_else(|| "openai".into());
let client = Client::default_for(&provider)?;
let cancel = Cancel::default();
let request = Request::new("Answer in a few plain paragraphs.")
.user("Why did the Portuguese empire decline?")
.max_output_tokens(600)
.cancel(cancel.clone());
let (tx, rx) = channel::<Chunk>();
let started = Instant::now();
let worker = thread::spawn(move || {
let result = client.stream(&request, &tx);
drop(tx);
(result, client.usage())
});
let mut answer = String::new();
let mut first_token = None;
for chunk in rx {
match chunk {
Chunk::Text(text) => {
if first_token.is_none() {
first_token = Some(started.elapsed());
}
print!("{text}");
answer.push_str(&text);
if answer.len() > PATIENCE && !cancel.load(Ordering::Relaxed) {
cancel.store(true, Ordering::Relaxed);
eprintln!("\n\n[that's enough — cancelling]");
}
}
Chunk::Tool(call) => eprintln!("\n[tool: {}]", call.name),
Chunk::Done {
input_tokens,
output_tokens,
} => eprintln!("\n\n[done: {input_tokens} in, {output_tokens} out]"),
Chunk::Failed(modelbridge::Error::Cancelled) => {
eprintln!("[stopped early, keeping {} chars]", answer.len())
}
Chunk::Failed(error) => eprintln!("\n[failed: {error}]"),
}
}
let (result, usage) = worker.join().map_err(|_| "the worker thread panicked")?;
match result {
Err(modelbridge::Error::Cancelled) | Ok(()) => {}
Err(error) => return Err(error.to_string()),
}
if let Some(latency) = first_token {
eprintln!("first token after {} ms", latency.as_millis());
}
eprintln!("{} tokens, about ${:.4}", usage.total(), usage.dollars());
Ok(())
}