orch 0.0.16

Language model orchestration library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

//! This example demonstrates how to use the `Executor` to generate a streaming response from the LLM.
//! Run like so: `cargo run --example text_generation_stream`

use orch::{execution::*, lm::LanguageModelProvider};
use tokio_stream::StreamExt;

mod example_utils;
use example_utils::get_lm;

#[tokio::main]
async fn main() {
    let (lm, provider) = get_lm();

    if provider == LanguageModelProvider::Anthropic {
        println!("Streaming is not currently supported for Anthropic. Skipping example.");
        return;
    }

    let prompt = "What is 2+2?";

    println!("Prompt: {prompt}");
    println!("---");

    let executor = TextExecutorBuilder::new()
        .with_lm(&*lm)
        .try_build()
        .unwrap();
    let mut response = executor
        .execute_stream(prompt)
        .await
        .expect("Execution failed");

    let mut response_text = String::new();
    println!("Response:");
    while let Some(chunk) = response.stream.next().await {
        match chunk {
            Ok(chunk) => {
                print!("{chunk}");
                response_text.push_str(&chunk);
            }
            Err(e) => {
                println!("Error: {e}");
                break;
            }
        }
    }
    println!();

    assert!(!response_text.is_empty());
}