bentoml 0.5.0

An unofficial async Rust client for BentoML services.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

//! Consumes a streaming endpoint chunk by chunk.
//!
//! Run with:
//!
//! ```sh
//! cargo run --example stream --features stream
//! ```

use bentoml::prelude::*;
use serde::Serialize;
use tokio_stream::StreamExt;

#[derive(Serialize)]
struct ChatRequest {
    prompt: String,
}

#[tokio::main]
async fn main() -> Result<()> {
    let client = Client::builder()
        .with_base_url("http://localhost:3000")
        .build()?;

    let request = ChatRequest {
        prompt: "Tell me about BentoML.".to_owned(),
    };

    let response = client.endpoint("chat").call(&request).await?;
    let mut stream = response.stream().text();
    while let Some(chunk) = stream.next().await {
        let chunk = chunk?;
        print!("{chunk}");
    }
    println!();

    Ok(())
}