Struct ChatResponseBuilder

pub struct ChatResponseBuilder<'a, M: CreateChatSession, Constraints = NoConstraints, Sampler = GenerationParameters> { /* private fields */ }

A builder for a chat response. This is returned by Chat::add_message and can be modified until you start awaiting the response.

Example

let model = Llama::new_chat().await.unwrap();
let mut chat = model
    .chat()
    .with_system_prompt("The first prime number larger than 4");

// Add a message to the chat session with the given message
let mut response = chat(&prompt_input("\n> ").unwrap());
// Before you start streaming the response, you can add constraints to the response
let mut response = response.with_constraints(i32::new_parser());
// Once you start streaming the response, the generation starts
response.to_std_out().await.unwrap();
// The response can be awaited to get the final (typed) result
let all_text: i32 = response.await.unwrap();
println!("{all_text}");

Implementations

impl<'a, M: CreateChatSession, Constraints, Sampler> ChatResponseBuilder<'a, M, Constraints, Sampler>

pub fn with_constraints<NewConstraints>( self, constraints: NewConstraints, ) -> ChatResponseBuilder<'a, M, NewConstraints, Sampler>

Constrains the model’s response to the given parser. This can be used to make the model start with a certain phrase, or to make the model respond in a certain way.

Example

let model = Llama::new_chat().await.unwrap();
// Create constraints that parses Yes! and then stops on the end of the assistant's response
let constraints = LiteralParser::new("Yes!").then(model.default_assistant_constraints());
// Create a chat session with the model
let mut chat = model.chat();

// Chat with the user
loop {
    // Set the constraints for the response
    let mut output_stream =
        chat(&prompt_input("\n> ").unwrap()).with_constraints(constraints.clone());
    output_stream.to_std_out().await.unwrap();
}

pub fn typed<T>( self, ) -> ChatResponseBuilder<'a, M, <M as CreateDefaultChatConstraintsForType<T>>::DefaultConstraints, Sampler>

where M: CreateDefaultChatConstraintsForType<T>,

Constrains the model’s response to the the default parser for the given type. This can be used to make the model return a specific type.

Example

#[derive(Schema, Parse, Clone, Debug)]
struct Pet {
    name: String,
    age: u32,
    description: String,
}

// First create a model
let model = Llama::new_chat().await.unwrap();
// Create a chat session
let mut chat = model.chat().with_system_prompt(format!(
    "The assistant turns descriptions of pets into JSON in this format {}",
    Pet::schema()
));
// Finally, add a message and make it typed to get the parsed response
let pet: Pet = chat("JSON for an adorable dog named ruffles")
    .typed()
    .await
    .unwrap();
println!("{pet:?}");

pub fn with_sampler<NewSampler>( self, sampler: NewSampler, ) -> ChatResponseBuilder<'a, M, Constraints, NewSampler>

Sets the sampler to use for generating responses. The sampler determines how tokens are choosen from the probability distribution the model generates. They can be used to make the model more or less predictable and prevent repetition.

Example

let model = Llama::new_chat().await.unwrap();
// Create the sampler to use for the chat session
let sampler = GenerationParameters::default();

// Create a chat session with the model
let mut chat = model.chat();

// Chat with the user
loop {
    // Stream a response with the sampler
    let mut output_stream = chat(&prompt_input("\n> ").unwrap()).with_sampler(sampler.clone());
    output_stream.to_std_out().await.unwrap();
}

Trait Implementations

impl<'a, M, Constraints, Sampler> IntoFuture for ChatResponseBuilder<'a, M, Constraints, Sampler>

where Constraints: ModelConstraints + Send + Sync + Unpin + 'static, Sampler: Send + Unpin + 'static, M: StructuredChatModel<Constraints, Sampler> + Send + Sync + Clone + Unpin + 'static, M::ChatSession: Clone + Send + Sync + Unpin + 'static, Constraints::Output: Send + 'static,

type Output = Result<<Constraints as ModelConstraints>::Output, <M as CreateChatSession>::Error>

The output that the future will produce on completion.

type IntoFuture = Pin<Box<dyn Future<Output = <ChatResponseBuilder<'a, M, Constraints, Sampler> as IntoFuture>::Output> + Send + 'a>>

Which kind of future are we turning this into?

fn into_future(self) -> Self::IntoFuture

Creates a future from a value.

impl<'a, M, Sampler> IntoFuture for ChatResponseBuilder<'a, M, NoConstraints, Sampler>

where Sampler: Send + Unpin + 'static, M: ChatModel<Sampler> + Send + Sync + Unpin + Clone + 'static, M::ChatSession: Clone + Send + Sync + Unpin + 'static,

type Output = Result<String, <M as CreateChatSession>::Error>

The output that the future will produce on completion.

type IntoFuture = Pin<Box<dyn Future<Output = <ChatResponseBuilder<'a, M, NoConstraints, Sampler> as IntoFuture>::Output> + Send + 'a>>

Which kind of future are we turning this into?

fn into_future(self) -> Self::IntoFuture

Creates a future from a value.

impl<M, Constraints, Sampler> Stream for ChatResponseBuilder<'_, M, Constraints, Sampler>

where Constraints: ModelConstraints + Send + Sync + Unpin + 'static, Sampler: Send + Unpin + 'static, M: StructuredChatModel<Constraints, Sampler> + Send + Sync + Clone + Unpin + 'static, M::ChatSession: Clone + Send + Sync + Unpin + 'static, M::Error: Send + Sync + Unpin, Constraints::Output: Send + 'static,

type Item = String

Values yielded by the stream.

fn poll_next( self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll<Option<Self::Item>>

Attempt to pull out the next value of this stream, registering the current task for wakeup if the value is not yet available, and returning None if the stream is exhausted.

fn size_hint(&self) -> (usize, Option<usize>)

Returns the bounds on the remaining length of the stream.

impl<M, Sampler> Stream for ChatResponseBuilder<'_, M, NoConstraints, Sampler>

where Sampler: Send + Unpin + 'static, M: ChatModel<Sampler> + Send + Sync + Clone + Unpin + 'static, M::ChatSession: Send + Sync + Unpin + 'static, M::Error: Send + Sync + Unpin,

type Item = String

Values yielded by the stream.

fn poll_next( self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll<Option<Self::Item>>

Attempt to pull out the next value of this stream, registering the current task for wakeup if the value is not yet available, and returning None if the stream is exhausted.

fn size_hint(&self) -> (usize, Option<usize>)

Returns the bounds on the remaining length of the stream.