Struct TextCompletionBuilder

pub struct TextCompletionBuilder<M: CreateTextCompletionSession, Constraints = NoConstraints, Sampler = GenerationParameters> { /* private fields */ }

A builder for a text completion response. This is returned by TextCompletionModelExt::complete and can be modified with TextCompletionBuilder::with_sampler and TextCompletionBuilder::with_constraints until you start awaiting the response.

Once you are done setting up the response, you can call .await for the full text completion, or ModelConstraints::Output:

use kalosm::language::*;

#[tokio::main]
async fn main() {
    let mut llm = Llama::new().await.unwrap();
    let prompt = "The following is a 300 word essay about why the capital of France is Paris:";
    print!("{prompt}");
    let mut completion = llm.complete(prompt).await.unwrap();
    println!("{completion}");
}

Or use the response as a Stream:

use kalosm::language::*;
use std::io::Write;

#[tokio::main]
async fn main() {
    let mut llm = Llama::new().await.unwrap();
    let prompt = "The following is a 300 word essay about why the capital of France is Paris:";
    print!("{prompt}");
    let mut completion = llm.complete(prompt);
    while let Some(token) = completion.next().await {
        print!("{token}");
        std::io::stdout().flush().unwrap();
    }
}

Implementations

impl<M: CreateTextCompletionSession, Constraints, Sampler> TextCompletionBuilder<M, Constraints, Sampler>

pub fn with_constraints<NewConstraints: ModelConstraints>( self, constraints: NewConstraints, ) -> TextCompletionBuilder<M, NewConstraints, Sampler>

Constrains the model’s response to the given parser. This can be used to make the model start with a certain phrase, or to make the model respond in a certain way.

Example

#[derive(Parse, Clone, Debug)]
struct Pet {
    name: String,
    age: u32,
    description: String,
}

// First create a model
let model = Llama::new().await.unwrap();
// Then create a parser for your data. Any type that implements the `Parse` trait has the `new_parser` method
let parser = Pet::new_parser();
// Create a text completion stream with the constraints
let description = model.complete("JSON for an adorable dog named ruffles: ")
    .with_constraints(parser);
// Finally, await the stream to get the parsed response
let pet: Pet = description.await.unwrap();
println!("{pet:?}");

pub fn typed<T>( self, ) -> TextCompletionBuilder<M, <M as CreateDefaultCompletionConstraintsForType<T>>::DefaultConstraints, Sampler>

where M: CreateDefaultCompletionConstraintsForType<T>,

Constrains the model’s response to the the default parser for the given type. This can be used to make the model return a specific type.

Example

#[derive(Parse, Clone, Debug)]
struct Pet {
    name: String,
    age: u32,
    description: String,
}

// First create a model
let model = Llama::new().await.unwrap();
// Create a text completion stream with the typed response
let description = model
    .complete("JSON for an adorable dog named ruffles: ")
    .typed();
// Finally, await the stream to get the parsed response
let pet: Pet = description.await.unwrap();
println!("{pet:?}");

pub fn with_sampler<NewSampler>( self, sampler: NewSampler, ) -> TextCompletionBuilder<M, Constraints, NewSampler>

Sets the sampler to use for generating responses. The sampler determines how tokens are chosen from the probability distribution the model generates. They can be used to make the model more or less predictable and prevent repetition.

Example

let model = Llama::new().await.unwrap();
// Create the sampler to use for the text completion
let sampler = GenerationParameters::default().sampler();
// Create a completion request with the sampler
let mut stream = model
    .complete("Here is a list of 5 primes: ")
    .with_sampler(sampler);
stream.to_std_out().await.unwrap();

Trait Implementations

impl<M, Constraints, Sampler> IntoFuture for TextCompletionBuilder<M, Constraints, Sampler>

where Constraints: ModelConstraints + Send + Sync + Unpin + 'static, Sampler: Send + Unpin + 'static, M: StructuredTextCompletionModel<Constraints, Sampler> + Send + Sync + Unpin + 'static, M::Session: Clone + Send + Sync + Unpin + 'static, Constraints::Output: Send + 'static,

type Output = Result<<Constraints as ModelConstraints>::Output, <M as CreateTextCompletionSession>::Error>

The output that the future will produce on completion.

type IntoFuture = Pin<Box<dyn Future<Output = <TextCompletionBuilder<M, Constraints, Sampler> as IntoFuture>::Output> + Send>>

Which kind of future are we turning this into?

fn into_future(self) -> Self::IntoFuture

Creates a future from a value.

impl<M, Sampler> IntoFuture for TextCompletionBuilder<M, NoConstraints, Sampler>

where Sampler: Send + Unpin + 'static, M: TextCompletionModel<Sampler> + Send + Sync + Unpin + 'static, M::Session: Clone + Send + Sync + Unpin + 'static,

type Output = Result<String, <M as CreateTextCompletionSession>::Error>

The output that the future will produce on completion.

type IntoFuture = Pin<Box<dyn Future<Output = <TextCompletionBuilder<M, NoConstraints, Sampler> as IntoFuture>::Output> + Send>>

Which kind of future are we turning this into?

fn into_future(self) -> Self::IntoFuture

Creates a future from a value.

impl<M, Constraints, Sampler> Stream for TextCompletionBuilder<M, Constraints, Sampler>

where Constraints: ModelConstraints + Send + Sync + Unpin + 'static, Sampler: Send + Unpin + 'static, M: StructuredTextCompletionModel<Constraints, Sampler> + Send + Sync + Unpin + 'static, M::Session: Clone + Send + Sync + Unpin + 'static, Constraints::Output: Send + 'static,

type Item = String

Values yielded by the stream.

fn poll_next( self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll<Option<Self::Item>>

Attempt to pull out the next value of this stream, registering the current task for wakeup if the value is not yet available, and returning None if the stream is exhausted.

fn size_hint(&self) -> (usize, Option<usize>)

Returns the bounds on the remaining length of the stream.

impl<M, Sampler> Stream for TextCompletionBuilder<M, NoConstraints, Sampler>

where Sampler: Send + Unpin + 'static, M: TextCompletionModel<Sampler> + Send + Sync + Unpin + 'static, M::Session: Send + Sync + Unpin + 'static,

type Item = String

Values yielded by the stream.

fn poll_next( self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll<Option<Self::Item>>

Attempt to pull out the next value of this stream, registering the current task for wakeup if the value is not yet available, and returning None if the stream is exhausted.

fn size_hint(&self) -> (usize, Option<usize>)

Returns the bounds on the remaining length of the stream.