Trait Model

pub trait Model:
    Send
    + Sync
    + 'static {
    type TextStream: Stream<Item = String> + Send + Sync + Unpin + 'static;
    type SyncModel: SyncModel;

    // Required methods
    fn tokenizer(&self) -> Arc<Tokenizer>;
    fn stream_text_inner<'life0, 'life1, 'async_trait>(
        &'life0 self,
        prompt: &'life1 str,
        parameters: GenerationParameters,
    ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait;

    // Provided methods
    fn run_sync_raw(
        &self,
        _f: Box<dyn for<'a> FnOnce(&'a mut Self::SyncModel) -> Pin<Box<dyn Future<Output = ()> + 'a>> + Send>,
    ) -> Result<(), Error> { ... }
    fn generate_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        prompt: &'life1 str,
        max_tokens: Option<u32>,
        stop_on: Option<&'life2 str>,
        sampler: Arc<Mutex<dyn Sampler>>,
    ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             Self: 'async_trait { ... }
    fn generate_text_inner<'life0, 'life1, 'async_trait>(
        &'life0 self,
        prompt: &'life1 str,
        parameters: GenerationParameters,
    ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn stream_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        _prompt: &'life1 str,
        _max_tokens: Option<u32>,
        _stop_on: Option<&'life2 str>,
        _sampler: Arc<Mutex<dyn Sampler>>,
    ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             Self: 'async_trait { ... }
    fn chat_markers(&self) -> Option<ChatMarkers> { ... }
}

Text Generation Models

Model and ModelExt are the core traits for text generation models. Any model that implements these traits can be used with Kalosm.

The simplest way to use a model is to create a Model and call ModelExt::stream_text to stream text from it:

use kalosm::language::*;
#[tokio::main]
async fn main() {
    let mut llm = Llama::new().await.unwrap();
    let prompt = "The following is a 300 word essay about why the capital of France is Paris:";
    print!("{prompt}");
    let mut stream = llm
        // Any model that implements the Model trait can be used to stream text
        .stream_text(prompt)
        // You can pass parameters to the model to control the output
        .with_max_length(300)
        // And run .await to start streaming
        .await
        .unwrap();
    // You can then use the stream however you need. to_std_out will print the text to the console as it is generated
    stream.to_std_out().await.unwrap();
}

Tasks

You can define a Task with a description then run it with an input. The task will cache the description to repeated calls faster. Tasks work with both chat and non-chat models, but they tend to perform significantly better with chat models.

// First create a model. Task can work with any type of model, but chat models tend to work better
let model = Llama::new_chat().await.unwrap();
// Create a new task that 
let task = Task::new("You take a long description and summarize it into a single short sentence");
let mut output = task.run("You can define a Task with a description then run it with an input. The task will cache the description to repeated calls faster. Tasks work with both chat and non-chat models, but they tend to perform significantly better with chat models.", &model);
// Then stream the output to the console
output.to_std_out().await.unwrap();

Structured Generation

Structured generation gives you more control over the output of the text generation. You have a few different ways to use structured generation:

1. Derive a parser for your data
2. Create a parser from the set of prebuilt combinators
3. Create a parser from a regex

The simplest way to get started is to derive a parser for your data:

use kalosm::language::*;
#[derive(Parse, Clone)]
struct Pet {
    name: String,
    age: u32,
    description: String,
}

Then you can generate text that works with the parser in a Task:

#[tokio::main]
async fn main() {
    // First create a model. Chat models tend to work best with structured generation
    let model = Llama::new_chat().await.unwrap();
    // Then create a parser for your data. Any type that implements the `Parse` trait has the `new_parser` method
    let parser = Pet::new_parser();
    // Then create a task with the parser as constraints
    let task = Task::builder("You generate realistic JSON placeholders")
        .with_constraints(parser)
        .build();
    // Finally, run the task
    let pet: Pet = task.run("Generate a pet in the form {\"name\": \"Pet name\", \"age\": 0, \"description\": \"Pet description\"}", &model).await.unwrap();
    println!("{pet:?}");
}

Creating a Parser from the Set of Prebuilt Combinators

Kalosm also provides a set of prebuilt combinators for creating more complex parsers. You can use these combinators to create a parser with a custom format:

use kalosm::language::*;

#[tokio::main]
async fn main() {
    // First create a model. Chat models tend to work best with structured generation
    let model = Llama::new_chat().await.unwrap();
    // Then create a parser for your custom format
    let parser = LiteralParser::from("[")
        .ignore_output_then(String::new_parser())
        .then_literal(", ")
        .then(u8::new_parser())
        .then_literal(", ")
        .then(String::new_parser())
        .then_literal("]");
    // Then create a task with the parser as constraints
    let task = Task::builder("You generate realistic JSON placeholders")
        .with_constraints(parser)
        .build();
    // Finally, run the task
    let ((name, age), description) = task.run("Generate a pet in the form [\"Pet name\", age number, \"Pet description\"]", &model).await.unwrap();
    println!("{name} {age} {description}");
}

Creating a Parser from a Regex

You can also create a parser from a regex:

use kalosm::language::*;

#[tokio::main]
async fn main() {
    // First create a model. Chat models tend to work best with structured generation
    let model = Llama::new_chat().await.unwrap();
    // Then create a parser for your data. Any 
    let parser = RegexParser::new(r"\[(\w+), (\d+), (\w+)\]").unwrap();
    // Then create a task with the parser as constraints
    let task = Task::builder("You generate realistic JSON placeholders")
        .with_constraints(parser)
        .build();
    // Finally, run the task. Unlike derived and custom parsers, regex parsers do not provide a useful output type
    task.run("Generate a pet in the form [\"Pet name\", age number, \"Pet description\"]", &model).to_std_out().await.unwrap();
}

Required Associated Types

type TextStream: Stream<Item = String> + Send + Sync + Unpin + 'static

The type of stream that this model generates.

type SyncModel: SyncModel

The raw sync model that backs this model.

Required Methods

fn tokenizer(&self) -> Arc<Tokenizer>

Get the tokenizer associated with this model to use for constrained generation.

fn stream_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters, ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

See ModelExt::stream_text for nicer API with an example.

Provided Methods

fn run_sync_raw( &self, _f: Box<dyn for<'a> FnOnce(&'a mut Self::SyncModel) -> Pin<Box<dyn Future<Output = ()> + 'a>> + Send>, ) -> Result<(), Error>

Run some code synchronously with the model.

See ModelExt::run_sync for nicer API with an example.

fn generate_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, prompt: &'life1 str, max_tokens: Option<u32>, stop_on: Option<&'life2 str>, sampler: Arc<Mutex<dyn Sampler>>, ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn generate_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters, ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

See ModelExt::generate_text for nicer API with an example.

fn stream_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, _prompt: &'life1 str, _max_tokens: Option<u32>, _stop_on: Option<&'life2 str>, _sampler: Arc<Mutex<dyn Sampler>>, ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn chat_markers(&self) -> Option<ChatMarkers>

Returns the chat markers to use for the model if this is a chat model.

Trait Implementations

impl Model for Box<dyn Model<TextStream = ChannelTextStream, SyncModel = Box<dyn SyncModel<Session = AnySession>>> + Send>

type TextStream = ChannelTextStream

The type of stream that this model generates.

type SyncModel = Box<dyn SyncModel<Session = AnySession>>

The raw sync model that backs this model.

fn tokenizer(&self) -> Arc<Tokenizer>

Get the tokenizer associated with this model to use for constrained generation.

fn stream_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters, ) -> Pin<Box<dyn Future<Output = Result<<Box<dyn Model<TextStream = ChannelTextStream, SyncModel = Box<dyn SyncModel<Session = AnySession>>> + Send> as Model>::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Box<dyn Model<TextStream = ChannelTextStream, SyncModel = Box<dyn SyncModel<Session = AnySession>>> + Send>: 'async_trait,

Generate text with the given prompt.

fn run_sync_raw( &self, _f: Box<dyn for<'a> FnOnce(&'a mut Self::SyncModel) -> Pin<Box<dyn Future<Output = ()> + 'a>> + Send>, ) -> Result<(), Error>

Run some code synchronously with the model.

fn generate_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, prompt: &'life1 str, max_tokens: Option<u32>, stop_on: Option<&'life2 str>, sampler: Arc<Mutex<dyn Sampler>>, ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn generate_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters, ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn stream_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, _prompt: &'life1 str, _max_tokens: Option<u32>, _stop_on: Option<&'life2 str>, _sampler: Arc<Mutex<dyn Sampler>>, ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn chat_markers(&self) -> Option<ChatMarkers>

Returns the chat markers to use for the model if this is a chat model.

Implementations on Foreign Types

impl Model for Box<dyn Model<TextStream = ChannelTextStream, SyncModel = Box<dyn SyncModel<Session = AnySession>>> + Send>

type TextStream = ChannelTextStream

type SyncModel = Box<dyn SyncModel<Session = AnySession>>

fn tokenizer(&self) -> Arc<Tokenizer>

fn stream_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters, ) -> Pin<Box<dyn Future<Output = Result<<Box<dyn Model<TextStream = ChannelTextStream, SyncModel = Box<dyn SyncModel<Session = AnySession>>> + Send> as Model>::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Box<dyn Model<TextStream = ChannelTextStream, SyncModel = Box<dyn SyncModel<Session = AnySession>>> + Send>: 'async_trait,

Implementors

impl Model for Phi

type TextStream = ChannelTextStream

type SyncModel = PhiModel