Trait kalosm_language::prelude::Model

pub trait Model: Send + Sync + 'static {
    type TextStream: Stream<Item = String> + Send + Sync + Unpin + 'static;
    type SyncModel: SyncModel;

    // Required methods
    fn tokenizer(&self) -> Arc<dyn Tokenizer + Sync + Send>;
    fn stream_text_inner<'life0, 'life1, 'async_trait>(
        &'life0 self,
        prompt: &'life1 str,
        parameters: GenerationParameters
    ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait;

    // Provided methods
    fn run_sync_raw(
        &self,
        _f: Box<dyn for<'a> FnOnce(&'a mut Self::SyncModel) -> Pin<Box<dyn Future<Output = ()> + 'a>> + Send>
    ) -> Result<(), Error> { ... }
    fn generate_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        prompt: &'life1 str,
        max_tokens: Option<u32>,
        stop_on: Option<&'life2 str>,
        sampler: Arc<Mutex<dyn Sampler>>
    ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             Self: 'async_trait { ... }
    fn generate_text_inner<'life0, 'life1, 'async_trait>(
        &'life0 self,
        prompt: &'life1 str,
        parameters: GenerationParameters
    ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn stream_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        _prompt: &'life1 str,
        _max_tokens: Option<u32>,
        _stop_on: Option<&'life2 str>,
        _sampler: Arc<Mutex<dyn Sampler>>
    ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             Self: 'async_trait { ... }
    fn chat_markers(&self) -> Option<ChatMarkers> { ... }
}

A model that can be used to generate text with an associated tokenizer.

The model may support using a custom sampler. If a specific model does not support a specific method, it will return an error.

Required Associated Types

type TextStream: Stream<Item = String> + Send + Sync + Unpin + 'static

The type of stream that this model generates.

type SyncModel: SyncModel

The raw sync model that backs this model.

Required Methods

fn tokenizer(&self) -> Arc<dyn Tokenizer + Sync + Send>

Get the tokenizer associated with this model to use for constrained generation.

fn stream_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

See ModelExt::stream_text for nicer API with an example.

Provided Methods

fn run_sync_raw( &self, _f: Box<dyn for<'a> FnOnce(&'a mut Self::SyncModel) -> Pin<Box<dyn Future<Output = ()> + 'a>> + Send> ) -> Result<(), Error>

Run some code synchronously with the model.

See ModelExt::run_sync for nicer API with an example.

fn generate_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, prompt: &'life1 str, max_tokens: Option<u32>, stop_on: Option<&'life2 str>, sampler: Arc<Mutex<dyn Sampler>> ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn generate_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

See ModelExt::generate_text for nicer API with an example.

fn stream_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, _prompt: &'life1 str, _max_tokens: Option<u32>, _stop_on: Option<&'life2 str>, _sampler: Arc<Mutex<dyn Sampler>> ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn chat_markers(&self) -> Option<ChatMarkers>

Returns the chat markers to use for the model if this is a chat model.

Trait Implementations

impl Model for Box<dyn Model<SyncModel = Box<dyn SyncModel<Session = AnySession>>, TextStream = ChannelTextStream<String>> + Send>

type TextStream = ChannelTextStream<String>

The type of stream that this model generates.

type SyncModel = Box<dyn SyncModel<Session = AnySession>>

The raw sync model that backs this model.

fn tokenizer(&self) -> Arc<dyn Tokenizer + Sync + Send>

Get the tokenizer associated with this model to use for constrained generation.

fn stream_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters ) -> Pin<Box<dyn Future<Output = Result<<Box<dyn Model<SyncModel = Box<dyn SyncModel<Session = AnySession>>, TextStream = ChannelTextStream<String>> + Send> as Model>::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Box<dyn Model<SyncModel = Box<dyn SyncModel<Session = AnySession>>, TextStream = ChannelTextStream<String>> + Send>: 'async_trait,

Generate text with the given prompt.

fn run_sync_raw( &self, _f: Box<dyn for<'a> FnOnce(&'a mut Self::SyncModel) -> Pin<Box<dyn Future<Output = ()> + 'a>> + Send> ) -> Result<(), Error>

Run some code synchronously with the model.

fn generate_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, prompt: &'life1 str, max_tokens: Option<u32>, stop_on: Option<&'life2 str>, sampler: Arc<Mutex<dyn Sampler>> ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn generate_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn stream_text_with_sampler<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, _prompt: &'life1 str, _max_tokens: Option<u32>, _stop_on: Option<&'life2 str>, _sampler: Arc<Mutex<dyn Sampler>> ) -> Pin<Box<dyn Future<Output = Result<Self::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, Self: 'async_trait,

Generate text with the given prompt.

fn chat_markers(&self) -> Option<ChatMarkers>

Returns the chat markers to use for the model if this is a chat model.

Implementations on Foreign Types

impl Model for Box<dyn Model<SyncModel = Box<dyn SyncModel<Session = AnySession>>, TextStream = ChannelTextStream<String>> + Send>

type TextStream = ChannelTextStream<String>

type SyncModel = Box<dyn SyncModel<Session = AnySession>>

fn tokenizer(&self) -> Arc<dyn Tokenizer + Sync + Send>

fn stream_text_inner<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, parameters: GenerationParameters ) -> Pin<Box<dyn Future<Output = Result<<Box<dyn Model<SyncModel = Box<dyn SyncModel<Session = AnySession>>, TextStream = ChannelTextStream<String>> + Send> as Model>::TextStream, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Box<dyn Model<SyncModel = Box<dyn SyncModel<Session = AnySession>>, TextStream = ChannelTextStream<String>> + Send>: 'async_trait,

Implementors

impl Model for Gpt3_5

type TextStream = ChannelTextStream<String>

type SyncModel = SyncModelNotSupported

impl Model for Gpt4

type TextStream = ChannelTextStream<String>

type SyncModel = SyncModelNotSupported

impl Model for Llama

type TextStream = ChannelTextStream<String>

type SyncModel = LlamaModel

impl Model for Phi

type TextStream = ChannelTextStream<String>

type SyncModel = PhiModel