Trait kalosm_language::prelude::ModelExt

pub trait ModelExt: Model + Send + Sync + 'static {
    // Provided methods
    fn generate_text<'a>(
        &'a self,
        prompt: &'a str
    ) -> GenerateTextBuilder<'a, Self>
       where Self: Sized { ... }
    fn stream_text<'a>(&'a self, prompt: &'a str) -> StreamTextBuilder<'a, Self>
       where Self: Sized { ... }
    fn run_sync(
        &self,
        f: impl for<'a> FnOnce(&'a mut Self::SyncModel) -> Pin<Box<dyn Future<Output = ()> + 'a>> + Send + 'static
    ) -> Result<(), Error> { ... }
    fn stream_structured_text<'life0, 'life1, 'async_trait, P>(
        &'life0 self,
        prompt: &'life1 str,
        parser: P
    ) -> Pin<Box<dyn Future<Output = Result<StructureParserResult<Self::TextStream, <P as Parser>::Output>, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self::TextStream: From<UnboundedReceiver<String>>,
             P: CreateParserState + Parser + Send + 'static + 'async_trait,
             <P as Parser>::PartialState: Send + 'static,
             <P as Parser>::Output: Clone + Send + 'static,
             Self: 'async_trait { ... }
    fn stream_structured_text_with_sampler<'life0, 'life1, 'async_trait, P>(
        &'life0 self,
        prompt: &'life1 str,
        parser: P,
        parser_state: <P as Parser>::PartialState,
        sampler: Arc<Mutex<dyn Sampler>>
    ) -> Pin<Box<dyn Future<Output = Result<StructureParserResult<Self::TextStream, <P as Parser>::Output>, Error>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self::TextStream: From<UnboundedReceiver<String>>,
             P: Parser + Send + 'static + 'async_trait,
             <P as Parser>::PartialState: Send + 'static,
             <P as Parser>::Output: Clone + Send + 'static,
             Self: 'async_trait { ... }
}

An extension trait for models.

Provided Methods

fn generate_text<'a>(&'a self, prompt: &'a str) -> GenerateTextBuilder<'a, Self>

where Self: Sized,

Generate text with the given prompt. This function generates a builder with extra parameters that can be set. To execute the builder, just call await on it.

use rphi::prelude::*;
use std::io::Write;

#[tokio::main]
async fn main() {
    let mut model = Phi::default();
    let prompt = "The capital of France is";
    let mut result = model.generate_text(prompt).with_max_length(300).await.unwrap();

    println!("{prompt}{result}");
}

fn stream_text<'a>(&'a self, prompt: &'a str) -> StreamTextBuilder<'a, Self>

where Self: Sized,

Generate text with the given prompt. This function generates a builder with extra parameters that can be set. To execute the builder, just call await on it.

use rphi::prelude::*;
use std::io::Write;

#[tokio::main]
async fn main() {
    let mut model = Phi::default();
    let prompt = "The capital of France is";
    let mut result = model.stream_text(prompt).with_max_length(300).await.unwrap();

    print!("{prompt}");
    while let Some(token) = result.next().await {
        print!("{token}");
        std::io::stdout().flush().unwrap();
    }
}

fn run_sync( &self, f: impl for<'a> FnOnce(&'a mut Self::SyncModel) -> Pin<Box<dyn Future<Output = ()> + 'a>> + Send + 'static ) -> Result<(), Error>

Run some code synchronously with the model.

Example

use rphi::prelude::*;
use kalosm_language_model::Model;

#[tokio::main]
async fn main() {
    let mut llm = Phi::start().await;

    let tokenizer = llm.tokenizer();
    // Start a sync task on the model
    llm.run_sync(move |llm: &mut <Phi as Model>::SyncModel| {
        Box::pin(async move {
            let question = "What is 10 + 10?";

            // Create a new session of the model
            let mut session = llm.new_session().unwrap();

            // Feed the question into the model
            let mut logits = llm.feed_text(&mut session, question, None).unwrap();

            println!("logits: {:?}", logits);
        })
    })
    .unwrap();
}

fn stream_structured_text<'life0, 'life1, 'async_trait, P>( &'life0 self, prompt: &'life1 str, parser: P ) -> Pin<Box<dyn Future<Output = Result<StructureParserResult<Self::TextStream, <P as Parser>::Output>, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self::TextStream: From<UnboundedReceiver<String>>, P: CreateParserState + Parser + Send + 'static + 'async_trait, <P as Parser>::PartialState: Send + 'static, <P as Parser>::Output: Clone + Send + 'static, Self: 'async_trait,

Generate structured text with the given prompt.

fn stream_structured_text_with_sampler<'life0, 'life1, 'async_trait, P>( &'life0 self, prompt: &'life1 str, parser: P, parser_state: <P as Parser>::PartialState, sampler: Arc<Mutex<dyn Sampler>> ) -> Pin<Box<dyn Future<Output = Result<StructureParserResult<Self::TextStream, <P as Parser>::Output>, Error>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self::TextStream: From<UnboundedReceiver<String>>, P: Parser + Send + 'static + 'async_trait, <P as Parser>::PartialState: Send + 'static, <P as Parser>::Output: Clone + Send + 'static, Self: 'async_trait,

Generate structured text with the given prompt and sampler.

Object Safety

This trait is not object safe.

Implementors

impl<M> ModelExt for M

where M: Model + Send + Sync + 'static,