Trait TextCompletionSession

pub trait TextCompletionSession {
    type Error: Send + Sync + 'static;

    // Required methods
    fn write_to(&self, into: &mut Vec<u8>) -> Result<(), Self::Error>;
    fn from_bytes(bytes: &[u8]) -> Result<Self, Self::Error>
       where Self: Sized;
    fn try_clone(&self) -> Result<Self, Self::Error>
       where Self: Sized;

    // Provided method
    fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> { ... }
}

Text Completion Session

The TextCompletionSession trait holds the state of a text completion model after it has been fed some text. It can be used in combination with TextCompletionModel to feed text and cache the results.

Saving and Loading Sessions

Sessions can be serialized and deserialized to and from bytes using the TextCompletionSession::to_bytes and TextCompletionSession::from_bytes methods. This can be useful for saving and loading sessions to disk. Caching a session avoids re-processing the text again when the session is resumed.

use kalosm::language::*;
use std::io::Write;

#[tokio::main]
async fn main() {
    let mut llm = Llama::new().await.unwrap();
    let mut session = llm.new_session().unwrap();

    // Feed some text into the session
    llm.stream_text_with_callback(&mut session, "The capital of France is ", GenerationParameters::new().with_max_length(0), |_| Ok(())).await.unwrap();

    // Save the session to bytes
    let session_as_bytes = session.to_bytes().unwrap();
    
    // Load the session from bytes
    let mut session = LlamaSession::from_bytes(&session_as_bytes).unwrap();

    // Feed some more text into the session
    llm.stream_text_with_callback(&mut session, "The capital of France is ", GenerationParameters::new(), |token| {println!("{token}"); Ok(())}).await.unwrap();
}

Cloning Sessions

Not all models support cloning sessions, but if a model does support cloning sessions, you can clone a session using the TextCompletionSession::try_clone method to clone a session state while retaining the original session.

use kalosm::language::*;
use std::io::Write;

#[tokio::main]
async fn main() {
    let mut llm = Llama::new().await.unwrap();
    let mut session = llm.new_session().unwrap();

    // Feed some text into the session
    llm.stream_text_with_callback(&mut session, "The capital of France is ", GenerationParameters::new().with_max_length(0), |_| Ok(())).await.unwrap();

    // Clone the session
    let cloned_session = session.try_clone().unwrap();

    // Feed some more text into the cloned session
    llm.stream_text_with_callback(&mut session, "The capital of France is ", GenerationParameters::new(), |token| {println!("{token}"); Ok(())}).await.unwrap();
}

Required Associated Types

type Error: Send + Sync + 'static

The type of error the session may return during operations.

Required Methods

fn write_to(&self, into: &mut Vec<u8>) -> Result<(), Self::Error>

Serialize the session into bytes. This method is identical to TextCompletionSession::to_bytes except it can re-use an existing Vec buffer.

fn from_bytes(bytes: &[u8]) -> Result<Self, Self::Error>

where Self: Sized,

Loading sessions

Sessions can be deserialized to and from bytes using the TextCompletionSession::from_bytes method. Caching a session avoids re-processing the text again when the session is resumed.

use kalosm::language::*;
use std::io::Write;

#[tokio::main]
async fn main() {
    let mut llm = Llama::new().await.unwrap();
    // Load a text completion session from a file
    let mut session =
        LlamaSession::from_bytes(std::fs::read("session.bin").unwrap().as_slice()).unwrap();

    // Feed some more text into the session
    llm.stream_text_with_callback(
        &mut session,
        "The capital of France is ",
        GenerationParameters::new(),
        |token| {
            println!("{token}");
            Ok(())
        },
    )
    .await
    .unwrap();
}

fn try_clone(&self) -> Result<Self, Self::Error>

where Self: Sized,

Cloning Sessions

Not all models support cloning sessions, but if a model does support cloning sessions, you can clone a session using the TextCompletionSession::try_clone method to clone a session state while retaining the original session.

use kalosm::language::*;
use std::io::Write;

#[tokio::main]
async fn main() {
    let mut llm = Llama::new().await.unwrap();
    let mut session = llm.new_session().unwrap();

    // Feed some text into the session
    llm.stream_text_with_callback(
        &mut session,
        "The capital of France is ",
        GenerationParameters::new().with_max_length(0),
        |_| Ok(()),
    )
    .await
    .unwrap();

    // Clone the session
    let cloned_session = session.try_clone().unwrap();

    // Feed some more text into the cloned session
    llm.stream_text_with_callback(
        &mut session,
        "The capital of France is ",
        GenerationParameters::new(),
        |token| {
            println!("{token}");
            Ok(())
        },
    )
    .await
    .unwrap();
}

Provided Methods

fn to_bytes(&self) -> Result<Vec<u8>, Self::Error>

Loading sessions

Sessions can be deserialized to and from bytes using the TextCompletionSession::from_bytes method. Caching a session avoids re-processing the text again when the session is resumed.

use kalosm::language::*;
use std::io::Write;

#[tokio::main]
async fn main() {
    let mut llm = Llama::new().await.unwrap();
    let mut session = llm.new_session().unwrap();

    // Feed some text into the session
    llm.stream_text_with_callback(
        &mut session,
        "The capital of France is ",
        GenerationParameters::new().with_max_length(0),
        |_| Ok(()),
    )
    .await
    .unwrap();

    // Save the session to bytes
    let session_as_bytes = session.to_bytes().unwrap();

    // And write those bytes to a file
    std::fs::write("session.bin", session_as_bytes).unwrap();
}

Implementations on Foreign Types

impl TextCompletionSession for ()

type Error = Infallible

fn write_to(&self, _into: &mut Vec<u8>) -> Result<(), Self::Error>

fn from_bytes(_bytes: &[u8]) -> Result<(), Self::Error>

fn try_clone(&self) -> Result<(), Self::Error>

Implementors

impl TextCompletionSession for BoxedTextCompletionSession

type Error = Box<dyn Error + Sync + Send>