orch 0.0.16

Language model orchestration library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

use std::pin::Pin;

use thiserror::Error;
use tokio_stream::Stream;

use crate::{
    alignment::AlignmentError,
    lm::{LanguageModel, LanguageModelError, OllamaError, TextCompleteOptions},
};

#[derive(Debug, Error)]
pub enum ExecutorError {
    #[error("{0}")]
    General(LanguageModelError),

    #[error("{0}")]
    LanguageModelError(LanguageModelError),

    #[error("Error when calling Ollama API: {0}")]
    OllamaApi(String),

    #[error("Parsing LM response failed: {0}")]
    Parsing(String),

    #[error("Alignment error: {0}")]
    Alignment(AlignmentError),
}

impl From<LanguageModelError> for ExecutorError {
    fn from(val: LanguageModelError) -> Self {
        match val {
            LanguageModelError::Ollama(OllamaError::Api(e)) => ExecutorError::OllamaApi(e),
            e => ExecutorError::LanguageModelError(e),
        }
    }
}

pub(crate) trait Executor<'a> {
    /// Generates a text completion from the LLM (non-streaming).
    async fn text_complete(
        &self,
        prompt: &str,
    ) -> Result<ExecutorTextCompleteResponse<String>, ExecutorError> {
        text_complete(self.lm(), prompt, &self.system_prompt()).await
    }

    /// System prompt (instructions) for the model.
    fn system_prompt(&self) -> String;

    fn lm(&self) -> &'a dyn LanguageModel;
}

// TODO: Support context for completions (e.g., IDs of past conversations in Ollama).
pub struct ExecutorContext;

pub struct ExecutorTextCompleteResponse<T> {
    pub content: T,
    pub context: ExecutorContext,
}

pub struct ExecutorTextCompleteStreamResponse {
    pub stream: Pin<Box<dyn Stream<Item = Result<String, LanguageModelError>> + Send>>,
    pub context: ExecutorContext,
}

pub async fn text_complete<'a>(
    lm: &'a dyn LanguageModel,
    prompt: &str,
    system_prompt: &str,
) -> Result<ExecutorTextCompleteResponse<String>, ExecutorError> {
    let options = TextCompleteOptions {
        ..Default::default()
    };
    let response = lm
        .text_complete(prompt, system_prompt, options)
        .await
        .map_err(ExecutorError::from)?;
    Ok(ExecutorTextCompleteResponse {
        content: response.text,
        context: ExecutorContext {},
    })
}

pub(crate) async fn generate_embedding<'a>(
    lm: &'a dyn LanguageModel,
    prompt: &str,
) -> Result<Vec<f32>, ExecutorError> {
    let response = lm
        .generate_embedding(prompt)
        .await
        .map_err(ExecutorError::from)?;
    Ok(response)
}