alith-client 0.4.3

The Easiest Rust Interface for Local LLMs, and an Interface for Deterministic Signals from Probabilistic LLM Vibes
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

use alith_interface::{
    llms::LLMBackend,
    requests::{
        completion::{CompletionRequest, CompletionResponse},
        logit_bias::{LogitBias, LogitBiasTrait},
        req_components::{RequestConfig, RequestConfigTrait},
    },
};
use alith_prompt::LLMPrompt;
use std::sync::Arc;

#[derive(Clone)]
pub struct BasicCompletion {
    pub base_req: CompletionRequest,
}

impl BasicCompletion {
    #[inline]
    pub fn new(backend: Arc<LLMBackend>) -> Self {
        Self {
            base_req: CompletionRequest::new(backend),
        }
    }

    #[inline]
    pub fn prompt(&mut self) -> &mut LLMPrompt {
        &mut self.base_req.prompt
    }

    #[inline]
    pub async fn run(&mut self) -> crate::Result<CompletionResponse> {
        Ok(self.base_req.request().await?)
    }

    pub fn parse_response(&self, content: &str) -> crate::Result<String> {
        if content.is_empty() {
            return Err(anyhow::format_err!(
                "parse_response error: content.is_empty()"
            ));
        }

        let content = content
            .strip_prefix("assistant\n\n")
            .or_else(|| content.strip_prefix("assistant\n"))
            .or_else(|| content.strip_prefix("assistant"))
            .unwrap_or(content);

        Ok(content.trim().to_owned())
    }
}

impl RequestConfigTrait for BasicCompletion {
    fn config(&mut self) -> &mut RequestConfig {
        &mut self.base_req.config
    }

    fn reset_request(&mut self) {
        self.base_req.reset_completion_request();
    }
}

impl LogitBiasTrait for BasicCompletion {
    fn lb_mut(&mut self) -> &mut Option<LogitBias> {
        &mut self.base_req.logit_bias
    }
}