pprog 0.0.4

An LLM pair programming server with web interface
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

use anyhow::Result;
use async_trait::async_trait;

use crate::chat::CommonMessage;
use super::types::{InferenceError, ModelResponse};

#[async_trait]
pub trait Inference: Send + Sync {
    fn new() -> Self where Self: Sized;
    
    async fn query_model(
        &self, 
        messages: Vec<CommonMessage>, 
        system_message: Option<&str>
    ) -> Result<ModelResponse, InferenceError>;

    async fn get_token_count(
        &self, 
        messages: Vec<CommonMessage>, 
        system_message: Option<&str>
    ) -> Result<u64, InferenceError>;
}