halo-agent 0.1.0

Halo Agent and Workflow execution kernel
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

use std::fmt;

use serde::Deserialize;

use super::compatible::{Connection, Kind};
use crate::model;

/// Configuration for a VLLM OpenAI-compatible server.
#[derive(Clone, Deserialize)]
pub struct Config {
    pub provider_id: i64,
    pub provider_name: String,
    pub base_url: String,
    pub api_key: String,
    /// Total HTTP request timeout, in seconds.
    pub timeout: u64,
}

impl Default for Config {
    fn default() -> Self {
        Self {
            provider_id: 0,
            provider_name: "vllm".into(),
            base_url: "http://127.0.0.1:8000/v1".into(),
            api_key: String::new(),
            timeout: 120,
        }
    }
}

impl fmt::Debug for Config {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Config")
            .field("provider_id", &self.provider_id)
            .field("provider_name", &self.provider_name)
            .field("api_key", &"[redacted]")
            .field("timeout", &self.timeout)
            .finish_non_exhaustive()
    }
}

#[derive(Clone)]
pub struct Vllm {
    connection: Connection,
}

impl Vllm {
    /// Start configuring a VLLM provider connection.
    pub fn builder() -> Builder {
        Builder {
            config: Config::default(),
        }
    }

    pub fn new(config: Config) -> Result<Self, model::Error> {
        Ok(Self {
            connection: Connection::new(
                Kind::Vllm,
                config.provider_id,
                &config.provider_name,
                &config.base_url,
                &config.api_key,
                config.timeout,
            )?,
        })
    }

    pub fn chat(&self, config: model::chat::Config) -> Result<chat::Chat, model::Error> {
        self.connection.chat(config)
    }

    pub fn embed(&self, config: model::embed::Config) -> Result<embed::Embed, model::Error> {
        self.connection.embed(config)
    }
}

/// Fluent configuration for a VLLM provider connection.
pub struct Builder {
    config: Config,
}

impl Builder {
    pub fn base_url(mut self, base_url: impl Into<String>) -> Self {
        self.config.base_url = base_url.into();
        self
    }

    pub fn api_key(mut self, api_key: impl Into<String>) -> Self {
        self.config.api_key = api_key.into();
        self
    }

    pub fn timeout(mut self, timeout: u64) -> Self {
        self.config.timeout = timeout;
        self
    }

    pub fn build(self) -> Result<Vllm, model::Error> {
        Vllm::new(self.config)
    }
}

pub mod chat {
    pub use super::super::compatible::{Chat, Events};
}

pub mod embed {
    pub use super::super::compatible::Embed;
}