llmprism 0.4.0

A unified, fluent Rust interface for integrating Large Language Model providers -- a Rust port of the PHP Prism library.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

//! The Ollama provider, for locally-hosted (or self-hosted) open-weight
//! models. Ollama exposes an OpenAI-compatible API surface --
//! `/v1/chat/completions` and `/v1/embeddings` -- so, like Phase 5's
//! `providers::openai_compatible` family (not linked here since that module
//! only exists when one of *its* features is enabled, which enabling only
//! `ollama` doesn't guarantee), this reuses [`OpenAiProvider`]'s
//! request/response handling rather than implementing a wire format of its
//! own. It's kept separate from that module (not generated by its macro) for
//! two reasons: Ollama additionally implements Embeddings (confirmed against
//! Ollama's own docs, unlike the rest of that family, where support varies
//! enough per vendor that this crate doesn't claim it), and Ollama typically
//! needs no API key at all, since it's a local server rather than a hosted
//! one.

use async_trait::async_trait;
use futures::stream::BoxStream;

use crate::embeddings::{EmbeddingsRequest, EmbeddingsResponse};
use crate::error::Error;
use crate::provider::Provider;
use crate::providers::openai::OpenAiProvider;
use crate::stream_event::StreamEvent;
use crate::text::{Step, TextRequest};

const DEFAULT_BASE_URL: &str = "http://localhost:11434/v1";

/// A [`Provider`] backed by [Ollama](https://ollama.com)'s OpenAI-compatible
/// API surface.
///
/// # Example
///
/// ```no_run
/// use llmprism::providers::ollama::OllamaProvider;
/// use llmprism::Registry;
///
/// // No API key needed for a local Ollama server.
/// let mut registry = Registry::new();
/// registry.register("ollama", OllamaProvider::new());
/// ```
///
/// [`Registry::from_env`](crate::Registry::from_env) always registers this
/// provider when the `ollama` feature is enabled -- unlike every other
/// provider, which only gets registered when its API key env var is set --
/// since Ollama typically needs no credentials at all. Set `OLLAMA_BASE_URL`
/// to point it at something other than the default local server.
pub struct OllamaProvider {
    inner: OpenAiProvider,
}

impl OllamaProvider {
    /// Creates a provider pointed at the default local Ollama server
    /// (`http://localhost:11434/v1`), with no API key.
    pub fn new() -> Self {
        Self::with_base_url(DEFAULT_BASE_URL)
    }

    /// Creates a provider pointed at a different base URL -- a non-default
    /// port, or a remote Ollama instance -- still with no API key.
    pub fn with_base_url(base_url: impl Into<String>) -> Self {
        Self::with_api_key_and_base_url("", base_url)
    }

    /// Creates a provider with an API key -- useful if your Ollama instance
    /// sits behind an authenticating proxy, since Ollama itself has no
    /// concept of API keys.
    pub fn with_api_key_and_base_url(
        api_key: impl Into<String>,
        base_url: impl Into<String>,
    ) -> Self {
        Self {
            inner: OpenAiProvider::with_name_and_base_url("ollama", api_key, base_url),
        }
    }

    /// Replaces the underlying HTTP client -- an escape hatch for anything
    /// [`build_http_client`](crate::client::build_http_client) doesn't cover
    /// (a request timeout, a proxy, a custom retry policy). See
    /// [`OpenAiProvider::with_client`] for a full example; this works the
    /// same way.
    pub fn with_client(mut self, client: reqwest_middleware::ClientWithMiddleware) -> Self {
        self.inner = self.inner.with_client(client);
        self
    }
}

impl Default for OllamaProvider {
    fn default() -> Self {
        Self::new()
    }
}

#[async_trait]
impl Provider for OllamaProvider {
    fn name(&self) -> &str {
        "ollama"
    }

    async fn text_step(&self, request: &TextRequest) -> Result<Step, Error> {
        self.inner.text_step(request).await
    }

    async fn stream_text_once(
        &self,
        request: &TextRequest,
    ) -> Result<BoxStream<'static, Result<StreamEvent, Error>>, Error> {
        self.inner.stream_text_once(request).await
    }

    async fn embeddings(&self, request: EmbeddingsRequest) -> Result<EmbeddingsResponse, Error> {
        self.inner.embeddings(request).await
    }
}