Skip to main content

rigspark_runtime/
adapters.rs

1use crate::{
2    http::{Request, Transport, read_json},
3    identity::{ProcessIdentity, ProcessProbe},
4    process_control::{
5        ProcessControl, SpawnSpec, listener, minimal_env, same_process, stop_owned, wait_owned,
6    },
7    state::ServerState,
8};
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11use std::{path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13
14#[derive(Debug, thiserror::Error)]
15#[error("{0}")]
16pub struct BackendError(pub String);
17impl From<String> for BackendError {
18    fn from(value: String) -> Self {
19        Self(value)
20    }
21}
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
23pub enum BackendKind {
24    #[serde(rename = "ollama")]
25    Ollama,
26    #[serde(rename = "llamacpp")]
27    LlamaCpp,
28}
29impl BackendKind {
30    pub fn name(self) -> &'static str {
31        match self {
32            Self::Ollama => "ollama",
33            Self::LlamaCpp => "llamacpp",
34        }
35    }
36    pub fn port(self) -> u16 {
37        match self {
38            Self::Ollama => 11434,
39            Self::LlamaCpp => 8080,
40        }
41    }
42}
43#[derive(Clone)]
44pub struct ServeRequest {
45    pub model_id: String,
46    pub endpoint: String,
47    pub model_path: Option<PathBuf>,
48    pub context: Option<u32>,
49    pub cache: Option<crate::cache::CacheProfile>,
50}
51#[async_trait::async_trait]
52pub trait BackendAdapter: Send + Sync {
53    fn name(&self) -> &'static str;
54    fn trusts(&self, identity: &ProcessIdentity) -> bool;
55    async fn attach_only(
56        &self,
57        _request: &ServeRequest,
58        _cancel: &CancellationToken,
59    ) -> Result<ServerState, BackendError> {
60        Err(BackendError("attach-only operation is unsupported".into()))
61    }
62    fn can_embed(&self) -> bool {
63        matches!(self.name(), "ollama" | "lmstudio")
64    }
65    fn can_stream(&self) -> bool {
66        self.name() == "ollama"
67    }
68    async fn chat(
69        &self,
70        _handle: &ServerState,
71        _input: &crate::ollama_inference::ChatInput,
72        _cancel: &CancellationToken,
73    ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
74        Err(BackendError(
75            "chat is not implemented for this adapter".into(),
76        ))
77    }
78    async fn chat_stream(
79        &self,
80        handle: &ServerState,
81        input: &crate::ollama_inference::ChatInput,
82        cancel: &CancellationToken,
83        on_delta: &mut (dyn for<'chunk> FnMut(&'chunk str) + Send),
84    ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
85        let result = self.chat(handle, input, cancel).await?;
86        on_delta(&result.content);
87        Ok(result)
88    }
89    async fn embed(
90        &self,
91        _handle: &ServerState,
92        _model: &str,
93        _inputs: &[String],
94        _cancel: &CancellationToken,
95    ) -> Result<crate::ollama_inference::EmbedResult, BackendError> {
96        Err(BackendError(
97            "embeddings are unsupported by this adapter".into(),
98        ))
99    }
100    async fn serve(
101        &self,
102        request: &ServeRequest,
103        cancel: &CancellationToken,
104    ) -> Result<ServerState, BackendError>;
105    async fn ready(
106        &self,
107        request: &ServeRequest,
108        cancel: &CancellationToken,
109    ) -> Result<(), BackendError>;
110    async fn stop(
111        &self,
112        handle: &ServerState,
113        cancel: &CancellationToken,
114    ) -> Result<(), BackendError>;
115    async fn ready_handle(
116        &self,
117        request: &ServeRequest,
118        _handle: &ServerState,
119        cancel: &CancellationToken,
120    ) -> Result<(), BackendError> {
121        self.ready(request, cancel).await
122    }
123}
124pub struct RuntimeAdapter<'runtime> {
125    pub kind: BackendKind,
126    pub binary: PathBuf,
127    pub http: &'runtime dyn Transport,
128    pub probe: &'runtime dyn ProcessProbe,
129    pub control: &'runtime dyn ProcessControl,
130    pub(crate) ollama_models: Option<PathBuf>,
131}
132pub fn model_id(value: &str) -> Result<(), BackendError> {
133    if value.is_empty()
134        || value.len() > 8192
135        || !value.as_bytes()[0].is_ascii_alphanumeric()
136        || !value.bytes().all(|byte| {
137            byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
138        })
139        || value.split('/').any(|part| part == "..")
140    {
141        return Err(BackendError("invalid runtime model id".into()));
142    }
143    Ok(())
144}
145impl<'runtime> RuntimeAdapter<'runtime> {
146    pub fn new(
147        kind: BackendKind,
148        binary: PathBuf,
149        http: &'runtime dyn Transport,
150        probe: &'runtime dyn ProcessProbe,
151        control: &'runtime dyn ProcessControl,
152    ) -> Self {
153        Self {
154            kind,
155            binary,
156            http,
157            probe,
158            control,
159            ollama_models: None,
160        }
161    }
162    pub fn with_ollama_models(mut self, models: PathBuf) -> Result<Self, BackendError> {
163        if self.kind != BackendKind::Ollama {
164            return Err(BackendError(
165                "model store configuration requires Ollama".into(),
166            ));
167        }
168        crate::command::OllamaCommandContext::new("http://127.0.0.1:11434", &models)?;
169        self.ollama_models = Some(models);
170        Ok(self)
171    }
172    async fn json(
173        &self,
174        endpoint: &str,
175        path: &str,
176        cancel: &CancellationToken,
177    ) -> Result<Value, BackendError> {
178        let request = Request::new(endpoint, path, None, None)
179            .map_err(|error| BackendError(error.to_string()))?;
180        tokio::time::timeout(
181            Duration::from_secs(5),
182            read_json(self.http, request, cancel, 4 * 1024 * 1024),
183        )
184        .await
185        .map_err(|_| BackendError("readiness request timed out".into()))?
186        .map_err(|error| BackendError(error.to_string()))
187    }
188    fn validate(&self, request: &ServeRequest) -> Result<url::Url, BackendError> {
189        model_id(&request.model_id)?;
190        if !self.binary.is_absolute()
191            || request
192                .context
193                .is_some_and(|value| !(1..=10000000).contains(&value))
194        {
195            return Err(BackendError("invalid runtime configuration".into()));
196        }
197        crate::state::loopback(&request.endpoint).map_err(|error| BackendError(error.to_string()))
198    }
199    async fn ready_once(
200        &self,
201        request: &ServeRequest,
202        cancel: &CancellationToken,
203    ) -> Result<(), BackendError> {
204        self.validate(request)?;
205        match self.kind {
206            BackendKind::Ollama => {
207                let version = self.json(&request.endpoint, "/api/version", cancel).await?;
208                if !version
209                    .get("version")
210                    .and_then(Value::as_str)
211                    .is_some_and(|version| !version.is_empty() && version.len() <= 100)
212                {
213                    return Err(BackendError("listener is not Ollama".into()));
214                }
215                crate::ollama_installed::parse_inventory(
216                    self.json(&request.endpoint, "/api/tags", cancel).await?,
217                )
218                .map_err(|error| BackendError(error.to_string()))?;
219            }
220            BackendKind::LlamaCpp => {
221                let health = self.json(&request.endpoint, "/health", cancel).await?;
222                if health.get("status").and_then(Value::as_str) != Some("ok") {
223                    return Err(BackendError("llama.cpp model is not ready".into()));
224                }
225                let props = self.json(&request.endpoint, "/props", cancel).await?;
226                let path = props.get("model_path").and_then(Value::as_str);
227                let alias = props.get("model_alias").and_then(Value::as_str);
228                if request
229                    .model_path
230                    .as_ref()
231                    .is_some_and(|expected| path != expected.to_str())
232                    || alias != Some(request.model_id.as_str())
233                {
234                    return Err(BackendError(
235                        "llama.cpp loaded artifact or alias differs".into(),
236                    ));
237                }
238            }
239        }
240        Ok(())
241    }
242    pub async fn ready(
243        &self,
244        request: &ServeRequest,
245        cancel: &CancellationToken,
246    ) -> Result<(), BackendError> {
247        let operation = async {
248            let mut last = BackendError("runtime is not ready".into());
249            for attempt in 0..20 {
250                match self.ready_once(request, cancel).await {
251                    Ok(()) => return Ok(()),
252                    Err(error) => last = error,
253                }
254                tokio::time::sleep(Duration::from_millis((100u64 << attempt.min(4)).min(2000)))
255                    .await;
256            }
257            Err(last)
258        };
259        tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(30),operation)=>result.map_err(|_|BackendError("runtime readiness timed out".into()))?}
260    }
261    pub async fn serve(
262        &self,
263        request: &ServeRequest,
264        cancel: &CancellationToken,
265    ) -> Result<ServerState, BackendError> {
266        let url = self.validate(request)?;
267        if cancel.is_cancelled() {
268            return Err(BackendError("cancelled".into()));
269        }
270        let occupied = tokio::select! {biased;_=cancel.cancelled()=>return Err(BackendError("cancelled".into())),result=self.control.occupied(&request.endpoint)=>result?};
271        let executable = self
272            .binary
273            .to_str()
274            .ok_or_else(|| BackendError("invalid executable path".into()))?;
275        let (observed, owned) = if occupied {
276            if crate::cache::effective(request.cache.as_ref()).is_some() {
277                return Err(BackendError(format!(
278                    "cache settings apply only to a {} runtime rigspark starts; stop the running server or omit the cache options",
279                    self.kind.name()
280                )));
281            }
282            let before = listener(self.probe, &request.endpoint, cancel).await?;
283            if before.identity.executable != executable {
284                return Err(BackendError(
285                    "refusing untrusted listener executable".into(),
286                ));
287            }
288            self.ready_once(request, cancel).await?;
289            let after = listener(self.probe, &request.endpoint, cancel).await?;
290            if !same_process(&before.identity, &after.identity) {
291                return Err(BackendError("listener changed during attach".into()));
292            }
293            (after, false)
294        } else {
295            let host = url
296                .host_str()
297                .ok_or_else(|| BackendError("missing runtime host".into()))?
298                .trim_matches(['[', ']']);
299            let port = url
300                .port_or_known_default()
301                .ok_or_else(|| BackendError("missing port".into()))?;
302            let mut env = minimal_env();
303            let delta = crate::cache::launch_delta(request.cache.as_ref(), self.kind.name(), true)
304                .map_err(|error| BackendError(error.to_string()))?;
305            let mut args = match self.kind {
306                BackendKind::Ollama => {
307                    if let Some(models) = &self.ollama_models {
308                        let context =
309                            crate::command::OllamaCommandContext::new(&request.endpoint, models)?;
310                        env.extend(context.environment().clone());
311                    }
312                    env.insert(
313                        "OLLAMA_HOST".into(),
314                        format!(
315                            "{}:{port}",
316                            if host.contains(':') {
317                                format!("[{host}]")
318                            } else {
319                                host.into()
320                            }
321                        ),
322                    );
323                    vec!["serve".into()]
324                }
325                BackendKind::LlamaCpp => {
326                    let path = request
327                        .model_path
328                        .as_ref()
329                        .filter(|path| path.is_absolute() && path.is_file())
330                        .ok_or_else(|| {
331                            BackendError("verified absolute GGUF path required".into())
332                        })?;
333                    let mut args = vec![
334                        "-m".into(),
335                        path.to_string_lossy().into_owned(),
336                        "--host".into(),
337                        host.into(),
338                        "--port".into(),
339                        port.to_string(),
340                        "--alias".into(),
341                        request.model_id.clone(),
342                    ];
343                    if let Some(context) = request.context {
344                        args.extend(["--ctx-size".into(), context.to_string()]);
345                    }
346                    args
347                }
348            };
349            args.extend(delta.args);
350            env.extend(delta.env);
351            if cancel.is_cancelled() {
352                return Err(BackendError("cancelled".into()));
353            }
354            let child = self
355                .control
356                .spawn(&SpawnSpec {
357                    binary: self.binary.clone(),
358                    args,
359                    env,
360                })
361                .await?;
362            let observed = wait_owned(
363                child,
364                &request.endpoint,
365                executable,
366                self.probe,
367                cancel,
368                || async { self.ready(request, cancel).await.map_err(|error| error.0) },
369            )
370            .await?;
371            (observed, true)
372        };
373        let state = ServerState {
374            backend: self.kind.name().into(),
375            model_id: request.model_id.clone(),
376            endpoint: request.endpoint.clone(),
377            port: observed.port,
378            owned_by_us: owned,
379            pid: Some(observed.identity.pid),
380            runtime_model_id: None,
381            context: request.context,
382            integrity: None,
383            local_manifest_digest: None,
384            model_path: None,
385            process_executable: Some(observed.identity.executable),
386            process_started_at: Some(observed.identity.started),
387            auth_token: None,
388            cache: if owned {
389                crate::cache::effective(request.cache.as_ref())
390            } else {
391                None
392            },
393        };
394        state
395            .validate()
396            .map_err(|error| BackendError(error.to_string()))?;
397        Ok(state)
398    }
399    pub async fn stop(
400        &self,
401        handle: &ServerState,
402        cancel: &CancellationToken,
403    ) -> Result<(), BackendError> {
404        handle
405            .validate()
406            .map_err(|error| BackendError(error.to_string()))?;
407        if handle.backend != self.kind.name() {
408            return Err(BackendError("backend handle mismatch".into()));
409        }
410        if !handle.owned_by_us {
411            return Ok(());
412        }
413        let expected = ProcessIdentity {
414            pid: handle
415                .pid
416                .ok_or_else(|| BackendError("missing PID".into()))?,
417            process: String::new(),
418            executable: handle
419                .process_executable
420                .clone()
421                .ok_or_else(|| BackendError("missing executable identity".into()))?,
422            started: handle
423                .process_started_at
424                .clone()
425                .ok_or_else(|| BackendError("missing start identity".into()))?,
426        };
427        if self.binary.to_str() != Some(&expected.executable) {
428            return Err(BackendError("untrusted stop executable".into()));
429        }
430        stop_owned(
431            &handle.endpoint,
432            &expected,
433            self.probe,
434            self.control,
435            cancel,
436        )
437        .await
438        .map_err(BackendError)
439    }
440}
441#[async_trait::async_trait]
442impl BackendAdapter for RuntimeAdapter<'_> {
443    async fn attach_only(
444        &self,
445        request: &ServeRequest,
446        cancel: &CancellationToken,
447    ) -> Result<ServerState, BackendError> {
448        self.validate(request)?;
449        if cancel.is_cancelled() {
450            return Err(BackendError("cancelled".into()));
451        }
452        if !self.control.occupied(&request.endpoint).await? {
453            return Err(BackendError(
454                "existing runtime required; refusing to spawn".into(),
455            ));
456        }
457        let before = listener(self.probe, &request.endpoint, cancel).await?;
458        if !self.trusts(&before.identity) {
459            return Err(BackendError("untrusted attached executable".into()));
460        }
461        self.ready_once(request, cancel).await?;
462        let after = listener(self.probe, &request.endpoint, cancel).await?;
463        if !same_process(&before.identity, &after.identity) {
464            return Err(BackendError("listener changed during attachment".into()));
465        }
466        let state=crate::state::RuntimeState::parse(&serde_json::json!({"schemaVersion":2,"active":{"backend":self.name(),"modelId":request.model_id,"endpoint":request.endpoint,"port":after.port,"ownedByUs":false,"pid":after.identity.pid,"processExecutable":after.identity.executable,"processStartedAt":after.identity.started}}).to_string()).map_err(|error|BackendError(error.to_string()))?;
467        state
468            .active
469            .ok_or_else(|| BackendError("missing attached state".into()))
470    }
471    async fn ready_handle(
472        &self,
473        request: &ServeRequest,
474        handle: &ServerState,
475        cancel: &CancellationToken,
476    ) -> Result<(), BackendError> {
477        handle
478            .validate()
479            .map_err(|error| BackendError(error.to_string()))?;
480        if handle.backend != self.name() || handle.endpoint != request.endpoint {
481            return Err(BackendError("readiness handle mismatch".into()));
482        }
483        let before = listener(self.probe, &handle.endpoint, cancel).await?;
484        crate::identity::capture(handle, &before, self.trusts(&before.identity))
485            .map_err(|error| BackendError(error.to_string()))?;
486        self.ready(request, cancel).await?;
487        let models = self.json(&handle.endpoint, "/v1/models", cancel).await?;
488        #[derive(Deserialize)]
489        struct Model {
490            id: String,
491        }
492        #[derive(Deserialize)]
493        struct Models {
494            data: Vec<Model>,
495        }
496        let models: Models = serde_json::from_value(models)
497            .map_err(|_| BackendError("invalid OpenAI models response".into()))?;
498        if models.data.len() > 10000
499            || models
500                .data
501                .iter()
502                .any(|model| model.id.is_empty() || model.id.len() > 8192)
503        {
504            return Err(BackendError("invalid OpenAI models".into()));
505        }
506        let after = listener(self.probe, &handle.endpoint, cancel).await?;
507        if !same_process(&before.identity, &after.identity) {
508            return Err(BackendError("readiness listener changed".into()));
509        }
510        Ok(())
511    }
512    fn name(&self) -> &'static str {
513        self.kind.name()
514    }
515    fn trusts(&self, identity: &ProcessIdentity) -> bool {
516        self.binary.to_str() == Some(identity.executable.as_str())
517    }
518    async fn chat(
519        &self,
520        handle: &ServerState,
521        input: &crate::ollama_inference::ChatInput,
522        cancel: &CancellationToken,
523    ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
524        if self.kind == BackendKind::LlamaCpp {
525            return crate::openai::OpenAiInference {
526                http: self.http,
527                probe: self.probe,
528                adapter: self,
529                token: None,
530            }
531            .chat(handle, input, cancel)
532            .await;
533        }
534        let expected = expected_process(handle)?;
535        let mut input = input.clone();
536        input.model = handle.runtime_model_id.clone().unwrap_or(input.model);
537        crate::ollama_inference::OllamaInference::new(
538            self.http,
539            self.probe,
540            self.binary
541                .to_str()
542                .ok_or_else(|| BackendError("invalid executable".into()))?,
543        )
544        .chat(&handle.endpoint, &expected, &input, cancel)
545        .await
546        .map_err(|error| BackendError(error.to_string()))
547    }
548    async fn chat_stream(
549        &self,
550        handle: &ServerState,
551        input: &crate::ollama_inference::ChatInput,
552        cancel: &CancellationToken,
553        on_delta: &mut (dyn for<'chunk> FnMut(&'chunk str) + Send),
554    ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
555        if self.kind == BackendKind::LlamaCpp {
556            let result = self.chat(handle, input, cancel).await?;
557            on_delta(&result.content);
558            return Ok(result);
559        }
560        let expected = expected_process(handle)?;
561        let mut input = input.clone();
562        input.model = handle.runtime_model_id.clone().unwrap_or(input.model);
563        crate::ollama_inference::OllamaInference::new(
564            self.http,
565            self.probe,
566            self.binary
567                .to_str()
568                .ok_or_else(|| BackendError("invalid executable".into()))?,
569        )
570        .chat_stream(&handle.endpoint, &expected, &input, cancel, on_delta)
571        .await
572        .map_err(|error| BackendError(error.to_string()))
573    }
574    async fn embed(
575        &self,
576        handle: &ServerState,
577        model: &str,
578        inputs: &[String],
579        cancel: &CancellationToken,
580    ) -> Result<crate::ollama_inference::EmbedResult, BackendError> {
581        if self.kind != BackendKind::Ollama {
582            return Err(BackendError("llama.cpp does not serve embeddings".into()));
583        }
584        let expected = expected_process(handle)?;
585        crate::ollama_inference::OllamaInference::new(
586            self.http,
587            self.probe,
588            self.binary
589                .to_str()
590                .ok_or_else(|| BackendError("invalid executable".into()))?,
591        )
592        .embed(&handle.endpoint, &expected, model, inputs, cancel)
593        .await
594        .map_err(|error| BackendError(error.to_string()))
595    }
596    async fn serve(
597        &self,
598        request: &ServeRequest,
599        cancel: &CancellationToken,
600    ) -> Result<ServerState, BackendError> {
601        RuntimeAdapter::serve(self, request, cancel).await
602    }
603    async fn ready(
604        &self,
605        request: &ServeRequest,
606        cancel: &CancellationToken,
607    ) -> Result<(), BackendError> {
608        RuntimeAdapter::ready(self, request, cancel).await
609    }
610    async fn stop(
611        &self,
612        handle: &ServerState,
613        cancel: &CancellationToken,
614    ) -> Result<(), BackendError> {
615        RuntimeAdapter::stop(self, handle, cancel).await
616    }
617}
618fn expected_process(handle: &ServerState) -> Result<ProcessIdentity, BackendError> {
619    handle
620        .validate()
621        .map_err(|error| BackendError(error.to_string()))?;
622    if handle.backend != "ollama" {
623        return Err(BackendError("Ollama handle required".into()));
624    }
625    Ok(ProcessIdentity {
626        pid: handle
627            .pid
628            .ok_or_else(|| BackendError("expected PID required".into()))?,
629        process: String::new(),
630        executable: handle
631            .process_executable
632            .clone()
633            .ok_or_else(|| BackendError("expected executable required".into()))?,
634        started: handle
635            .process_started_at
636            .clone()
637            .ok_or_else(|| BackendError("expected start identity required".into()))?,
638    })
639}