1use crate::{
2 http::{Request, Transport, read_json},
3 identity::{ProcessIdentity, ProcessProbe},
4 process_control::{
5 ProcessControl, SpawnSpec, listener, minimal_env, same_process, stop_owned, wait_owned,
6 },
7 state::ServerState,
8};
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11use std::{path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13
14#[derive(Debug, thiserror::Error)]
15#[error("{0}")]
16pub struct BackendError(pub String);
17impl From<String> for BackendError {
18 fn from(value: String) -> Self {
19 Self(value)
20 }
21}
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
23pub enum BackendKind {
24 #[serde(rename = "ollama")]
25 Ollama,
26 #[serde(rename = "llamacpp")]
27 LlamaCpp,
28}
29impl BackendKind {
30 pub fn name(self) -> &'static str {
31 match self {
32 Self::Ollama => "ollama",
33 Self::LlamaCpp => "llamacpp",
34 }
35 }
36 pub fn port(self) -> u16 {
37 match self {
38 Self::Ollama => 11434,
39 Self::LlamaCpp => 8080,
40 }
41 }
42}
43#[derive(Clone)]
44pub struct ServeRequest {
45 pub model_id: String,
46 pub endpoint: String,
47 pub model_path: Option<PathBuf>,
48 pub context: Option<u32>,
49 pub cache: Option<crate::cache::CacheProfile>,
50}
51#[async_trait::async_trait]
52pub trait BackendAdapter: Send + Sync {
53 fn name(&self) -> &'static str;
54 fn trusts(&self, identity: &ProcessIdentity) -> bool;
55 async fn attach_only(
56 &self,
57 _request: &ServeRequest,
58 _cancel: &CancellationToken,
59 ) -> Result<ServerState, BackendError> {
60 Err(BackendError("attach-only operation is unsupported".into()))
61 }
62 fn can_embed(&self) -> bool {
63 matches!(self.name(), "ollama" | "lmstudio")
64 }
65 fn can_stream(&self) -> bool {
66 self.name() == "ollama"
67 }
68 async fn chat(
69 &self,
70 _handle: &ServerState,
71 _input: &crate::ollama_inference::ChatInput,
72 _cancel: &CancellationToken,
73 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
74 Err(BackendError(
75 "chat is not implemented for this adapter".into(),
76 ))
77 }
78 async fn chat_stream(
79 &self,
80 handle: &ServerState,
81 input: &crate::ollama_inference::ChatInput,
82 cancel: &CancellationToken,
83 on_delta: &mut (dyn for<'chunk> FnMut(&'chunk str) + Send),
84 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
85 let result = self.chat(handle, input, cancel).await?;
86 on_delta(&result.content);
87 Ok(result)
88 }
89 async fn embed(
90 &self,
91 _handle: &ServerState,
92 _model: &str,
93 _inputs: &[String],
94 _cancel: &CancellationToken,
95 ) -> Result<crate::ollama_inference::EmbedResult, BackendError> {
96 Err(BackendError(
97 "embeddings are unsupported by this adapter".into(),
98 ))
99 }
100 async fn serve(
101 &self,
102 request: &ServeRequest,
103 cancel: &CancellationToken,
104 ) -> Result<ServerState, BackendError>;
105 async fn ready(
106 &self,
107 request: &ServeRequest,
108 cancel: &CancellationToken,
109 ) -> Result<(), BackendError>;
110 async fn stop(
111 &self,
112 handle: &ServerState,
113 cancel: &CancellationToken,
114 ) -> Result<(), BackendError>;
115 async fn ready_handle(
116 &self,
117 request: &ServeRequest,
118 _handle: &ServerState,
119 cancel: &CancellationToken,
120 ) -> Result<(), BackendError> {
121 self.ready(request, cancel).await
122 }
123}
124pub struct RuntimeAdapter<'runtime> {
125 pub kind: BackendKind,
126 pub binary: PathBuf,
127 pub http: &'runtime dyn Transport,
128 pub probe: &'runtime dyn ProcessProbe,
129 pub control: &'runtime dyn ProcessControl,
130 pub(crate) ollama_models: Option<PathBuf>,
131}
132pub fn model_id(value: &str) -> Result<(), BackendError> {
133 if value.is_empty()
134 || value.len() > 8192
135 || !value.as_bytes()[0].is_ascii_alphanumeric()
136 || !value.bytes().all(|byte| {
137 byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
138 })
139 || value.split('/').any(|part| part == "..")
140 {
141 return Err(BackendError("invalid runtime model id".into()));
142 }
143 Ok(())
144}
145impl<'runtime> RuntimeAdapter<'runtime> {
146 pub fn new(
147 kind: BackendKind,
148 binary: PathBuf,
149 http: &'runtime dyn Transport,
150 probe: &'runtime dyn ProcessProbe,
151 control: &'runtime dyn ProcessControl,
152 ) -> Self {
153 Self {
154 kind,
155 binary,
156 http,
157 probe,
158 control,
159 ollama_models: None,
160 }
161 }
162 pub fn with_ollama_models(mut self, models: PathBuf) -> Result<Self, BackendError> {
163 if self.kind != BackendKind::Ollama {
164 return Err(BackendError(
165 "model store configuration requires Ollama".into(),
166 ));
167 }
168 crate::command::OllamaCommandContext::new("http://127.0.0.1:11434", &models)?;
169 self.ollama_models = Some(models);
170 Ok(self)
171 }
172 async fn json(
173 &self,
174 endpoint: &str,
175 path: &str,
176 cancel: &CancellationToken,
177 ) -> Result<Value, BackendError> {
178 let request = Request::new(endpoint, path, None, None)
179 .map_err(|error| BackendError(error.to_string()))?;
180 tokio::time::timeout(
181 Duration::from_secs(5),
182 read_json(self.http, request, cancel, 4 * 1024 * 1024),
183 )
184 .await
185 .map_err(|_| BackendError("readiness request timed out".into()))?
186 .map_err(|error| BackendError(error.to_string()))
187 }
188 fn validate(&self, request: &ServeRequest) -> Result<url::Url, BackendError> {
189 model_id(&request.model_id)?;
190 if !self.binary.is_absolute()
191 || request
192 .context
193 .is_some_and(|value| !(1..=10000000).contains(&value))
194 {
195 return Err(BackendError("invalid runtime configuration".into()));
196 }
197 crate::state::loopback(&request.endpoint).map_err(|error| BackendError(error.to_string()))
198 }
199 async fn ready_once(
200 &self,
201 request: &ServeRequest,
202 cancel: &CancellationToken,
203 ) -> Result<(), BackendError> {
204 self.validate(request)?;
205 match self.kind {
206 BackendKind::Ollama => {
207 let version = self.json(&request.endpoint, "/api/version", cancel).await?;
208 if !version
209 .get("version")
210 .and_then(Value::as_str)
211 .is_some_and(|version| !version.is_empty() && version.len() <= 100)
212 {
213 return Err(BackendError("listener is not Ollama".into()));
214 }
215 crate::ollama_installed::parse_inventory(
216 self.json(&request.endpoint, "/api/tags", cancel).await?,
217 )
218 .map_err(|error| BackendError(error.to_string()))?;
219 }
220 BackendKind::LlamaCpp => {
221 let health = self.json(&request.endpoint, "/health", cancel).await?;
222 if health.get("status").and_then(Value::as_str) != Some("ok") {
223 return Err(BackendError("llama.cpp model is not ready".into()));
224 }
225 let props = self.json(&request.endpoint, "/props", cancel).await?;
226 let path = props.get("model_path").and_then(Value::as_str);
227 let alias = props.get("model_alias").and_then(Value::as_str);
228 if request
229 .model_path
230 .as_ref()
231 .is_some_and(|expected| path != expected.to_str())
232 || alias != Some(request.model_id.as_str())
233 {
234 return Err(BackendError(
235 "llama.cpp loaded artifact or alias differs".into(),
236 ));
237 }
238 }
239 }
240 Ok(())
241 }
242 pub async fn ready(
243 &self,
244 request: &ServeRequest,
245 cancel: &CancellationToken,
246 ) -> Result<(), BackendError> {
247 let operation = async {
248 let mut last = BackendError("runtime is not ready".into());
249 for attempt in 0..20 {
250 match self.ready_once(request, cancel).await {
251 Ok(()) => return Ok(()),
252 Err(error) => last = error,
253 }
254 tokio::time::sleep(Duration::from_millis((100u64 << attempt.min(4)).min(2000)))
255 .await;
256 }
257 Err(last)
258 };
259 tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(30),operation)=>result.map_err(|_|BackendError("runtime readiness timed out".into()))?}
260 }
261 pub async fn serve(
262 &self,
263 request: &ServeRequest,
264 cancel: &CancellationToken,
265 ) -> Result<ServerState, BackendError> {
266 let url = self.validate(request)?;
267 if cancel.is_cancelled() {
268 return Err(BackendError("cancelled".into()));
269 }
270 let occupied = tokio::select! {biased;_=cancel.cancelled()=>return Err(BackendError("cancelled".into())),result=self.control.occupied(&request.endpoint)=>result?};
271 let executable = self
272 .binary
273 .to_str()
274 .ok_or_else(|| BackendError("invalid executable path".into()))?;
275 let (observed, owned) = if occupied {
276 if crate::cache::effective(request.cache.as_ref()).is_some() {
277 return Err(BackendError(format!(
278 "cache settings apply only to a {} runtime rigspark starts; stop the running server or omit the cache options",
279 self.kind.name()
280 )));
281 }
282 let before = listener(self.probe, &request.endpoint, cancel).await?;
283 if before.identity.executable != executable {
284 return Err(BackendError(
285 "refusing untrusted listener executable".into(),
286 ));
287 }
288 self.ready_once(request, cancel).await?;
289 let after = listener(self.probe, &request.endpoint, cancel).await?;
290 if !same_process(&before.identity, &after.identity) {
291 return Err(BackendError("listener changed during attach".into()));
292 }
293 (after, false)
294 } else {
295 let host = url
296 .host_str()
297 .ok_or_else(|| BackendError("missing runtime host".into()))?
298 .trim_matches(['[', ']']);
299 let port = url
300 .port_or_known_default()
301 .ok_or_else(|| BackendError("missing port".into()))?;
302 let mut env = minimal_env();
303 let delta = crate::cache::launch_delta(request.cache.as_ref(), self.kind.name(), true)
304 .map_err(|error| BackendError(error.to_string()))?;
305 let mut args = match self.kind {
306 BackendKind::Ollama => {
307 if let Some(models) = &self.ollama_models {
308 let context =
309 crate::command::OllamaCommandContext::new(&request.endpoint, models)?;
310 env.extend(context.environment().clone());
311 }
312 env.insert(
313 "OLLAMA_HOST".into(),
314 format!(
315 "{}:{port}",
316 if host.contains(':') {
317 format!("[{host}]")
318 } else {
319 host.into()
320 }
321 ),
322 );
323 vec!["serve".into()]
324 }
325 BackendKind::LlamaCpp => {
326 let path = request
327 .model_path
328 .as_ref()
329 .filter(|path| path.is_absolute() && path.is_file())
330 .ok_or_else(|| {
331 BackendError("verified absolute GGUF path required".into())
332 })?;
333 let mut args = vec![
334 "-m".into(),
335 path.to_string_lossy().into_owned(),
336 "--host".into(),
337 host.into(),
338 "--port".into(),
339 port.to_string(),
340 "--alias".into(),
341 request.model_id.clone(),
342 ];
343 if let Some(context) = request.context {
344 args.extend(["--ctx-size".into(), context.to_string()]);
345 }
346 args
347 }
348 };
349 args.extend(delta.args);
350 env.extend(delta.env);
351 if cancel.is_cancelled() {
352 return Err(BackendError("cancelled".into()));
353 }
354 let child = self
355 .control
356 .spawn(&SpawnSpec {
357 binary: self.binary.clone(),
358 args,
359 env,
360 })
361 .await?;
362 let observed = wait_owned(
363 child,
364 &request.endpoint,
365 executable,
366 self.probe,
367 cancel,
368 || async { self.ready(request, cancel).await.map_err(|error| error.0) },
369 )
370 .await?;
371 (observed, true)
372 };
373 let state = ServerState {
374 backend: self.kind.name().into(),
375 model_id: request.model_id.clone(),
376 endpoint: request.endpoint.clone(),
377 port: observed.port,
378 owned_by_us: owned,
379 pid: Some(observed.identity.pid),
380 runtime_model_id: None,
381 context: request.context,
382 integrity: None,
383 local_manifest_digest: None,
384 model_path: None,
385 process_executable: Some(observed.identity.executable),
386 process_started_at: Some(observed.identity.started),
387 auth_token: None,
388 cache: if owned {
389 crate::cache::effective(request.cache.as_ref())
390 } else {
391 None
392 },
393 };
394 state
395 .validate()
396 .map_err(|error| BackendError(error.to_string()))?;
397 Ok(state)
398 }
399 pub async fn stop(
400 &self,
401 handle: &ServerState,
402 cancel: &CancellationToken,
403 ) -> Result<(), BackendError> {
404 handle
405 .validate()
406 .map_err(|error| BackendError(error.to_string()))?;
407 if handle.backend != self.kind.name() {
408 return Err(BackendError("backend handle mismatch".into()));
409 }
410 if !handle.owned_by_us {
411 return Ok(());
412 }
413 let expected = ProcessIdentity {
414 pid: handle
415 .pid
416 .ok_or_else(|| BackendError("missing PID".into()))?,
417 process: String::new(),
418 executable: handle
419 .process_executable
420 .clone()
421 .ok_or_else(|| BackendError("missing executable identity".into()))?,
422 started: handle
423 .process_started_at
424 .clone()
425 .ok_or_else(|| BackendError("missing start identity".into()))?,
426 };
427 if self.binary.to_str() != Some(&expected.executable) {
428 return Err(BackendError("untrusted stop executable".into()));
429 }
430 stop_owned(
431 &handle.endpoint,
432 &expected,
433 self.probe,
434 self.control,
435 cancel,
436 )
437 .await
438 .map_err(BackendError)
439 }
440}
441#[async_trait::async_trait]
442impl BackendAdapter for RuntimeAdapter<'_> {
443 async fn attach_only(
444 &self,
445 request: &ServeRequest,
446 cancel: &CancellationToken,
447 ) -> Result<ServerState, BackendError> {
448 self.validate(request)?;
449 if cancel.is_cancelled() {
450 return Err(BackendError("cancelled".into()));
451 }
452 if !self.control.occupied(&request.endpoint).await? {
453 return Err(BackendError(
454 "existing runtime required; refusing to spawn".into(),
455 ));
456 }
457 let before = listener(self.probe, &request.endpoint, cancel).await?;
458 if !self.trusts(&before.identity) {
459 return Err(BackendError("untrusted attached executable".into()));
460 }
461 self.ready_once(request, cancel).await?;
462 let after = listener(self.probe, &request.endpoint, cancel).await?;
463 if !same_process(&before.identity, &after.identity) {
464 return Err(BackendError("listener changed during attachment".into()));
465 }
466 let state=crate::state::RuntimeState::parse(&serde_json::json!({"schemaVersion":2,"active":{"backend":self.name(),"modelId":request.model_id,"endpoint":request.endpoint,"port":after.port,"ownedByUs":false,"pid":after.identity.pid,"processExecutable":after.identity.executable,"processStartedAt":after.identity.started}}).to_string()).map_err(|error|BackendError(error.to_string()))?;
467 state
468 .active
469 .ok_or_else(|| BackendError("missing attached state".into()))
470 }
471 async fn ready_handle(
472 &self,
473 request: &ServeRequest,
474 handle: &ServerState,
475 cancel: &CancellationToken,
476 ) -> Result<(), BackendError> {
477 handle
478 .validate()
479 .map_err(|error| BackendError(error.to_string()))?;
480 if handle.backend != self.name() || handle.endpoint != request.endpoint {
481 return Err(BackendError("readiness handle mismatch".into()));
482 }
483 let before = listener(self.probe, &handle.endpoint, cancel).await?;
484 crate::identity::capture(handle, &before, self.trusts(&before.identity))
485 .map_err(|error| BackendError(error.to_string()))?;
486 self.ready(request, cancel).await?;
487 let models = self.json(&handle.endpoint, "/v1/models", cancel).await?;
488 #[derive(Deserialize)]
489 struct Model {
490 id: String,
491 }
492 #[derive(Deserialize)]
493 struct Models {
494 data: Vec<Model>,
495 }
496 let models: Models = serde_json::from_value(models)
497 .map_err(|_| BackendError("invalid OpenAI models response".into()))?;
498 if models.data.len() > 10000
499 || models
500 .data
501 .iter()
502 .any(|model| model.id.is_empty() || model.id.len() > 8192)
503 {
504 return Err(BackendError("invalid OpenAI models".into()));
505 }
506 let after = listener(self.probe, &handle.endpoint, cancel).await?;
507 if !same_process(&before.identity, &after.identity) {
508 return Err(BackendError("readiness listener changed".into()));
509 }
510 Ok(())
511 }
512 fn name(&self) -> &'static str {
513 self.kind.name()
514 }
515 fn trusts(&self, identity: &ProcessIdentity) -> bool {
516 self.binary.to_str() == Some(identity.executable.as_str())
517 }
518 async fn chat(
519 &self,
520 handle: &ServerState,
521 input: &crate::ollama_inference::ChatInput,
522 cancel: &CancellationToken,
523 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
524 if self.kind == BackendKind::LlamaCpp {
525 return crate::openai::OpenAiInference {
526 http: self.http,
527 probe: self.probe,
528 adapter: self,
529 token: None,
530 }
531 .chat(handle, input, cancel)
532 .await;
533 }
534 let expected = expected_process(handle)?;
535 let mut input = input.clone();
536 input.model = handle.runtime_model_id.clone().unwrap_or(input.model);
537 crate::ollama_inference::OllamaInference::new(
538 self.http,
539 self.probe,
540 self.binary
541 .to_str()
542 .ok_or_else(|| BackendError("invalid executable".into()))?,
543 )
544 .chat(&handle.endpoint, &expected, &input, cancel)
545 .await
546 .map_err(|error| BackendError(error.to_string()))
547 }
548 async fn chat_stream(
549 &self,
550 handle: &ServerState,
551 input: &crate::ollama_inference::ChatInput,
552 cancel: &CancellationToken,
553 on_delta: &mut (dyn for<'chunk> FnMut(&'chunk str) + Send),
554 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
555 if self.kind == BackendKind::LlamaCpp {
556 let result = self.chat(handle, input, cancel).await?;
557 on_delta(&result.content);
558 return Ok(result);
559 }
560 let expected = expected_process(handle)?;
561 let mut input = input.clone();
562 input.model = handle.runtime_model_id.clone().unwrap_or(input.model);
563 crate::ollama_inference::OllamaInference::new(
564 self.http,
565 self.probe,
566 self.binary
567 .to_str()
568 .ok_or_else(|| BackendError("invalid executable".into()))?,
569 )
570 .chat_stream(&handle.endpoint, &expected, &input, cancel, on_delta)
571 .await
572 .map_err(|error| BackendError(error.to_string()))
573 }
574 async fn embed(
575 &self,
576 handle: &ServerState,
577 model: &str,
578 inputs: &[String],
579 cancel: &CancellationToken,
580 ) -> Result<crate::ollama_inference::EmbedResult, BackendError> {
581 if self.kind != BackendKind::Ollama {
582 return Err(BackendError("llama.cpp does not serve embeddings".into()));
583 }
584 let expected = expected_process(handle)?;
585 crate::ollama_inference::OllamaInference::new(
586 self.http,
587 self.probe,
588 self.binary
589 .to_str()
590 .ok_or_else(|| BackendError("invalid executable".into()))?,
591 )
592 .embed(&handle.endpoint, &expected, model, inputs, cancel)
593 .await
594 .map_err(|error| BackendError(error.to_string()))
595 }
596 async fn serve(
597 &self,
598 request: &ServeRequest,
599 cancel: &CancellationToken,
600 ) -> Result<ServerState, BackendError> {
601 RuntimeAdapter::serve(self, request, cancel).await
602 }
603 async fn ready(
604 &self,
605 request: &ServeRequest,
606 cancel: &CancellationToken,
607 ) -> Result<(), BackendError> {
608 RuntimeAdapter::ready(self, request, cancel).await
609 }
610 async fn stop(
611 &self,
612 handle: &ServerState,
613 cancel: &CancellationToken,
614 ) -> Result<(), BackendError> {
615 RuntimeAdapter::stop(self, handle, cancel).await
616 }
617}
618fn expected_process(handle: &ServerState) -> Result<ProcessIdentity, BackendError> {
619 handle
620 .validate()
621 .map_err(|error| BackendError(error.to_string()))?;
622 if handle.backend != "ollama" {
623 return Err(BackendError("Ollama handle required".into()));
624 }
625 Ok(ProcessIdentity {
626 pid: handle
627 .pid
628 .ok_or_else(|| BackendError("expected PID required".into()))?,
629 process: String::new(),
630 executable: handle
631 .process_executable
632 .clone()
633 .ok_or_else(|| BackendError("expected executable required".into()))?,
634 started: handle
635 .process_started_at
636 .clone()
637 .ok_or_else(|| BackendError("expected start identity required".into()))?,
638 })
639}