1use crate::{
2 http::{Request, Transport, read_json},
3 identity::{ProcessIdentity, ProcessProbe},
4 process_control::{
5 ProcessControl, SpawnSpec, listener, minimal_env, same_process, stop_owned, wait_owned,
6 },
7 state::ServerState,
8};
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11use std::{path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13
14#[derive(Debug, thiserror::Error)]
15#[error("{0}")]
16pub struct BackendError(pub String);
17impl From<String> for BackendError {
18 fn from(value: String) -> Self {
19 Self(value)
20 }
21}
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
23pub enum BackendKind {
24 #[serde(rename = "ollama")]
25 Ollama,
26 #[serde(rename = "llamacpp")]
27 LlamaCpp,
28 #[serde(rename = "bitnet")]
29 BitNet,
30}
31impl BackendKind {
32 pub fn name(self) -> &'static str {
33 match self {
34 Self::Ollama => "ollama",
35 Self::LlamaCpp => "llamacpp",
36 Self::BitNet => "bitnet",
37 }
38 }
39 pub fn port(self) -> u16 {
40 match self {
41 Self::Ollama => 11434,
42 Self::LlamaCpp | Self::BitNet => 8080,
43 }
44 }
45}
46#[derive(Clone)]
47pub struct ServeRequest {
48 pub model_id: String,
49 pub endpoint: String,
50 pub model_path: Option<PathBuf>,
51 pub draft_model_path: Option<PathBuf>,
53 pub context: Option<u32>,
54 pub cache: Option<crate::cache::CacheProfile>,
55 pub slot_save_path: Option<PathBuf>,
57}
58#[async_trait::async_trait]
59pub trait BackendAdapter: Send + Sync {
60 fn name(&self) -> &'static str;
61 fn trusts(&self, identity: &ProcessIdentity) -> bool;
62 async fn attach_only(
63 &self,
64 _request: &ServeRequest,
65 _cancel: &CancellationToken,
66 ) -> Result<ServerState, BackendError> {
67 Err(BackendError("attach-only operation is unsupported".into()))
68 }
69 fn can_embed(&self) -> bool {
70 matches!(self.name(), "ollama" | "lmstudio")
71 }
72 fn can_stream(&self) -> bool {
73 self.name() == "ollama"
74 }
75 async fn chat(
76 &self,
77 _handle: &ServerState,
78 _input: &crate::ollama_inference::ChatInput,
79 _cancel: &CancellationToken,
80 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
81 Err(BackendError(
82 "chat is not implemented for this adapter".into(),
83 ))
84 }
85 async fn chat_stream(
86 &self,
87 handle: &ServerState,
88 input: &crate::ollama_inference::ChatInput,
89 cancel: &CancellationToken,
90 on_delta: &mut (dyn for<'chunk> FnMut(&'chunk str) + Send),
91 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
92 let result = self.chat(handle, input, cancel).await?;
93 on_delta(&result.content);
94 Ok(result)
95 }
96 async fn embed(
97 &self,
98 _handle: &ServerState,
99 _model: &str,
100 _inputs: &[String],
101 _cancel: &CancellationToken,
102 ) -> Result<crate::ollama_inference::EmbedResult, BackendError> {
103 Err(BackendError(
104 "embeddings are unsupported by this adapter".into(),
105 ))
106 }
107 async fn serve(
108 &self,
109 request: &ServeRequest,
110 cancel: &CancellationToken,
111 ) -> Result<ServerState, BackendError>;
112 async fn ready(
113 &self,
114 request: &ServeRequest,
115 cancel: &CancellationToken,
116 ) -> Result<(), BackendError>;
117 async fn stop(
118 &self,
119 handle: &ServerState,
120 cancel: &CancellationToken,
121 ) -> Result<(), BackendError>;
122 async fn ready_handle(
123 &self,
124 request: &ServeRequest,
125 _handle: &ServerState,
126 cancel: &CancellationToken,
127 ) -> Result<(), BackendError> {
128 self.ready(request, cancel).await
129 }
130}
131pub struct RuntimeAdapter<'runtime> {
132 pub kind: BackendKind,
133 pub binary: PathBuf,
134 pub http: &'runtime dyn Transport,
135 pub probe: &'runtime dyn ProcessProbe,
136 pub control: &'runtime dyn ProcessControl,
137 pub(crate) ollama_models: Option<PathBuf>,
138}
139pub fn model_id(value: &str) -> Result<(), BackendError> {
140 if value.is_empty()
141 || value.len() > 8192
142 || !value.as_bytes()[0].is_ascii_alphanumeric()
143 || !value.bytes().all(|byte| {
144 byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
145 })
146 || value.split('/').any(|part| part == "..")
147 {
148 return Err(BackendError("invalid runtime model id".into()));
149 }
150 Ok(())
151}
152impl<'runtime> RuntimeAdapter<'runtime> {
153 pub fn new(
154 kind: BackendKind,
155 binary: PathBuf,
156 http: &'runtime dyn Transport,
157 probe: &'runtime dyn ProcessProbe,
158 control: &'runtime dyn ProcessControl,
159 ) -> Self {
160 Self {
161 kind,
162 binary,
163 http,
164 probe,
165 control,
166 ollama_models: None,
167 }
168 }
169 pub fn with_ollama_models(mut self, models: PathBuf) -> Result<Self, BackendError> {
170 if self.kind != BackendKind::Ollama {
171 return Err(BackendError(
172 "model store configuration requires Ollama".into(),
173 ));
174 }
175 crate::command::OllamaCommandContext::new("http://127.0.0.1:11434", &models)?;
176 self.ollama_models = Some(models);
177 Ok(self)
178 }
179 async fn json(
180 &self,
181 endpoint: &str,
182 path: &str,
183 cancel: &CancellationToken,
184 ) -> Result<Value, BackendError> {
185 let request = Request::new(endpoint, path, None, None)
186 .map_err(|error| BackendError(error.to_string()))?;
187 tokio::time::timeout(
188 Duration::from_secs(5),
189 read_json(self.http, request, cancel, 4 * 1024 * 1024),
190 )
191 .await
192 .map_err(|_| BackendError("readiness request timed out".into()))?
193 .map_err(|error| BackendError(error.to_string()))
194 }
195 fn validate(&self, request: &ServeRequest) -> Result<url::Url, BackendError> {
196 model_id(&request.model_id)?;
197 if request.draft_model_path.is_some() && self.kind != BackendKind::LlamaCpp {
198 return Err(BackendError(
199 "a drafter applies only to llama.cpp draft decoding".into(),
200 ));
201 }
202 if self.kind == BackendKind::BitNet
203 && request
204 .context
205 .is_some_and(|value| !(1..=4096).contains(&value))
206 {
207 return Err(BackendError(
208 "BitNet context must be between 1 and 4096 tokens".into(),
209 ));
210 }
211 if !self.binary.is_absolute()
212 || request
213 .context
214 .is_some_and(|value| !(1..=10000000).contains(&value))
215 {
216 return Err(BackendError("invalid runtime configuration".into()));
217 }
218 crate::state::loopback(&request.endpoint).map_err(|error| BackendError(error.to_string()))
219 }
220 fn gguf_args(
221 &self,
222 request: &ServeRequest,
223 host: &str,
224 port: u16,
225 ) -> Result<Vec<String>, BackendError> {
226 let path = request
227 .model_path
228 .as_ref()
229 .filter(|path| path.is_absolute() && path.is_file())
230 .ok_or_else(|| BackendError("verified absolute GGUF path required".into()))?;
231 let mut args = vec![
232 "-m".into(),
233 path.to_string_lossy().into_owned(),
234 "--host".into(),
235 host.into(),
236 "--port".into(),
237 port.to_string(),
238 "--alias".into(),
239 request.model_id.clone(),
240 ];
241 let context = request
242 .context
243 .or_else(|| (self.kind == BackendKind::BitNet).then_some(4096));
244 if let Some(context) = context {
245 args.extend(["--ctx-size".into(), context.to_string()]);
246 }
247 if self.kind == BackendKind::BitNet {
248 args.extend(["-ngl".into(), "0".into()]);
249 }
250 if let Some(directory) = &request.slot_save_path {
251 if self.kind != BackendKind::LlamaCpp {
252 return Err(BackendError("KV slot files apply only to llama.cpp".into()));
253 }
254 if !(directory.is_absolute() && directory.is_dir()) {
255 return Err(BackendError(
256 "KV slot directory must be an existing absolute directory".into(),
257 ));
258 }
259 args.extend([
260 "--slot-save-path".into(),
261 directory.to_string_lossy().into_owned(),
262 ]);
263 }
264 let drafting = request
265 .cache
266 .is_some_and(|profile| profile.speculative == crate::cache::Speculative::Draft);
267 match (&request.draft_model_path, drafting) {
268 (None, false) => {}
269 (Some(draft), true) if self.kind == BackendKind::LlamaCpp => {
270 if !(draft.is_absolute() && draft.is_file())
271 || Some(draft) == request.model_path.as_ref()
272 {
273 return Err(BackendError(
274 "verified absolute drafter GGUF path required".into(),
275 ));
276 }
277 args.extend(["--model-draft".into(), draft.to_string_lossy().into_owned()]);
278 }
279 (None, true) => {
280 return Err(BackendError(
281 "speculative draft decoding needs a verified drafter".into(),
282 ));
283 }
284 _ => {
285 return Err(BackendError(
286 "a drafter applies only to llama.cpp draft decoding".into(),
287 ));
288 }
289 }
290 Ok(args)
291 }
292
293 async fn legacy_bitnet_identity(
294 &self,
295 request: &ServeRequest,
296 props: Value,
297 cancel: &CancellationToken,
298 ) -> Result<(), BackendError> {
299 #[derive(Deserialize)]
300 struct Generation {
301 model: String,
302 n_ctx: u32,
303 }
304 #[derive(Deserialize)]
305 struct Props {
306 default_generation_settings: Generation,
307 total_slots: u32,
308 chat_template: String,
309 }
310 let props: Props = serde_json::from_value(props)
311 .map_err(|_| BackendError("invalid legacy BitNet properties".into()))?;
312 if props.default_generation_settings.model != request.model_id
313 || props.default_generation_settings.n_ctx != request.context.unwrap_or(4096)
314 || props.total_slots != 1
315 || props.chat_template.is_empty()
316 {
317 return Err(BackendError(
318 "legacy BitNet model or context differs".into(),
319 ));
320 }
321 let before = listener(self.probe, &request.endpoint, cancel).await?;
322 if !self.trusts(&before.identity) {
323 return Err(BackendError("untrusted BitNet executable".into()));
324 }
325 let command = self
327 .probe
328 .command_line(before.identity.pid)
329 .await
330 .map_err(|error| BackendError(error.to_string()))?;
331 let mut expected = vec![before.identity.executable.clone()];
332 expected.extend(self.gguf_args(request, &before.address, before.port)?);
333 let after = listener(self.probe, &request.endpoint, cancel).await?;
334 if command != expected || !same_process(&before.identity, &after.identity) {
335 return Err(BackendError("legacy BitNet launch identity differs".into()));
336 }
337 Ok(())
338 }
339
340 async fn ready_once(
341 &self,
342 request: &ServeRequest,
343 cancel: &CancellationToken,
344 ) -> Result<(), BackendError> {
345 self.validate(request)?;
346 match self.kind {
347 BackendKind::Ollama => {
348 let version = self.json(&request.endpoint, "/api/version", cancel).await?;
349 if !version
350 .get("version")
351 .and_then(Value::as_str)
352 .is_some_and(|version| !version.is_empty() && version.len() <= 100)
353 {
354 return Err(BackendError("listener is not Ollama".into()));
355 }
356 crate::ollama_installed::parse_inventory(
357 self.json(&request.endpoint, "/api/tags", cancel).await?,
358 )
359 .map_err(|error| BackendError(error.to_string()))?;
360 }
361 BackendKind::LlamaCpp | BackendKind::BitNet => {
362 let health = self.json(&request.endpoint, "/health", cancel).await?;
363 if health.get("status").and_then(Value::as_str) != Some("ok") {
364 return Err(BackendError("llama.cpp model is not ready".into()));
365 }
366 let props = self.json(&request.endpoint, "/props", cancel).await?;
367 if self.kind == BackendKind::BitNet
368 && props.get("model_path").is_none()
369 && props.get("model_alias").is_none()
370 {
371 return self.legacy_bitnet_identity(request, props, cancel).await;
372 }
373 let path = props.get("model_path").and_then(Value::as_str);
374 let alias = props.get("model_alias").and_then(Value::as_str);
375 if request
376 .model_path
377 .as_ref()
378 .is_some_and(|expected| path != expected.to_str())
379 || alias != Some(request.model_id.as_str())
380 {
381 return Err(BackendError(
382 "llama.cpp loaded artifact or alias differs".into(),
383 ));
384 }
385 }
386 }
387 Ok(())
388 }
389 pub async fn ready(
390 &self,
391 request: &ServeRequest,
392 cancel: &CancellationToken,
393 ) -> Result<(), BackendError> {
394 let operation = async {
395 let mut last = BackendError("runtime is not ready".into());
396 for attempt in 0..20 {
397 match self.ready_once(request, cancel).await {
398 Ok(()) => return Ok(()),
399 Err(error) => last = error,
400 }
401 tokio::time::sleep(Duration::from_millis((100u64 << attempt.min(4)).min(2000)))
402 .await;
403 }
404 Err(last)
405 };
406 tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(30),operation)=>result.map_err(|_|BackendError("runtime readiness timed out".into()))?}
407 }
408 pub async fn serve(
409 &self,
410 request: &ServeRequest,
411 cancel: &CancellationToken,
412 ) -> Result<ServerState, BackendError> {
413 let mut request = request.clone();
414 if self.kind == BackendKind::BitNet {
415 request.context.get_or_insert(4096);
416 }
417 let request = &request;
418 let url = self.validate(request)?;
419 if cancel.is_cancelled() {
420 return Err(BackendError("cancelled".into()));
421 }
422 let occupied = tokio::select! {biased;_=cancel.cancelled()=>return Err(BackendError("cancelled".into())),result=self.control.occupied(&request.endpoint)=>result?};
423 let executable = self
424 .binary
425 .to_str()
426 .ok_or_else(|| BackendError("invalid executable path".into()))?;
427 let (observed, owned) = if occupied {
428 if self.kind == BackendKind::BitNet {
429 return Err(BackendError(
430 "BitNet requires an unused port; refusing to attach to an existing listener"
431 .into(),
432 ));
433 }
434 if crate::cache::effective(request.cache.as_ref()).is_some() {
435 return Err(BackendError(format!(
436 "cache settings apply only to a {} runtime rigspark starts; stop the running server or omit the cache options",
437 self.kind.name()
438 )));
439 }
440 let before = listener(self.probe, &request.endpoint, cancel).await?;
441 if before.identity.executable != executable {
442 return Err(BackendError(
443 "refusing untrusted listener executable".into(),
444 ));
445 }
446 self.ready_once(request, cancel).await?;
447 let after = listener(self.probe, &request.endpoint, cancel).await?;
448 if !same_process(&before.identity, &after.identity) {
449 return Err(BackendError("listener changed during attach".into()));
450 }
451 (after, false)
452 } else {
453 let host = url
454 .host_str()
455 .ok_or_else(|| BackendError("missing runtime host".into()))?
456 .trim_matches(['[', ']']);
457 let port = url
458 .port_or_known_default()
459 .ok_or_else(|| BackendError("missing port".into()))?;
460 let mut env = minimal_env();
461 let delta = crate::cache::launch_delta(request.cache.as_ref(), self.kind.name(), true)
462 .map_err(|error| BackendError(error.to_string()))?;
463 let mut args = match self.kind {
464 BackendKind::Ollama => {
465 if let Some(models) = &self.ollama_models {
466 let context =
467 crate::command::OllamaCommandContext::new(&request.endpoint, models)?;
468 env.extend(context.environment().clone());
469 }
470 env.insert(
471 "OLLAMA_HOST".into(),
472 format!(
473 "{}:{port}",
474 if host.contains(':') {
475 format!("[{host}]")
476 } else {
477 host.into()
478 }
479 ),
480 );
481 vec!["serve".into()]
482 }
483 BackendKind::LlamaCpp | BackendKind::BitNet => {
484 self.gguf_args(request, host, port)?
485 }
486 };
487 args.extend(delta.args);
488 env.extend(delta.env);
489 if cancel.is_cancelled() {
490 return Err(BackendError("cancelled".into()));
491 }
492 let child = self
493 .control
494 .spawn(&SpawnSpec {
495 binary: self.binary.clone(),
496 args,
497 env,
498 })
499 .await?;
500 let observed = wait_owned(
501 child,
502 &request.endpoint,
503 executable,
504 self.probe,
505 cancel,
506 || async { self.ready(request, cancel).await.map_err(|error| error.0) },
507 )
508 .await?;
509 (observed, true)
510 };
511 let state = ServerState {
512 backend: self.kind.name().into(),
513 model_id: request.model_id.clone(),
514 endpoint: request.endpoint.clone(),
515 port: observed.port,
516 owned_by_us: owned,
517 pid: Some(observed.identity.pid),
518 runtime_model_id: None,
519 context: request.context,
520 integrity: None,
521 local_manifest_digest: None,
522 model_path: if self.kind == BackendKind::BitNet {
523 request
524 .model_path
525 .as_ref()
526 .map(|path| path.to_string_lossy().into_owned())
527 } else {
528 None
529 },
530 process_executable: Some(observed.identity.executable),
531 process_started_at: Some(observed.identity.started),
532 auth_token: None,
533 cache: if owned {
534 crate::cache::effective(request.cache.as_ref())
535 } else {
536 None
537 },
538 };
539 state
540 .validate()
541 .map_err(|error| BackendError(error.to_string()))?;
542 Ok(state)
543 }
544 pub async fn stop(
545 &self,
546 handle: &ServerState,
547 cancel: &CancellationToken,
548 ) -> Result<(), BackendError> {
549 handle
550 .validate()
551 .map_err(|error| BackendError(error.to_string()))?;
552 if handle.backend != self.kind.name() {
553 return Err(BackendError("backend handle mismatch".into()));
554 }
555 if !handle.owned_by_us {
556 return Ok(());
557 }
558 let expected = ProcessIdentity {
559 pid: handle
560 .pid
561 .ok_or_else(|| BackendError("missing PID".into()))?,
562 process: String::new(),
563 executable: handle
564 .process_executable
565 .clone()
566 .ok_or_else(|| BackendError("missing executable identity".into()))?,
567 started: handle
568 .process_started_at
569 .clone()
570 .ok_or_else(|| BackendError("missing start identity".into()))?,
571 };
572 if self.binary.to_str() != Some(&expected.executable) {
573 return Err(BackendError("untrusted stop executable".into()));
574 }
575 stop_owned(
576 &handle.endpoint,
577 &expected,
578 self.probe,
579 self.control,
580 cancel,
581 )
582 .await
583 .map_err(BackendError)
584 }
585}
586#[async_trait::async_trait]
587impl BackendAdapter for RuntimeAdapter<'_> {
588 async fn attach_only(
589 &self,
590 request: &ServeRequest,
591 cancel: &CancellationToken,
592 ) -> Result<ServerState, BackendError> {
593 if self.kind == BackendKind::BitNet {
594 return Err(BackendError(
595 "BitNet requires an owned server; use up".into(),
596 ));
597 }
598 self.validate(request)?;
599 if cancel.is_cancelled() {
600 return Err(BackendError("cancelled".into()));
601 }
602 if !self.control.occupied(&request.endpoint).await? {
603 return Err(BackendError(
604 "existing runtime required; refusing to spawn".into(),
605 ));
606 }
607 let before = listener(self.probe, &request.endpoint, cancel).await?;
608 if !self.trusts(&before.identity) {
609 return Err(BackendError("untrusted attached executable".into()));
610 }
611 self.ready_once(request, cancel).await?;
612 let after = listener(self.probe, &request.endpoint, cancel).await?;
613 if !same_process(&before.identity, &after.identity) {
614 return Err(BackendError("listener changed during attachment".into()));
615 }
616 let state=crate::state::RuntimeState::parse(&serde_json::json!({"schemaVersion":2,"active":{"backend":self.name(),"modelId":request.model_id,"endpoint":request.endpoint,"port":after.port,"ownedByUs":false,"pid":after.identity.pid,"processExecutable":after.identity.executable,"processStartedAt":after.identity.started}}).to_string()).map_err(|error|BackendError(error.to_string()))?;
617 state
618 .active
619 .ok_or_else(|| BackendError("missing attached state".into()))
620 }
621 async fn ready_handle(
622 &self,
623 request: &ServeRequest,
624 handle: &ServerState,
625 cancel: &CancellationToken,
626 ) -> Result<(), BackendError> {
627 handle
628 .validate()
629 .map_err(|error| BackendError(error.to_string()))?;
630 if handle.backend != self.name() || handle.endpoint != request.endpoint {
631 return Err(BackendError("readiness handle mismatch".into()));
632 }
633 if self.kind == BackendKind::BitNet
634 && (handle.model_id != request.model_id
635 || handle.model_path.as_deref()
636 != request.model_path.as_ref().and_then(|path| path.to_str())
637 || handle.context != Some(request.context.unwrap_or(4096)))
638 {
639 return Err(BackendError(
640 "BitNet readiness model or context mismatch".into(),
641 ));
642 }
643 let before = listener(self.probe, &handle.endpoint, cancel).await?;
644 crate::identity::capture(handle, &before, self.trusts(&before.identity))
645 .map_err(|error| BackendError(error.to_string()))?;
646 self.ready(request, cancel).await?;
647 let models = self.json(&handle.endpoint, "/v1/models", cancel).await?;
648 #[derive(Deserialize)]
649 struct Model {
650 id: String,
651 }
652 #[derive(Deserialize)]
653 struct Models {
654 data: Vec<Model>,
655 }
656 let models: Models = serde_json::from_value(models)
657 .map_err(|_| BackendError("invalid OpenAI models response".into()))?;
658 if models.data.len() > 10000
659 || models
660 .data
661 .iter()
662 .any(|model| model.id.is_empty() || model.id.len() > 8192)
663 {
664 return Err(BackendError("invalid OpenAI models".into()));
665 }
666 let after = listener(self.probe, &handle.endpoint, cancel).await?;
667 if !same_process(&before.identity, &after.identity) {
668 return Err(BackendError("readiness listener changed".into()));
669 }
670 Ok(())
671 }
672 fn name(&self) -> &'static str {
673 self.kind.name()
674 }
675 fn trusts(&self, identity: &ProcessIdentity) -> bool {
676 self.binary.to_str() == Some(identity.executable.as_str())
677 }
678 async fn chat(
679 &self,
680 handle: &ServerState,
681 input: &crate::ollama_inference::ChatInput,
682 cancel: &CancellationToken,
683 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
684 if matches!(self.kind, BackendKind::LlamaCpp | BackendKind::BitNet) {
685 return crate::openai::OpenAiInference {
686 http: self.http,
687 probe: self.probe,
688 adapter: self,
689 token: None,
690 }
691 .chat(handle, input, cancel)
692 .await;
693 }
694 let expected = expected_process(handle)?;
695 let mut input = input.clone();
696 input.model = handle.runtime_model_id.clone().unwrap_or(input.model);
697 crate::ollama_inference::OllamaInference::new(
698 self.http,
699 self.probe,
700 self.binary
701 .to_str()
702 .ok_or_else(|| BackendError("invalid executable".into()))?,
703 )
704 .chat(&handle.endpoint, &expected, &input, cancel)
705 .await
706 .map_err(|error| BackendError(error.to_string()))
707 }
708 async fn chat_stream(
709 &self,
710 handle: &ServerState,
711 input: &crate::ollama_inference::ChatInput,
712 cancel: &CancellationToken,
713 on_delta: &mut (dyn for<'chunk> FnMut(&'chunk str) + Send),
714 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
715 if matches!(self.kind, BackendKind::LlamaCpp | BackendKind::BitNet) {
716 let result = self.chat(handle, input, cancel).await?;
717 on_delta(&result.content);
718 return Ok(result);
719 }
720 let expected = expected_process(handle)?;
721 let mut input = input.clone();
722 input.model = handle.runtime_model_id.clone().unwrap_or(input.model);
723 crate::ollama_inference::OllamaInference::new(
724 self.http,
725 self.probe,
726 self.binary
727 .to_str()
728 .ok_or_else(|| BackendError("invalid executable".into()))?,
729 )
730 .chat_stream(&handle.endpoint, &expected, &input, cancel, on_delta)
731 .await
732 .map_err(|error| BackendError(error.to_string()))
733 }
734 async fn embed(
735 &self,
736 handle: &ServerState,
737 model: &str,
738 inputs: &[String],
739 cancel: &CancellationToken,
740 ) -> Result<crate::ollama_inference::EmbedResult, BackendError> {
741 if self.kind != BackendKind::Ollama {
742 return Err(BackendError(format!(
743 "{} does not serve embeddings",
744 self.name()
745 )));
746 }
747 let expected = expected_process(handle)?;
748 crate::ollama_inference::OllamaInference::new(
749 self.http,
750 self.probe,
751 self.binary
752 .to_str()
753 .ok_or_else(|| BackendError("invalid executable".into()))?,
754 )
755 .embed(&handle.endpoint, &expected, model, inputs, cancel)
756 .await
757 .map_err(|error| BackendError(error.to_string()))
758 }
759 async fn serve(
760 &self,
761 request: &ServeRequest,
762 cancel: &CancellationToken,
763 ) -> Result<ServerState, BackendError> {
764 RuntimeAdapter::serve(self, request, cancel).await
765 }
766 async fn ready(
767 &self,
768 request: &ServeRequest,
769 cancel: &CancellationToken,
770 ) -> Result<(), BackendError> {
771 RuntimeAdapter::ready(self, request, cancel).await
772 }
773 async fn stop(
774 &self,
775 handle: &ServerState,
776 cancel: &CancellationToken,
777 ) -> Result<(), BackendError> {
778 RuntimeAdapter::stop(self, handle, cancel).await
779 }
780}
781fn expected_process(handle: &ServerState) -> Result<ProcessIdentity, BackendError> {
782 handle
783 .validate()
784 .map_err(|error| BackendError(error.to_string()))?;
785 if handle.backend != "ollama" {
786 return Err(BackendError("Ollama handle required".into()));
787 }
788 Ok(ProcessIdentity {
789 pid: handle
790 .pid
791 .ok_or_else(|| BackendError("expected PID required".into()))?,
792 process: String::new(),
793 executable: handle
794 .process_executable
795 .clone()
796 .ok_or_else(|| BackendError("expected executable required".into()))?,
797 started: handle
798 .process_started_at
799 .clone()
800 .ok_or_else(|| BackendError("expected start identity required".into()))?,
801 })
802}