Skip to main content

rigspark_runtime/
lifecycle.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest},
3    identity::{Confirmation, LiveIdentity, Operation, ProcessProbe, capture},
4    process_control::listener,
5    state::{LockGuard, RuntimeState, ServerState, StateStore},
6};
7use std::time::Duration;
8use tokio_util::sync::CancellationToken;
9pub struct Registry<'runtime> {
10    adapters: Vec<&'runtime dyn BackendAdapter>,
11}
12impl<'runtime> Registry<'runtime> {
13    pub fn new(adapters: Vec<&'runtime dyn BackendAdapter>) -> Self {
14        Self { adapters }
15    }
16    pub fn get(&self, name: &str) -> Result<&'runtime dyn BackendAdapter, BackendError> {
17        let mut matching = self
18            .adapters
19            .iter()
20            .filter(|adapter| adapter.name() == name);
21        let adapter = matching
22            .next()
23            .ok_or_else(|| BackendError("backend not registered".into()))?;
24        if matching.next().is_some() {
25            return Err(BackendError("duplicate backend registration".into()));
26        }
27        Ok(*adapter)
28    }
29}
30pub struct Reviewed {
31    state: RuntimeState,
32    snapshot: Confirmation,
33}
34pub struct Lifecycle<'runtime> {
35    pub store: &'runtime StateStore,
36    pub registry: &'runtime Registry<'runtime>,
37    pub probe: &'runtime dyn ProcessProbe,
38}
39#[async_trait::async_trait]
40pub trait Activation: Send + Sync {
41    async fn finalize(
42        &self,
43        handle: &ServerState,
44        cancel: &CancellationToken,
45    ) -> Result<ServerState, BackendError>;
46}
47struct Unchanged;
48#[async_trait::async_trait]
49impl Activation for Unchanged {
50    async fn finalize(
51        &self,
52        handle: &ServerState,
53        _cancel: &CancellationToken,
54    ) -> Result<ServerState, BackendError> {
55        Ok(handle.clone())
56    }
57}
58impl Lifecycle<'_> {
59    async fn stop_verified(
60        &self,
61        active: &ServerState,
62        cancel: &CancellationToken,
63    ) -> Result<(), BackendError> {
64        let state = RuntimeState::for_active(active.clone());
65        let live = self
66            .live(&state, cancel)
67            .await?
68            .ok_or_else(|| BackendError("owned process identity unavailable".into()))?;
69        let mut handle = active.clone();
70        handle.pid = Some(live.expected.pid);
71        handle.process_executable = Some(live.expected.executable);
72        handle.process_started_at = Some(live.expected.started);
73        self.registry
74            .get(&active.backend)?
75            .stop(&handle, cancel)
76            .await
77    }
78    pub async fn health(
79        &self,
80        active: &ServerState,
81        request: &ServeRequest,
82        cancel: &CancellationToken,
83    ) -> Result<(), BackendError> {
84        let state = RuntimeState::for_active(active.clone());
85        let operation = async {
86            if self
87                .store
88                .read()
89                .map_err(|error| BackendError(error.to_string()))?
90                != state
91            {
92                return Err(BackendError("active state changed".into()));
93            }
94            let before = self.live(&state, cancel).await?;
95            self.registry
96                .get(&active.backend)?
97                .ready_handle(request, active, cancel)
98                .await?;
99            let after = self.live(&state, cancel).await?;
100            if before != after
101                || self
102                    .store
103                    .read()
104                    .map_err(|error| BackendError(error.to_string()))?
105                    != state
106            {
107                return Err(BackendError(
108                    "active runtime changed during health check".into(),
109                ));
110            }
111            Ok(())
112        };
113        tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(10),operation)=>result.map_err(|_|BackendError("active runtime health timed out".into()))?}
114    }
115    async fn live(
116        &self,
117        state: &RuntimeState,
118        cancel: &CancellationToken,
119    ) -> Result<Option<LiveIdentity>, BackendError> {
120        let Some(active) = &state.active else {
121            return Ok(None);
122        };
123        let adapter = self.registry.get(&active.backend)?;
124        let observed = listener(self.probe, &active.endpoint, cancel).await?;
125        capture(active, &observed, adapter.trusts(&observed.identity))
126            .map(Some)
127            .map_err(|error| {
128                BackendError(if active.owned_by_us {
129                    error.to_string()
130                } else {
131                    format!(
132                        "{error}; if the attached {} daemon restarted, run `llmup down --forget` to clear the stale pointer",
133                        active.backend
134                    )
135                })
136            })
137    }
138    /// Clears an attached (not owned) pointer under the state lock without probing or signalling any process.
139    pub async fn forget_attached(
140        &self,
141        cancel: &CancellationToken,
142    ) -> Result<Option<ServerState>, BackendError> {
143        let guard = self.lock(cancel).await?;
144        let prior = self
145            .store
146            .read()
147            .map_err(|error| BackendError(error.to_string()))?;
148        let outcome = match &prior.active {
149            None => Ok(None),
150            Some(active) if active.owned_by_us => Err(BackendError(
151                "--forget only clears attached servers; this server is owned by rigspark, so use `llmup down`"
152                    .into(),
153            )),
154            Some(_) if cancel.is_cancelled() => Err(BackendError("cancelled".into())),
155            Some(_) => self
156                .store
157                .compare_and_write(&guard, &prior, &RuntimeState::default())
158                .map(|()| prior.active.clone())
159                .map_err(|error| BackendError(error.to_string())),
160        };
161        guard
162            .release()
163            .map_err(|error| BackendError(error.to_string()))?;
164        outcome
165    }
166    /// Clears a record whose process is provably gone (after a reboot, or when the daemon it
167    /// attached to exited), so a stale pointer cannot wedge every later launch. Never probes
168    /// listeners or signals processes; a live PID or unknown liveness keeps the record.
169    pub async fn forget_dead(
170        &self,
171        cancel: &CancellationToken,
172    ) -> Result<Option<ServerState>, BackendError> {
173        let Some(pid) = self
174            .store
175            .read()
176            .map_err(|error| BackendError(error.to_string()))?
177            .active
178            .and_then(|active| active.pid)
179        else {
180            return Ok(None);
181        };
182        if !matches!(self.probe.alive(pid).await, Ok(false)) {
183            return Ok(None);
184        }
185        let guard = self.lock(cancel).await?;
186        let prior = self
187            .store
188            .read()
189            .map_err(|error| BackendError(error.to_string()))?;
190        let outcome = match &prior.active {
191            Some(active) if active.pid == Some(pid) => self
192                .store
193                .compare_and_write(&guard, &prior, &RuntimeState::default())
194                .map(|()| prior.active.clone())
195                .map_err(|error| BackendError(error.to_string())),
196            _ => Ok(None),
197        };
198        guard
199            .release()
200            .map_err(|error| BackendError(error.to_string()))?;
201        outcome
202    }
203    pub async fn review(
204        &self,
205        target: &str,
206        cancel: &CancellationToken,
207    ) -> Result<Reviewed, BackendError> {
208        self.forget_dead(cancel).await?;
209        let state = self
210            .store
211            .read()
212            .map_err(|error| BackendError(error.to_string()))?;
213        let live = self.live(&state, cancel).await?;
214        let snapshot = Confirmation::prepare(
215            Operation::ReplaceServer,
216            &state,
217            Some(target),
218            live.as_ref(),
219        )
220        .map_err(|error| BackendError(error.to_string()))?;
221        Ok(Reviewed { state, snapshot })
222    }
223    async fn lock(&self, cancel: &CancellationToken) -> Result<LockGuard, BackendError> {
224        if cancel.is_cancelled() {
225            return Err(BackendError("cancelled".into()));
226        }
227        let config = self.store.config.clone();
228        let pending = tokio::task::spawn_blocking(move || {
229            StateStore::new(config).lock(Duration::from_secs(10))
230        });
231        tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=pending=>result.map_err(|_|BackendError("state lock task failed".into()))?.map_err(|error|BackendError(error.to_string()))}
232    }
233    async fn revalidate(
234        &self,
235        target: &str,
236        reviewed: &Reviewed,
237        cancel: &CancellationToken,
238    ) -> Result<RuntimeState, BackendError> {
239        let state = self
240            .store
241            .read()
242            .map_err(|error| BackendError(error.to_string()))?;
243        if state != reviewed.state {
244            return Err(BackendError(
245                "active state changed; retry preparation".into(),
246            ));
247        }
248        let live = self.live(&state, cancel).await?;
249        let snapshot = Confirmation::prepare(
250            Operation::ReplaceServer,
251            &state,
252            Some(target),
253            live.as_ref(),
254        )
255        .map_err(|error| BackendError(error.to_string()))?;
256        reviewed
257            .snapshot
258            .verify(&snapshot)
259            .map_err(|error| BackendError(error.to_string()))?;
260        Ok(state)
261    }
262    pub async fn replace(
263        &self,
264        backend: &str,
265        request: &ServeRequest,
266        reviewed: &Reviewed,
267        cancel: &CancellationToken,
268    ) -> Result<ServerState, BackendError> {
269        self.replace_with(backend, request, reviewed, cancel, &Unchanged)
270            .await
271    }
272    pub async fn replace_with(
273        &self,
274        backend: &str,
275        request: &ServeRequest,
276        reviewed: &Reviewed,
277        cancel: &CancellationToken,
278        activation: &dyn Activation,
279    ) -> Result<ServerState, BackendError> {
280        let adapter = self.registry.get(backend)?;
281        crate::cache::launch_delta(request.cache.as_ref(), backend, true)
282            .map_err(|error| BackendError(error.to_string()))?;
283        let guard = self.lock(cancel).await?;
284        let prior = self.revalidate(&request.model_id, reviewed, cancel).await?;
285        if let Some(active) = &prior.active
286            && active.owned_by_us
287        {
288            self.stop_verified(active, cancel).await?;
289            self.store
290                .write(&guard, &RuntimeState::default())
291                .map_err(|error| BackendError(error.to_string()))?;
292        }
293        let expected_state = if prior
294            .active
295            .as_ref()
296            .is_some_and(|active| active.owned_by_us)
297        {
298            RuntimeState::default()
299        } else {
300            prior.clone()
301        };
302        let handle = adapter.serve(request, cancel).await?;
303        let outcome = async {
304            adapter.ready_handle(request, &handle, cancel).await?;
305            let final_state = activation.finalize(&handle, cancel).await?;
306            if final_state.pid != handle.pid
307                || final_state.endpoint != handle.endpoint
308                || final_state.owned_by_us != handle.owned_by_us
309                || final_state.backend != backend
310                || final_state.model_id != request.model_id
311                || final_state.process_executable != handle.process_executable
312                || final_state.process_started_at != handle.process_started_at
313                || final_state.cache != handle.cache
314            {
315                return Err(BackendError("activation changed runtime ownership".into()));
316            }
317            if cancel.is_cancelled() {
318                return Err(BackendError("cancelled".into()));
319            }
320            self.live(&RuntimeState::for_active(final_state.clone()), cancel)
321                .await?;
322            self.store
323                .compare_and_write(
324                    &guard,
325                    &expected_state,
326                    &RuntimeState::for_active(final_state.clone()),
327                )
328                .map_err(|error| BackendError(error.to_string()))?;
329            Ok(final_state)
330        }
331        .await;
332        if outcome.is_err() && handle.owned_by_us {
333            adapter.stop(&handle, &CancellationToken::new()).await?;
334        }
335        guard
336            .release()
337            .map_err(|error| BackendError(error.to_string()))?;
338        outcome
339    }
340    pub async fn switch_pointer(
341        &self,
342        target: &str,
343        reviewed: &Reviewed,
344        cancel: &CancellationToken,
345    ) -> Result<ServerState, BackendError> {
346        let guard = self.lock(cancel).await?;
347        let current = self.revalidate(target, reviewed, cancel).await?;
348        let mut active = current
349            .active
350            .clone()
351            .ok_or_else(|| BackendError("no active server to switch".into()))?;
352        if active.backend != "ollama" {
353            return Err(BackendError(
354                "single-model or delegated runtime requires up".into(),
355            ));
356        }
357        self.registry
358            .get("ollama")?
359            .ready_handle(
360                &ServeRequest {
361                    draft_model_path: None,
362                    model_id: target.into(),
363                    endpoint: active.endpoint.clone(),
364                    model_path: None,
365                    context: None,
366                    cache: None,
367                },
368                &active,
369                cancel,
370            )
371            .await?;
372        self.revalidate(target, reviewed, cancel).await?;
373        active.model_id = target.into();
374        active.runtime_model_id = None;
375        active.context = None;
376        active.integrity = None;
377        active.local_manifest_digest = None;
378        self.store
379            .compare_and_write(&guard, &current, &RuntimeState::for_active(active.clone()))
380            .map_err(|error| BackendError(error.to_string()))?;
381        guard
382            .release()
383            .map_err(|error| BackendError(error.to_string()))?;
384        Ok(active)
385    }
386    pub async fn attach_installed(
387        &self,
388        request: &ServeRequest,
389        reviewed: &Reviewed,
390        cancel: &CancellationToken,
391        activation: &dyn Activation,
392    ) -> Result<ServerState, BackendError> {
393        if crate::cache::effective(request.cache.as_ref()).is_some() {
394            return Err(BackendError(
395                "cache settings cannot be applied to an installed model on a running daemon".into(),
396            ));
397        }
398        let guard = self.lock(cancel).await?;
399        let prior = self.revalidate(&request.model_id, reviewed, cancel).await?;
400        if prior.active.as_ref().is_some_and(|active| {
401            active.owned_by_us
402                && (active.backend != "ollama" || active.endpoint != request.endpoint)
403        }) {
404            return Err(BackendError(
405                "stop the prior owned runtime before attaching another daemon".into(),
406            ));
407        }
408        let adapter = self.registry.get("ollama")?;
409        let handle = adapter.attach_only(request, cancel).await?;
410        if handle.owned_by_us {
411            adapter.stop(&handle, &CancellationToken::new()).await?;
412            return Err(BackendError(
413                "installed activation requires an existing daemon".into(),
414            ));
415        }
416        let mut active = activation.finalize(&handle, cancel).await?;
417        if active.pid != handle.pid
418            || active.endpoint != handle.endpoint
419            || active.backend != "ollama"
420            || active.owned_by_us
421            || active.process_executable != handle.process_executable
422            || active.process_started_at != handle.process_started_at
423        {
424            return Err(BackendError(
425                "installed activation changed daemon identity".into(),
426            ));
427        }
428        adapter.ready_handle(request, &active, cancel).await?;
429        active.owned_by_us = prior
430            .active
431            .as_ref()
432            .is_some_and(|prior| prior.owned_by_us && prior.pid == active.pid);
433        self.live(&RuntimeState::for_active(active.clone()), cancel)
434            .await?;
435        self.store
436            .compare_and_write(&guard, &prior, &RuntimeState::for_active(active.clone()))
437            .map_err(|error| BackendError(error.to_string()))?;
438        guard
439            .release()
440            .map_err(|error| BackendError(error.to_string()))?;
441        Ok(active)
442    }
443    pub async fn down(
444        &self,
445        cancel: &CancellationToken,
446    ) -> Result<Option<ServerState>, BackendError> {
447        self.down_with_target(cancel, || Ok(None)).await
448    }
449    pub async fn down_with_target(
450        &self,
451        cancel: &CancellationToken,
452        resolve_target: impl FnOnce() -> Result<Option<String>, BackendError> + Send,
453    ) -> Result<Option<ServerState>, BackendError> {
454        if cancel.is_cancelled() {
455            return Err(BackendError("cancelled".into()));
456        }
457        if self
458            .store
459            .read()
460            .map_err(|error| BackendError(error.to_string()))?
461            .active
462            .is_none()
463        {
464            return Ok(None);
465        }
466        let guard = self.lock(cancel).await?;
467        let prepared = self
468            .store
469            .read()
470            .map_err(|error| BackendError(error.to_string()))?;
471        let Some(active) = &prepared.active else {
472            return Ok(None);
473        };
474        if let Some(target) = resolve_target()?
475            && target != active.model_id
476        {
477            return Err(BackendError(format!(
478                "{target} is not the active model ({})",
479                active.model_id
480            )));
481        }
482        // The record the caller asked to stop may be cleared once its process is provably gone.
483        if let Some(pid) = active.pid
484            && matches!(self.probe.alive(pid).await, Ok(false))
485        {
486            let outcome = self
487                .store
488                .compare_and_write(&guard, &prepared, &RuntimeState::default())
489                .map(|()| prepared.active.clone())
490                .map_err(|error| BackendError(error.to_string()));
491            guard
492                .release()
493                .map_err(|error| BackendError(error.to_string()))?;
494            return outcome;
495        }
496        let operation = if active.owned_by_us {
497            Operation::Down
498        } else {
499            Operation::Detach
500        };
501        let live = self.live(&prepared, cancel).await?;
502        let approved = Confirmation::prepare(operation, &prepared, None, live.as_ref())
503            .map_err(|error| BackendError(error.to_string()))?;
504        let current = self
505            .store
506            .read()
507            .map_err(|error| BackendError(error.to_string()))?;
508        let live = self.live(&current, cancel).await?;
509        let current_snapshot = Confirmation::prepare(operation, &current, None, live.as_ref())
510            .map_err(|error| BackendError(error.to_string()))?;
511        approved
512            .verify(&current_snapshot)
513            .map_err(|error| BackendError(error.to_string()))?;
514        if cancel.is_cancelled() {
515            return Err(BackendError("cancelled".into()));
516        }
517        self.store
518            .compare_and_write(&guard, &current, &RuntimeState::default())
519            .map_err(|error| BackendError(error.to_string()))?;
520        if active.owned_by_us
521            && let Err(error) = self.stop_verified(active, cancel).await
522        {
523            self.store.write(&guard, &prepared).map_err(|restore| {
524                BackendError(format!("{error}; restoring state failed: {restore}"))
525            })?;
526            return Err(error);
527        }
528        guard
529            .release()
530            .map_err(|error| BackendError(error.to_string()))?;
531        Ok(prepared.active)
532    }
533}