Skip to main content

rigspark_runtime/
lifecycle.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest},
3    identity::{Confirmation, LiveIdentity, Operation, ProcessProbe, capture},
4    process_control::listener,
5    state::{LockGuard, RuntimeState, ServerState, StateStore},
6};
7use std::time::Duration;
8use tokio_util::sync::CancellationToken;
9pub struct Registry<'runtime> {
10    adapters: Vec<&'runtime dyn BackendAdapter>,
11}
12impl<'runtime> Registry<'runtime> {
13    pub fn new(adapters: Vec<&'runtime dyn BackendAdapter>) -> Self {
14        Self { adapters }
15    }
16    pub fn get(&self, name: &str) -> Result<&'runtime dyn BackendAdapter, BackendError> {
17        let mut matching = self
18            .adapters
19            .iter()
20            .filter(|adapter| adapter.name() == name);
21        let adapter = matching
22            .next()
23            .ok_or_else(|| BackendError("backend not registered".into()))?;
24        if matching.next().is_some() {
25            return Err(BackendError("duplicate backend registration".into()));
26        }
27        Ok(*adapter)
28    }
29}
30pub struct Reviewed {
31    state: RuntimeState,
32    snapshot: Confirmation,
33}
34pub struct Lifecycle<'runtime> {
35    pub store: &'runtime StateStore,
36    pub registry: &'runtime Registry<'runtime>,
37    pub probe: &'runtime dyn ProcessProbe,
38}
39#[async_trait::async_trait]
40pub trait Activation: Send + Sync {
41    async fn finalize(
42        &self,
43        handle: &ServerState,
44        cancel: &CancellationToken,
45    ) -> Result<ServerState, BackendError>;
46}
47struct Unchanged;
48#[async_trait::async_trait]
49impl Activation for Unchanged {
50    async fn finalize(
51        &self,
52        handle: &ServerState,
53        _cancel: &CancellationToken,
54    ) -> Result<ServerState, BackendError> {
55        Ok(handle.clone())
56    }
57}
58impl Lifecycle<'_> {
59    async fn stop_verified(
60        &self,
61        active: &ServerState,
62        cancel: &CancellationToken,
63    ) -> Result<(), BackendError> {
64        let state = RuntimeState::for_active(active.clone());
65        let live = self
66            .live(&state, cancel)
67            .await?
68            .ok_or_else(|| BackendError("owned process identity unavailable".into()))?;
69        let mut handle = active.clone();
70        handle.pid = Some(live.expected.pid);
71        handle.process_executable = Some(live.expected.executable);
72        handle.process_started_at = Some(live.expected.started);
73        self.registry
74            .get(&active.backend)?
75            .stop(&handle, cancel)
76            .await
77    }
78    pub async fn health(
79        &self,
80        active: &ServerState,
81        request: &ServeRequest,
82        cancel: &CancellationToken,
83    ) -> Result<(), BackendError> {
84        let state = RuntimeState::for_active(active.clone());
85        let operation = async {
86            if self
87                .store
88                .read()
89                .map_err(|error| BackendError(error.to_string()))?
90                != state
91            {
92                return Err(BackendError("active state changed".into()));
93            }
94            let before = self.live(&state, cancel).await?;
95            self.registry
96                .get(&active.backend)?
97                .ready_handle(request, active, cancel)
98                .await?;
99            let after = self.live(&state, cancel).await?;
100            if before != after
101                || self
102                    .store
103                    .read()
104                    .map_err(|error| BackendError(error.to_string()))?
105                    != state
106            {
107                return Err(BackendError(
108                    "active runtime changed during health check".into(),
109                ));
110            }
111            Ok(())
112        };
113        tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(10),operation)=>result.map_err(|_|BackendError("active runtime health timed out".into()))?}
114    }
115    async fn live(
116        &self,
117        state: &RuntimeState,
118        cancel: &CancellationToken,
119    ) -> Result<Option<LiveIdentity>, BackendError> {
120        let Some(active) = &state.active else {
121            return Ok(None);
122        };
123        let adapter = self.registry.get(&active.backend)?;
124        let observed = listener(self.probe, &active.endpoint, cancel).await?;
125        capture(active, &observed, adapter.trusts(&observed.identity))
126            .map(Some)
127            .map_err(|error| {
128                BackendError(if active.owned_by_us {
129                    error.to_string()
130                } else {
131                    format!(
132                        "{error}; if the attached {} daemon restarted, run `llmup down --forget` to clear the stale pointer",
133                        active.backend
134                    )
135                })
136            })
137    }
138    /// Clears an attached (not owned) pointer under the state lock without probing or signalling any process.
139    pub async fn forget_attached(
140        &self,
141        cancel: &CancellationToken,
142    ) -> Result<Option<ServerState>, BackendError> {
143        let guard = self.lock(cancel).await?;
144        let prior = self
145            .store
146            .read()
147            .map_err(|error| BackendError(error.to_string()))?;
148        let outcome = match &prior.active {
149            None => Ok(None),
150            Some(active) if active.owned_by_us => Err(BackendError(
151                "--forget only clears attached servers; this server is owned by rigspark, so use `llmup down`"
152                    .into(),
153            )),
154            Some(_) if cancel.is_cancelled() => Err(BackendError("cancelled".into())),
155            Some(_) => self
156                .store
157                .compare_and_write(&guard, &prior, &RuntimeState::default())
158                .map(|()| prior.active.clone())
159                .map_err(|error| BackendError(error.to_string())),
160        };
161        guard
162            .release()
163            .map_err(|error| BackendError(error.to_string()))?;
164        outcome
165    }
166    pub async fn review(
167        &self,
168        target: &str,
169        cancel: &CancellationToken,
170    ) -> Result<Reviewed, BackendError> {
171        let state = self
172            .store
173            .read()
174            .map_err(|error| BackendError(error.to_string()))?;
175        let live = self.live(&state, cancel).await?;
176        let snapshot = Confirmation::prepare(
177            Operation::ReplaceServer,
178            &state,
179            Some(target),
180            live.as_ref(),
181        )
182        .map_err(|error| BackendError(error.to_string()))?;
183        Ok(Reviewed { state, snapshot })
184    }
185    async fn lock(&self, cancel: &CancellationToken) -> Result<LockGuard, BackendError> {
186        if cancel.is_cancelled() {
187            return Err(BackendError("cancelled".into()));
188        }
189        let config = self.store.config.clone();
190        let pending = tokio::task::spawn_blocking(move || {
191            StateStore::new(config).lock(Duration::from_secs(10))
192        });
193        tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=pending=>result.map_err(|_|BackendError("state lock task failed".into()))?.map_err(|error|BackendError(error.to_string()))}
194    }
195    async fn revalidate(
196        &self,
197        target: &str,
198        reviewed: &Reviewed,
199        cancel: &CancellationToken,
200    ) -> Result<RuntimeState, BackendError> {
201        let state = self
202            .store
203            .read()
204            .map_err(|error| BackendError(error.to_string()))?;
205        if state != reviewed.state {
206            return Err(BackendError(
207                "active state changed; retry preparation".into(),
208            ));
209        }
210        let live = self.live(&state, cancel).await?;
211        let snapshot = Confirmation::prepare(
212            Operation::ReplaceServer,
213            &state,
214            Some(target),
215            live.as_ref(),
216        )
217        .map_err(|error| BackendError(error.to_string()))?;
218        reviewed
219            .snapshot
220            .verify(&snapshot)
221            .map_err(|error| BackendError(error.to_string()))?;
222        Ok(state)
223    }
224    pub async fn replace(
225        &self,
226        backend: &str,
227        request: &ServeRequest,
228        reviewed: &Reviewed,
229        cancel: &CancellationToken,
230    ) -> Result<ServerState, BackendError> {
231        self.replace_with(backend, request, reviewed, cancel, &Unchanged)
232            .await
233    }
234    pub async fn replace_with(
235        &self,
236        backend: &str,
237        request: &ServeRequest,
238        reviewed: &Reviewed,
239        cancel: &CancellationToken,
240        activation: &dyn Activation,
241    ) -> Result<ServerState, BackendError> {
242        let adapter = self.registry.get(backend)?;
243        crate::cache::launch_delta(request.cache.as_ref(), backend, true)
244            .map_err(|error| BackendError(error.to_string()))?;
245        let guard = self.lock(cancel).await?;
246        let prior = self.revalidate(&request.model_id, reviewed, cancel).await?;
247        if let Some(active) = &prior.active
248            && active.owned_by_us
249        {
250            self.stop_verified(active, cancel).await?;
251            self.store
252                .write(&guard, &RuntimeState::default())
253                .map_err(|error| BackendError(error.to_string()))?;
254        }
255        let expected_state = if prior
256            .active
257            .as_ref()
258            .is_some_and(|active| active.owned_by_us)
259        {
260            RuntimeState::default()
261        } else {
262            prior.clone()
263        };
264        let handle = adapter.serve(request, cancel).await?;
265        let outcome = async {
266            adapter.ready_handle(request, &handle, cancel).await?;
267            let final_state = activation.finalize(&handle, cancel).await?;
268            if final_state.pid != handle.pid
269                || final_state.endpoint != handle.endpoint
270                || final_state.owned_by_us != handle.owned_by_us
271                || final_state.backend != backend
272                || final_state.model_id != request.model_id
273                || final_state.process_executable != handle.process_executable
274                || final_state.process_started_at != handle.process_started_at
275                || final_state.cache != handle.cache
276            {
277                return Err(BackendError("activation changed runtime ownership".into()));
278            }
279            if cancel.is_cancelled() {
280                return Err(BackendError("cancelled".into()));
281            }
282            self.live(&RuntimeState::for_active(final_state.clone()), cancel)
283                .await?;
284            self.store
285                .compare_and_write(
286                    &guard,
287                    &expected_state,
288                    &RuntimeState::for_active(final_state.clone()),
289                )
290                .map_err(|error| BackendError(error.to_string()))?;
291            Ok(final_state)
292        }
293        .await;
294        if outcome.is_err() && handle.owned_by_us {
295            adapter.stop(&handle, &CancellationToken::new()).await?;
296        }
297        guard
298            .release()
299            .map_err(|error| BackendError(error.to_string()))?;
300        outcome
301    }
302    pub async fn switch_pointer(
303        &self,
304        target: &str,
305        reviewed: &Reviewed,
306        cancel: &CancellationToken,
307    ) -> Result<ServerState, BackendError> {
308        let guard = self.lock(cancel).await?;
309        let current = self.revalidate(target, reviewed, cancel).await?;
310        let mut active = current
311            .active
312            .clone()
313            .ok_or_else(|| BackendError("no active server to switch".into()))?;
314        if active.backend != "ollama" {
315            return Err(BackendError(
316                "single-model or delegated runtime requires up".into(),
317            ));
318        }
319        self.registry
320            .get("ollama")?
321            .ready_handle(
322                &ServeRequest {
323                    model_id: target.into(),
324                    endpoint: active.endpoint.clone(),
325                    model_path: None,
326                    context: None,
327                    cache: None,
328                },
329                &active,
330                cancel,
331            )
332            .await?;
333        self.revalidate(target, reviewed, cancel).await?;
334        active.model_id = target.into();
335        active.runtime_model_id = None;
336        active.context = None;
337        active.integrity = None;
338        active.local_manifest_digest = None;
339        self.store
340            .compare_and_write(&guard, &current, &RuntimeState::for_active(active.clone()))
341            .map_err(|error| BackendError(error.to_string()))?;
342        guard
343            .release()
344            .map_err(|error| BackendError(error.to_string()))?;
345        Ok(active)
346    }
347    pub async fn attach_installed(
348        &self,
349        request: &ServeRequest,
350        reviewed: &Reviewed,
351        cancel: &CancellationToken,
352        activation: &dyn Activation,
353    ) -> Result<ServerState, BackendError> {
354        if crate::cache::effective(request.cache.as_ref()).is_some() {
355            return Err(BackendError(
356                "cache settings cannot be applied to an installed model on a running daemon".into(),
357            ));
358        }
359        let guard = self.lock(cancel).await?;
360        let prior = self.revalidate(&request.model_id, reviewed, cancel).await?;
361        if prior.active.as_ref().is_some_and(|active| {
362            active.owned_by_us
363                && (active.backend != "ollama" || active.endpoint != request.endpoint)
364        }) {
365            return Err(BackendError(
366                "stop the prior owned runtime before attaching another daemon".into(),
367            ));
368        }
369        let adapter = self.registry.get("ollama")?;
370        let handle = adapter.attach_only(request, cancel).await?;
371        if handle.owned_by_us {
372            adapter.stop(&handle, &CancellationToken::new()).await?;
373            return Err(BackendError(
374                "installed activation requires an existing daemon".into(),
375            ));
376        }
377        let mut active = activation.finalize(&handle, cancel).await?;
378        if active.pid != handle.pid
379            || active.endpoint != handle.endpoint
380            || active.backend != "ollama"
381            || active.owned_by_us
382            || active.process_executable != handle.process_executable
383            || active.process_started_at != handle.process_started_at
384        {
385            return Err(BackendError(
386                "installed activation changed daemon identity".into(),
387            ));
388        }
389        adapter.ready_handle(request, &active, cancel).await?;
390        active.owned_by_us = prior
391            .active
392            .as_ref()
393            .is_some_and(|prior| prior.owned_by_us && prior.pid == active.pid);
394        self.live(&RuntimeState::for_active(active.clone()), cancel)
395            .await?;
396        self.store
397            .compare_and_write(&guard, &prior, &RuntimeState::for_active(active.clone()))
398            .map_err(|error| BackendError(error.to_string()))?;
399        guard
400            .release()
401            .map_err(|error| BackendError(error.to_string()))?;
402        Ok(active)
403    }
404    pub async fn down(
405        &self,
406        cancel: &CancellationToken,
407    ) -> Result<Option<ServerState>, BackendError> {
408        self.down_with_target(cancel, || Ok(None)).await
409    }
410    pub async fn down_with_target(
411        &self,
412        cancel: &CancellationToken,
413        resolve_target: impl FnOnce() -> Result<Option<String>, BackendError> + Send,
414    ) -> Result<Option<ServerState>, BackendError> {
415        if cancel.is_cancelled() {
416            return Err(BackendError("cancelled".into()));
417        }
418        if self
419            .store
420            .read()
421            .map_err(|error| BackendError(error.to_string()))?
422            .active
423            .is_none()
424        {
425            return Ok(None);
426        }
427        let guard = self.lock(cancel).await?;
428        let prepared = self
429            .store
430            .read()
431            .map_err(|error| BackendError(error.to_string()))?;
432        let Some(active) = &prepared.active else {
433            return Ok(None);
434        };
435        if let Some(target) = resolve_target()?
436            && target != active.model_id
437        {
438            return Err(BackendError(format!(
439                "{target} is not the active model ({})",
440                active.model_id
441            )));
442        }
443        let operation = if active.owned_by_us {
444            Operation::Down
445        } else {
446            Operation::Detach
447        };
448        let live = self.live(&prepared, cancel).await?;
449        let approved = Confirmation::prepare(operation, &prepared, None, live.as_ref())
450            .map_err(|error| BackendError(error.to_string()))?;
451        let current = self
452            .store
453            .read()
454            .map_err(|error| BackendError(error.to_string()))?;
455        let live = self.live(&current, cancel).await?;
456        let current_snapshot = Confirmation::prepare(operation, &current, None, live.as_ref())
457            .map_err(|error| BackendError(error.to_string()))?;
458        approved
459            .verify(&current_snapshot)
460            .map_err(|error| BackendError(error.to_string()))?;
461        if cancel.is_cancelled() {
462            return Err(BackendError("cancelled".into()));
463        }
464        self.store
465            .compare_and_write(&guard, &current, &RuntimeState::default())
466            .map_err(|error| BackendError(error.to_string()))?;
467        if active.owned_by_us
468            && let Err(error) = self.stop_verified(active, cancel).await
469        {
470            self.store.write(&guard, &prepared).map_err(|restore| {
471                BackendError(format!("{error}; restoring state failed: {restore}"))
472            })?;
473            return Err(error);
474        }
475        guard
476            .release()
477            .map_err(|error| BackendError(error.to_string()))?;
478        Ok(prepared.active)
479    }
480}