Skip to main content

rigspark_runtime/application/
events.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest},
3    identity::ProcessIdentity,
4    state::ServerState,
5};
6use std::{
7    collections::VecDeque,
8    future::Future,
9    sync::{Arc, Mutex},
10};
11use tokio::sync::mpsc::{self, Receiver, Sender};
12use tokio_util::sync::CancellationToken;
13
14pub const EVENT_CAPACITY: usize = 64;
15
16#[derive(Clone, Copy)]
17pub(crate) enum LifecycleWarning {
18    InstalledBypass,
19    EstimatedFit,
20    SizeOnly,
21}
22
23impl LifecycleWarning {
24    fn message(self) -> &'static str {
25        match self {
26            Self::InstalledBypass => {
27                "up: bypassing estimated fit; local content integrity is not catalog verification; throughput may be unknown"
28            }
29            Self::EstimatedFit => {
30                "up: requested quantization may not fit this hardware; continuing because it was explicitly requested"
31            }
32            Self::SizeOnly => {
33                "up: weights passed a size-floor check; no catalog SHA-256 was available"
34            }
35        }
36    }
37}
38
39#[derive(Clone, Default)]
40pub struct DiagnosticObserver {
41    state: Arc<Mutex<DiagnosticSnapshot>>,
42}
43
44#[derive(Clone, Default)]
45pub struct DiagnosticSnapshot {
46    warnings: [u64; 3],
47    progress: VecDeque<(u64, u64)>,
48    omitted_progress: u64,
49}
50
51impl DiagnosticObserver {
52    pub fn snapshot(&self) -> DiagnosticSnapshot {
53        self.state
54            .lock()
55            .unwrap_or_else(|error| error.into_inner())
56            .clone()
57    }
58
59    fn warning(&self, warning: LifecycleWarning) {
60        let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
61        let count = &mut state.warnings[warning as usize];
62        *count = count.saturating_add(1);
63    }
64
65    fn progress(&self, completed: u64, total: u64) {
66        let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
67        if state.progress.len() == EVENT_CAPACITY {
68            state.progress.pop_front();
69            state.omitted_progress = state.omitted_progress.saturating_add(1);
70        }
71        state.progress.push_back((completed, total));
72    }
73}
74
75impl DiagnosticSnapshot {
76    pub fn text(&self) -> String {
77        let mut lines = Vec::new();
78        for warning in [
79            LifecycleWarning::InstalledBypass,
80            LifecycleWarning::EstimatedFit,
81            LifecycleWarning::SizeOnly,
82        ] {
83            let count = self.warnings[warning as usize];
84            if count > 0 {
85                lines.push(format!("{} ({count} occurrences)", warning.message()));
86            }
87        }
88        if self.omitted_progress > 0 {
89            lines.push(format!(
90                "{} earlier progress diagnostics omitted",
91                self.omitted_progress
92            ));
93        }
94        for (completed, total) in &self.progress {
95            lines.push(format!("  [redacted filename]: {completed}/{total} bytes"));
96        }
97        if lines.is_empty() {
98            String::new()
99        } else {
100            format!("{}\n", lines.join("\n"))
101        }
102    }
103}
104
105pub(crate) fn warning(observer: Option<&LifecycleObserver>, warning: LifecycleWarning) {
106    match observer.and_then(|observer| observer.diagnostics.as_ref()) {
107        Some(diagnostics) => diagnostics.warning(warning),
108        None => eprintln!("{}", warning.message()),
109    }
110}
111
112pub(crate) fn download_progress(
113    diagnostics: Option<&DiagnosticObserver>,
114    completed: u64,
115    total: u64,
116    file: &str,
117) {
118    match diagnostics {
119        Some(diagnostics) => diagnostics.progress(completed, total),
120        None => eprintln!(
121            "  {}: {completed}/{total} bytes",
122            rigspark_core::reports::strip_control(file)
123        ),
124    }
125}
126
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub enum LifecycleScope {
129    Runtime,
130    AcquisitionDaemon,
131}
132
133#[derive(Debug, Clone, Copy, PartialEq, Eq)]
134pub enum LifecycleStage {
135    Acquisition,
136    Verification,
137    AcquisitionVerification,
138    Start,
139    Attach,
140    Activation,
141    Readiness,
142    Stop,
143}
144
145#[derive(Debug, Clone, Copy, PartialEq, Eq)]
146pub enum LifecycleStatus {
147    Started,
148    Completed,
149    Failed,
150}
151
152/// Operation-level evidence, not command success or state-commit confirmation.
153/// Start covers the adapter's serve operation, which may reuse an existing server.
154/// AcquisitionVerification covers acquisition APIs that verify before returning.
155/// No strings are carried (maximum string payload: 0 bytes), including diagnostics,
156/// model identifiers, paths, endpoints, credentials, or subprocess output.
157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub struct LifecycleEvent {
159    pub scope: LifecycleScope,
160    pub stage: LifecycleStage,
161    pub status: LifecycleStatus,
162}
163
164/// Best-effort, nonblocking observations. Full or closed queues drop new events.
165/// Consumers must use the command result, not events, to determine final success.
166pub struct LifecycleObserver {
167    sender: Sender<LifecycleEvent>,
168    pub(crate) diagnostics: Option<DiagnosticObserver>,
169}
170
171impl LifecycleObserver {
172    pub fn channel() -> (Self, Receiver<LifecycleEvent>) {
173        let (sender, receiver) = mpsc::channel(EVENT_CAPACITY);
174        (
175            Self {
176                sender,
177                diagnostics: None,
178            },
179            receiver,
180        )
181    }
182
183    pub fn with_diagnostics(mut self, diagnostics: DiagnosticObserver) -> Self {
184        self.diagnostics = Some(diagnostics);
185        self
186    }
187
188    fn emit(&self, scope: LifecycleScope, stage: LifecycleStage, status: LifecycleStatus) {
189        let _ = self.sender.try_send(LifecycleEvent {
190            scope,
191            stage,
192            status,
193        });
194    }
195}
196
197pub(crate) async fn observe<T>(
198    observer: Option<&LifecycleObserver>,
199    scope: LifecycleScope,
200    stage: LifecycleStage,
201    operation: impl Future<Output = Result<T, BackendError>>,
202) -> Result<T, BackendError> {
203    if let Some(observer) = observer {
204        observer.emit(scope, stage, LifecycleStatus::Started);
205    }
206    let result = operation.await;
207    if let Some(observer) = observer {
208        observer.emit(
209            scope,
210            stage,
211            if result.is_ok() {
212                LifecycleStatus::Completed
213            } else {
214                LifecycleStatus::Failed
215            },
216        );
217    }
218    result
219}
220
221pub(super) struct ObservedAdapter<'runtime> {
222    inner: &'runtime dyn BackendAdapter,
223    observer: Option<&'runtime LifecycleObserver>,
224    scope: LifecycleScope,
225}
226
227impl<'runtime> ObservedAdapter<'runtime> {
228    pub(super) fn new(
229        inner: &'runtime dyn BackendAdapter,
230        observer: Option<&'runtime LifecycleObserver>,
231        scope: LifecycleScope,
232    ) -> Self {
233        Self {
234            inner,
235            observer,
236            scope,
237        }
238    }
239}
240
241#[async_trait::async_trait]
242impl BackendAdapter for ObservedAdapter<'_> {
243    fn name(&self) -> &'static str {
244        self.inner.name()
245    }
246    fn trusts(&self, identity: &ProcessIdentity) -> bool {
247        self.inner.trusts(identity)
248    }
249    fn can_embed(&self) -> bool {
250        self.inner.can_embed()
251    }
252    fn can_stream(&self) -> bool {
253        self.inner.can_stream()
254    }
255
256    async fn serve(
257        &self,
258        request: &ServeRequest,
259        cancel: &CancellationToken,
260    ) -> Result<ServerState, BackendError> {
261        observe(
262            self.observer,
263            self.scope,
264            LifecycleStage::Start,
265            self.inner.serve(request, cancel),
266        )
267        .await
268    }
269    async fn attach_only(
270        &self,
271        request: &ServeRequest,
272        cancel: &CancellationToken,
273    ) -> Result<ServerState, BackendError> {
274        observe(
275            self.observer,
276            self.scope,
277            LifecycleStage::Attach,
278            self.inner.attach_only(request, cancel),
279        )
280        .await
281    }
282    async fn ready(
283        &self,
284        request: &ServeRequest,
285        cancel: &CancellationToken,
286    ) -> Result<(), BackendError> {
287        observe(
288            self.observer,
289            self.scope,
290            LifecycleStage::Readiness,
291            self.inner.ready(request, cancel),
292        )
293        .await
294    }
295    async fn ready_handle(
296        &self,
297        request: &ServeRequest,
298        handle: &ServerState,
299        cancel: &CancellationToken,
300    ) -> Result<(), BackendError> {
301        observe(
302            self.observer,
303            self.scope,
304            LifecycleStage::Readiness,
305            self.inner.ready_handle(request, handle, cancel),
306        )
307        .await
308    }
309    async fn stop(
310        &self,
311        handle: &ServerState,
312        cancel: &CancellationToken,
313    ) -> Result<(), BackendError> {
314        observe(
315            self.observer,
316            self.scope,
317            LifecycleStage::Stop,
318            self.inner.stop(handle, cancel),
319        )
320        .await
321    }
322}
323
324#[cfg(test)]
325mod tests {
326    use super::*;
327    use crate::{
328        adapters::{BackendAdapter, ServeRequest},
329        identity::ProcessIdentity,
330        state::{RuntimeState, ServerState},
331    };
332    use std::sync::Mutex;
333    use tokio_util::sync::CancellationToken;
334
335    #[test]
336    fn diagnostic_sink_bounds_progress_without_evicting_security_warnings() {
337        let diagnostics = DiagnosticObserver::default();
338        diagnostics.warning(LifecycleWarning::InstalledBypass);
339        diagnostics.warning(LifecycleWarning::EstimatedFit);
340        diagnostics.warning(LifecycleWarning::SizeOnly);
341        for completed in 0..100 {
342            download_progress(
343                Some(&diagnostics),
344                completed,
345                100,
346                "\u{1b}]52;c;secret\u{7}/token=secret.bin",
347            );
348        }
349        let snapshot = diagnostics.snapshot();
350        assert_eq!(snapshot.progress.len(), EVENT_CAPACITY);
351        assert_eq!(snapshot.omitted_progress, 36);
352        let text = snapshot.text();
353        assert!(text.contains("no catalog SHA-256"));
354        assert!(text.contains("bypassing estimated fit"));
355        assert!(text.contains("may not fit"));
356        assert!(text.contains("99/100 bytes"));
357        assert!(text.contains("36 earlier progress diagnostics omitted"));
358        assert!(text.contains("[redacted filename]"));
359        assert!(!text.contains("secret"));
360        assert!(!text.contains('\u{1b}'));
361    }
362
363    #[test]
364    fn diagnostic_sink_is_injected_and_keeps_repeated_warning_counts() {
365        let (plain, _) = LifecycleObserver::channel();
366        assert!(plain.diagnostics.is_none());
367        let diagnostics = DiagnosticObserver::default();
368        let observed = plain.with_diagnostics(diagnostics.clone());
369        for _ in 0..100 {
370            warning(Some(&observed), LifecycleWarning::SizeOnly);
371        }
372        assert!(diagnostics.snapshot().text().contains("100 occurrences"));
373        assert!(DiagnosticObserver::default().snapshot().text().is_empty());
374    }
375
376    struct Adapter {
377        fail: Option<LifecycleStage>,
378        calls: Mutex<Vec<LifecycleStage>>,
379    }
380
381    impl Adapter {
382        fn attempt(
383            &self,
384            stage: LifecycleStage,
385            cancel: &CancellationToken,
386        ) -> Result<(), BackendError> {
387            self.calls.lock().unwrap().push(stage);
388            if cancel.is_cancelled() {
389                Err(BackendError("cancelled".into()))
390            } else if self.fail == Some(stage) {
391                Err(BackendError("mock operation failed".into()))
392            } else {
393                Ok(())
394            }
395        }
396    }
397
398    fn handle() -> ServerState {
399        RuntimeState::parse(r#"{"schemaVersion":2,"active":{"backend":"ollama","modelId":"model:latest","endpoint":"http://127.0.0.1:11434","port":11434,"ownedByUs":true,"pid":123}}"#)
400            .unwrap().active.unwrap()
401    }
402
403    fn request() -> ServeRequest {
404        ServeRequest {
405            model_id: "model:latest".into(),
406            endpoint: "http://127.0.0.1:11434".into(),
407            model_path: None,
408            context: Some(4096),
409            cache: None,
410        }
411    }
412
413    #[async_trait::async_trait]
414    impl BackendAdapter for Adapter {
415        fn name(&self) -> &'static str {
416            "ollama"
417        }
418        fn trusts(&self, _identity: &ProcessIdentity) -> bool {
419            false
420        }
421        async fn serve(
422            &self,
423            request: &ServeRequest,
424            cancel: &CancellationToken,
425        ) -> Result<ServerState, BackendError> {
426            assert_eq!(
427                request.context,
428                if request.model_id == "llmup-acquisition" {
429                    None
430                } else {
431                    Some(4096)
432                }
433            );
434            self.attempt(LifecycleStage::Start, cancel)?;
435            Ok(handle())
436        }
437        async fn attach_only(
438            &self,
439            _request: &ServeRequest,
440            cancel: &CancellationToken,
441        ) -> Result<ServerState, BackendError> {
442            self.attempt(LifecycleStage::Attach, cancel)?;
443            let mut handle = handle();
444            handle.owned_by_us = false;
445            Ok(handle)
446        }
447        async fn ready(
448            &self,
449            _request: &ServeRequest,
450            _cancel: &CancellationToken,
451        ) -> Result<(), BackendError> {
452            panic!("ready_handle override must be forwarded")
453        }
454        async fn ready_handle(
455            &self,
456            _request: &ServeRequest,
457            active: &ServerState,
458            cancel: &CancellationToken,
459        ) -> Result<(), BackendError> {
460            assert_eq!(active.pid, Some(123));
461            self.attempt(LifecycleStage::Readiness, cancel)
462        }
463        async fn stop(
464            &self,
465            active: &ServerState,
466            cancel: &CancellationToken,
467        ) -> Result<(), BackendError> {
468            assert!(active.owned_by_us);
469            self.attempt(LifecycleStage::Stop, cancel)
470        }
471    }
472
473    #[tokio::test]
474    async fn runtime_events_adapter_orders_real_calls_and_preserves_failures() {
475        for failure in [
476            None,
477            Some(LifecycleStage::Start),
478            Some(LifecycleStage::Readiness),
479            Some(LifecycleStage::Stop),
480        ] {
481            let (observer, mut receiver) = LifecycleObserver::channel();
482            let adapter = Adapter {
483                fail: failure,
484                calls: Mutex::new(Vec::new()),
485            };
486            let observed = ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::Runtime);
487            let cancel = CancellationToken::new();
488            let outcome = async {
489                let active = observed.serve(&request(), &cancel).await?;
490                observed.ready_handle(&request(), &active, &cancel).await?;
491                observed.stop(&active, &cancel).await
492            }
493            .await;
494            assert_eq!(outcome.is_err(), failure.is_some());
495            if let Err(error) = outcome {
496                assert_eq!(error.0, "mock operation failed");
497            }
498            let mut expected = Vec::new();
499            for stage in [
500                LifecycleStage::Start,
501                LifecycleStage::Readiness,
502                LifecycleStage::Stop,
503            ] {
504                expected.push(stage);
505                assert_eq!(
506                    receiver.try_recv().unwrap(),
507                    LifecycleEvent {
508                        scope: LifecycleScope::Runtime,
509                        stage,
510                        status: LifecycleStatus::Started
511                    }
512                );
513                assert_eq!(
514                    receiver.try_recv().unwrap(),
515                    LifecycleEvent {
516                        scope: LifecycleScope::Runtime,
517                        stage,
518                        status: if failure == Some(stage) {
519                            LifecycleStatus::Failed
520                        } else {
521                            LifecycleStatus::Completed
522                        }
523                    }
524                );
525                if failure == Some(stage) {
526                    break;
527                }
528            }
529            assert_eq!(*adapter.calls.lock().unwrap(), expected);
530            assert!(receiver.try_recv().is_err());
531        }
532    }
533
534    #[tokio::test]
535    async fn runtime_events_adapter_preserves_attach_and_cancellation() {
536        let (observer, mut receiver) = LifecycleObserver::channel();
537        let adapter = Adapter {
538            fail: None,
539            calls: Mutex::new(Vec::new()),
540        };
541        let observed =
542            ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
543        let cancel = CancellationToken::new();
544        assert!(
545            !observed
546                .attach_only(&request(), &cancel)
547                .await
548                .unwrap()
549                .owned_by_us
550        );
551        assert_eq!(receiver.try_recv().unwrap().stage, LifecycleStage::Attach);
552        assert_eq!(
553            receiver.try_recv().unwrap().status,
554            LifecycleStatus::Completed
555        );
556        cancel.cancel();
557        assert_eq!(
558            observed.serve(&request(), &cancel).await.unwrap_err().0,
559            "cancelled"
560        );
561        assert_eq!(
562            receiver.try_recv().unwrap().scope,
563            LifecycleScope::AcquisitionDaemon
564        );
565        assert_eq!(receiver.try_recv().unwrap().status, LifecycleStatus::Failed);
566        assert!(receiver.try_recv().is_err());
567    }
568
569    #[tokio::test]
570    async fn saturated_or_closed_observation_never_retries_or_replaces_runtime_result() {
571        for closed in [false, true] {
572            let (observer, mut receiver) = LifecycleObserver::channel();
573            for _ in 0..EVENT_CAPACITY {
574                observer.emit(
575                    LifecycleScope::Runtime,
576                    LifecycleStage::Start,
577                    LifecycleStatus::Started,
578                );
579            }
580            if closed {
581                receiver.close();
582            }
583            let calls = std::sync::atomic::AtomicUsize::new(0);
584            let result = tokio::time::timeout(
585                std::time::Duration::from_secs(1),
586                observe(
587                    Some(&observer),
588                    LifecycleScope::Runtime,
589                    LifecycleStage::Readiness,
590                    async {
591                        calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
592                        Err::<(), _>(BackendError("authoritative runtime failure".into()))
593                    },
594                ),
595            )
596            .await
597            .unwrap();
598            assert_eq!(result.unwrap_err().0, "authoritative runtime failure");
599            assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
600            assert_eq!(receiver.len(), EVENT_CAPACITY);
601            let result = observe(
602                Some(&observer),
603                LifecycleScope::Runtime,
604                LifecycleStage::Stop,
605                async { Ok::<_, BackendError>("authoritative success") },
606            )
607            .await
608            .unwrap();
609            assert_eq!(result, "authoritative success");
610            assert_eq!(receiver.len(), EVENT_CAPACITY);
611        }
612    }
613
614    #[tokio::test]
615    async fn runtime_events_temporary_daemon_cleanup_does_not_claim_pull_success() {
616        let (observer, mut receiver) = LifecycleObserver::channel();
617        let adapter = Adapter {
618            fail: None,
619            calls: Mutex::new(Vec::new()),
620        };
621        let observed =
622            ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
623        let cancel = CancellationToken::new();
624        let operation = observe(
625            Some(&observer),
626            LifecycleScope::Runtime,
627            LifecycleStage::Acquisition,
628            async {
629                cancel.cancel();
630                Err::<(), _>(BackendError("pull cancelled".into()))
631            },
632        );
633        let result = crate::pull::with_ollama_daemon(
634            &observed,
635            "http://127.0.0.1:11434",
636            &cancel,
637            operation,
638        )
639        .await;
640        assert_eq!(result.unwrap_err().0, "pull cancelled");
641        assert_eq!(
642            *adapter.calls.lock().unwrap(),
643            [LifecycleStage::Start, LifecycleStage::Stop]
644        );
645        for (scope, stage, status) in [
646            (
647                LifecycleScope::AcquisitionDaemon,
648                LifecycleStage::Start,
649                LifecycleStatus::Started,
650            ),
651            (
652                LifecycleScope::AcquisitionDaemon,
653                LifecycleStage::Start,
654                LifecycleStatus::Completed,
655            ),
656            (
657                LifecycleScope::Runtime,
658                LifecycleStage::Acquisition,
659                LifecycleStatus::Started,
660            ),
661            (
662                LifecycleScope::Runtime,
663                LifecycleStage::Acquisition,
664                LifecycleStatus::Failed,
665            ),
666            (
667                LifecycleScope::AcquisitionDaemon,
668                LifecycleStage::Stop,
669                LifecycleStatus::Started,
670            ),
671            (
672                LifecycleScope::AcquisitionDaemon,
673                LifecycleStage::Stop,
674                LifecycleStatus::Completed,
675            ),
676        ] {
677            assert_eq!(
678                receiver.try_recv().unwrap(),
679                LifecycleEvent {
680                    scope,
681                    stage,
682                    status
683                }
684            );
685        }
686        assert!(receiver.try_recv().is_err());
687    }
688
689    #[tokio::test]
690    async fn runtime_events_bracket_success_and_failure_without_false_completion() {
691        let (observer, mut receiver) = LifecycleObserver::channel();
692        let operation = async {
693            assert_eq!(
694                receiver.try_recv().unwrap(),
695                LifecycleEvent {
696                    scope: LifecycleScope::Runtime,
697                    stage: LifecycleStage::Verification,
698                    status: LifecycleStatus::Started,
699                }
700            );
701            Ok::<_, BackendError>(42)
702        };
703        assert_eq!(
704            observe(
705                Some(&observer),
706                LifecycleScope::Runtime,
707                LifecycleStage::Verification,
708                operation
709            )
710            .await
711            .unwrap(),
712            42
713        );
714        assert_eq!(
715            receiver.try_recv().unwrap().status,
716            LifecycleStatus::Completed
717        );
718        let error = observe(
719            Some(&observer),
720            LifecycleScope::Runtime,
721            LifecycleStage::Start,
722            async { Err::<(), _>(BackendError("private diagnostic".into())) },
723        )
724        .await
725        .unwrap_err();
726        assert_eq!(error.0, "private diagnostic");
727        assert_eq!(
728            receiver.try_recv().unwrap().status,
729            LifecycleStatus::Started
730        );
731        let failed = receiver.try_recv().unwrap();
732        assert_eq!(failed.status, LifecycleStatus::Failed);
733        assert!(!format!("{failed:?}").contains("private diagnostic"));
734        assert!(receiver.try_recv().is_err());
735    }
736
737    #[tokio::test]
738    async fn runtime_events_full_closed_or_absent_observer_never_blocks_operations() {
739        let (observer, mut receiver) = LifecycleObserver::channel();
740        for _ in 0..EVENT_CAPACITY * 2 {
741            observe(
742                Some(&observer),
743                LifecycleScope::Runtime,
744                LifecycleStage::Readiness,
745                async { Ok::<_, BackendError>(()) },
746            )
747            .await
748            .unwrap();
749        }
750        assert_eq!(receiver.len(), EVENT_CAPACITY);
751        receiver.close();
752        for observer in [Some(&observer), None] {
753            assert_eq!(
754                observe(
755                    observer,
756                    LifecycleScope::Runtime,
757                    LifecycleStage::Stop,
758                    async { Ok::<_, BackendError>(7) }
759                )
760                .await
761                .unwrap(),
762                7
763            );
764        }
765    }
766
767    #[tokio::test]
768    async fn runtime_events_dropped_operation_has_no_completion() {
769        let (observer, mut receiver) = LifecycleObserver::channel();
770        {
771            let operation = observe(
772                Some(&observer),
773                LifecycleScope::Runtime,
774                LifecycleStage::Readiness,
775                std::future::pending::<Result<(), BackendError>>(),
776            );
777            tokio::pin!(operation);
778            tokio::select! {
779                biased;
780                _ = &mut operation => panic!("pending operation completed"),
781                event = receiver.recv() => assert_eq!(event.unwrap().status, LifecycleStatus::Started),
782            }
783        }
784        assert!(receiver.try_recv().is_err());
785    }
786}