1use crate::{
2 adapters::{BackendAdapter, BackendError, ServeRequest},
3 identity::ProcessIdentity,
4 state::ServerState,
5};
6use std::{
7 collections::VecDeque,
8 future::Future,
9 sync::{Arc, Mutex},
10};
11use tokio::sync::mpsc::{self, Receiver, Sender};
12use tokio_util::sync::CancellationToken;
13
14pub const EVENT_CAPACITY: usize = 64;
15
16#[derive(Clone, Copy)]
17pub(crate) enum LifecycleWarning {
18 InstalledBypass,
19 EstimatedFit,
20 SizeOnly,
21}
22
23impl LifecycleWarning {
24 fn message(self) -> &'static str {
25 match self {
26 Self::InstalledBypass => {
27 "up: bypassing estimated fit; local content integrity is not catalog verification; throughput may be unknown"
28 }
29 Self::EstimatedFit => {
30 "up: requested quantization may not fit this hardware; continuing because it was explicitly requested"
31 }
32 Self::SizeOnly => {
33 "up: weights passed a size-floor check; no catalog SHA-256 was available"
34 }
35 }
36 }
37}
38
39#[derive(Clone, Default)]
40pub struct DiagnosticObserver {
41 state: Arc<Mutex<DiagnosticSnapshot>>,
42}
43
44#[derive(Clone, Default)]
45pub struct DiagnosticSnapshot {
46 warnings: [u64; 3],
47 progress: VecDeque<(u64, u64)>,
48 omitted_progress: u64,
49}
50
51impl DiagnosticObserver {
52 pub fn snapshot(&self) -> DiagnosticSnapshot {
53 self.state
54 .lock()
55 .unwrap_or_else(|error| error.into_inner())
56 .clone()
57 }
58
59 fn warning(&self, warning: LifecycleWarning) {
60 let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
61 let count = &mut state.warnings[warning as usize];
62 *count = count.saturating_add(1);
63 }
64
65 fn progress(&self, completed: u64, total: u64) {
66 let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
67 if state.progress.len() == EVENT_CAPACITY {
68 state.progress.pop_front();
69 state.omitted_progress = state.omitted_progress.saturating_add(1);
70 }
71 state.progress.push_back((completed, total));
72 }
73}
74
75impl DiagnosticSnapshot {
76 pub fn text(&self) -> String {
77 let mut lines = Vec::new();
78 for warning in [
79 LifecycleWarning::InstalledBypass,
80 LifecycleWarning::EstimatedFit,
81 LifecycleWarning::SizeOnly,
82 ] {
83 let count = self.warnings[warning as usize];
84 if count > 0 {
85 lines.push(format!("{} ({count} occurrences)", warning.message()));
86 }
87 }
88 if self.omitted_progress > 0 {
89 lines.push(format!(
90 "{} earlier progress diagnostics omitted",
91 self.omitted_progress
92 ));
93 }
94 for (completed, total) in &self.progress {
95 lines.push(format!(" [redacted filename]: {completed}/{total} bytes"));
96 }
97 if lines.is_empty() {
98 String::new()
99 } else {
100 format!("{}\n", lines.join("\n"))
101 }
102 }
103}
104
105pub(crate) fn warning(observer: Option<&LifecycleObserver>, warning: LifecycleWarning) {
106 match observer.and_then(|observer| observer.diagnostics.as_ref()) {
107 Some(diagnostics) => diagnostics.warning(warning),
108 None => eprintln!("{}", warning.message()),
109 }
110}
111
112pub(crate) fn download_progress(
113 diagnostics: Option<&DiagnosticObserver>,
114 completed: u64,
115 total: u64,
116 file: &str,
117) {
118 match diagnostics {
119 Some(diagnostics) => diagnostics.progress(completed, total),
120 None => eprintln!(
121 " {}: {completed}/{total} bytes",
122 rigspark_core::reports::strip_control(file)
123 ),
124 }
125}
126
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub enum LifecycleScope {
129 Runtime,
130 AcquisitionDaemon,
131}
132
133#[derive(Debug, Clone, Copy, PartialEq, Eq)]
134pub enum LifecycleStage {
135 Acquisition,
136 Verification,
137 AcquisitionVerification,
138 Start,
139 Attach,
140 Activation,
141 Readiness,
142 Stop,
143}
144
145#[derive(Debug, Clone, Copy, PartialEq, Eq)]
146pub enum LifecycleStatus {
147 Started,
148 Completed,
149 Failed,
150}
151
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub struct LifecycleEvent {
159 pub scope: LifecycleScope,
160 pub stage: LifecycleStage,
161 pub status: LifecycleStatus,
162}
163
164pub struct LifecycleObserver {
167 sender: Sender<LifecycleEvent>,
168 pub(crate) diagnostics: Option<DiagnosticObserver>,
169}
170
171impl LifecycleObserver {
172 pub fn channel() -> (Self, Receiver<LifecycleEvent>) {
173 let (sender, receiver) = mpsc::channel(EVENT_CAPACITY);
174 (
175 Self {
176 sender,
177 diagnostics: None,
178 },
179 receiver,
180 )
181 }
182
183 pub fn with_diagnostics(mut self, diagnostics: DiagnosticObserver) -> Self {
184 self.diagnostics = Some(diagnostics);
185 self
186 }
187
188 fn emit(&self, scope: LifecycleScope, stage: LifecycleStage, status: LifecycleStatus) {
189 let _ = self.sender.try_send(LifecycleEvent {
190 scope,
191 stage,
192 status,
193 });
194 }
195}
196
197pub(crate) async fn observe<T>(
198 observer: Option<&LifecycleObserver>,
199 scope: LifecycleScope,
200 stage: LifecycleStage,
201 operation: impl Future<Output = Result<T, BackendError>>,
202) -> Result<T, BackendError> {
203 if let Some(observer) = observer {
204 observer.emit(scope, stage, LifecycleStatus::Started);
205 }
206 let result = operation.await;
207 if let Some(observer) = observer {
208 observer.emit(
209 scope,
210 stage,
211 if result.is_ok() {
212 LifecycleStatus::Completed
213 } else {
214 LifecycleStatus::Failed
215 },
216 );
217 }
218 result
219}
220
221pub(super) struct ObservedAdapter<'runtime> {
222 inner: &'runtime dyn BackendAdapter,
223 observer: Option<&'runtime LifecycleObserver>,
224 scope: LifecycleScope,
225}
226
227impl<'runtime> ObservedAdapter<'runtime> {
228 pub(super) fn new(
229 inner: &'runtime dyn BackendAdapter,
230 observer: Option<&'runtime LifecycleObserver>,
231 scope: LifecycleScope,
232 ) -> Self {
233 Self {
234 inner,
235 observer,
236 scope,
237 }
238 }
239}
240
241#[async_trait::async_trait]
242impl BackendAdapter for ObservedAdapter<'_> {
243 fn name(&self) -> &'static str {
244 self.inner.name()
245 }
246 fn trusts(&self, identity: &ProcessIdentity) -> bool {
247 self.inner.trusts(identity)
248 }
249 fn can_embed(&self) -> bool {
250 self.inner.can_embed()
251 }
252 fn can_stream(&self) -> bool {
253 self.inner.can_stream()
254 }
255
256 async fn serve(
257 &self,
258 request: &ServeRequest,
259 cancel: &CancellationToken,
260 ) -> Result<ServerState, BackendError> {
261 observe(
262 self.observer,
263 self.scope,
264 LifecycleStage::Start,
265 self.inner.serve(request, cancel),
266 )
267 .await
268 }
269 async fn attach_only(
270 &self,
271 request: &ServeRequest,
272 cancel: &CancellationToken,
273 ) -> Result<ServerState, BackendError> {
274 observe(
275 self.observer,
276 self.scope,
277 LifecycleStage::Attach,
278 self.inner.attach_only(request, cancel),
279 )
280 .await
281 }
282 async fn ready(
283 &self,
284 request: &ServeRequest,
285 cancel: &CancellationToken,
286 ) -> Result<(), BackendError> {
287 observe(
288 self.observer,
289 self.scope,
290 LifecycleStage::Readiness,
291 self.inner.ready(request, cancel),
292 )
293 .await
294 }
295 async fn ready_handle(
296 &self,
297 request: &ServeRequest,
298 handle: &ServerState,
299 cancel: &CancellationToken,
300 ) -> Result<(), BackendError> {
301 observe(
302 self.observer,
303 self.scope,
304 LifecycleStage::Readiness,
305 self.inner.ready_handle(request, handle, cancel),
306 )
307 .await
308 }
309 async fn stop(
310 &self,
311 handle: &ServerState,
312 cancel: &CancellationToken,
313 ) -> Result<(), BackendError> {
314 observe(
315 self.observer,
316 self.scope,
317 LifecycleStage::Stop,
318 self.inner.stop(handle, cancel),
319 )
320 .await
321 }
322}
323
324#[cfg(test)]
325mod tests {
326 use super::*;
327 use crate::{
328 adapters::{BackendAdapter, ServeRequest},
329 identity::ProcessIdentity,
330 state::{RuntimeState, ServerState},
331 };
332 use std::sync::Mutex;
333 use tokio_util::sync::CancellationToken;
334
335 #[test]
336 fn diagnostic_sink_bounds_progress_without_evicting_security_warnings() {
337 let diagnostics = DiagnosticObserver::default();
338 diagnostics.warning(LifecycleWarning::InstalledBypass);
339 diagnostics.warning(LifecycleWarning::EstimatedFit);
340 diagnostics.warning(LifecycleWarning::SizeOnly);
341 for completed in 0..100 {
342 download_progress(
343 Some(&diagnostics),
344 completed,
345 100,
346 "\u{1b}]52;c;secret\u{7}/token=secret.bin",
347 );
348 }
349 let snapshot = diagnostics.snapshot();
350 assert_eq!(snapshot.progress.len(), EVENT_CAPACITY);
351 assert_eq!(snapshot.omitted_progress, 36);
352 let text = snapshot.text();
353 assert!(text.contains("no catalog SHA-256"));
354 assert!(text.contains("bypassing estimated fit"));
355 assert!(text.contains("may not fit"));
356 assert!(text.contains("99/100 bytes"));
357 assert!(text.contains("36 earlier progress diagnostics omitted"));
358 assert!(text.contains("[redacted filename]"));
359 assert!(!text.contains("secret"));
360 assert!(!text.contains('\u{1b}'));
361 }
362
363 #[test]
364 fn diagnostic_sink_is_injected_and_keeps_repeated_warning_counts() {
365 let (plain, _) = LifecycleObserver::channel();
366 assert!(plain.diagnostics.is_none());
367 let diagnostics = DiagnosticObserver::default();
368 let observed = plain.with_diagnostics(diagnostics.clone());
369 for _ in 0..100 {
370 warning(Some(&observed), LifecycleWarning::SizeOnly);
371 }
372 assert!(diagnostics.snapshot().text().contains("100 occurrences"));
373 assert!(DiagnosticObserver::default().snapshot().text().is_empty());
374 }
375
376 struct Adapter {
377 fail: Option<LifecycleStage>,
378 calls: Mutex<Vec<LifecycleStage>>,
379 }
380
381 impl Adapter {
382 fn attempt(
383 &self,
384 stage: LifecycleStage,
385 cancel: &CancellationToken,
386 ) -> Result<(), BackendError> {
387 self.calls.lock().unwrap().push(stage);
388 if cancel.is_cancelled() {
389 Err(BackendError("cancelled".into()))
390 } else if self.fail == Some(stage) {
391 Err(BackendError("mock operation failed".into()))
392 } else {
393 Ok(())
394 }
395 }
396 }
397
398 fn handle() -> ServerState {
399 RuntimeState::parse(r#"{"schemaVersion":2,"active":{"backend":"ollama","modelId":"model:latest","endpoint":"http://127.0.0.1:11434","port":11434,"ownedByUs":true,"pid":123}}"#)
400 .unwrap().active.unwrap()
401 }
402
403 fn request() -> ServeRequest {
404 ServeRequest {
405 model_id: "model:latest".into(),
406 endpoint: "http://127.0.0.1:11434".into(),
407 model_path: None,
408 context: Some(4096),
409 cache: None,
410 }
411 }
412
413 #[async_trait::async_trait]
414 impl BackendAdapter for Adapter {
415 fn name(&self) -> &'static str {
416 "ollama"
417 }
418 fn trusts(&self, _identity: &ProcessIdentity) -> bool {
419 false
420 }
421 async fn serve(
422 &self,
423 request: &ServeRequest,
424 cancel: &CancellationToken,
425 ) -> Result<ServerState, BackendError> {
426 assert_eq!(
427 request.context,
428 if request.model_id == "llmup-acquisition" {
429 None
430 } else {
431 Some(4096)
432 }
433 );
434 self.attempt(LifecycleStage::Start, cancel)?;
435 Ok(handle())
436 }
437 async fn attach_only(
438 &self,
439 _request: &ServeRequest,
440 cancel: &CancellationToken,
441 ) -> Result<ServerState, BackendError> {
442 self.attempt(LifecycleStage::Attach, cancel)?;
443 let mut handle = handle();
444 handle.owned_by_us = false;
445 Ok(handle)
446 }
447 async fn ready(
448 &self,
449 _request: &ServeRequest,
450 _cancel: &CancellationToken,
451 ) -> Result<(), BackendError> {
452 panic!("ready_handle override must be forwarded")
453 }
454 async fn ready_handle(
455 &self,
456 _request: &ServeRequest,
457 active: &ServerState,
458 cancel: &CancellationToken,
459 ) -> Result<(), BackendError> {
460 assert_eq!(active.pid, Some(123));
461 self.attempt(LifecycleStage::Readiness, cancel)
462 }
463 async fn stop(
464 &self,
465 active: &ServerState,
466 cancel: &CancellationToken,
467 ) -> Result<(), BackendError> {
468 assert!(active.owned_by_us);
469 self.attempt(LifecycleStage::Stop, cancel)
470 }
471 }
472
473 #[tokio::test]
474 async fn runtime_events_adapter_orders_real_calls_and_preserves_failures() {
475 for failure in [
476 None,
477 Some(LifecycleStage::Start),
478 Some(LifecycleStage::Readiness),
479 Some(LifecycleStage::Stop),
480 ] {
481 let (observer, mut receiver) = LifecycleObserver::channel();
482 let adapter = Adapter {
483 fail: failure,
484 calls: Mutex::new(Vec::new()),
485 };
486 let observed = ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::Runtime);
487 let cancel = CancellationToken::new();
488 let outcome = async {
489 let active = observed.serve(&request(), &cancel).await?;
490 observed.ready_handle(&request(), &active, &cancel).await?;
491 observed.stop(&active, &cancel).await
492 }
493 .await;
494 assert_eq!(outcome.is_err(), failure.is_some());
495 if let Err(error) = outcome {
496 assert_eq!(error.0, "mock operation failed");
497 }
498 let mut expected = Vec::new();
499 for stage in [
500 LifecycleStage::Start,
501 LifecycleStage::Readiness,
502 LifecycleStage::Stop,
503 ] {
504 expected.push(stage);
505 assert_eq!(
506 receiver.try_recv().unwrap(),
507 LifecycleEvent {
508 scope: LifecycleScope::Runtime,
509 stage,
510 status: LifecycleStatus::Started
511 }
512 );
513 assert_eq!(
514 receiver.try_recv().unwrap(),
515 LifecycleEvent {
516 scope: LifecycleScope::Runtime,
517 stage,
518 status: if failure == Some(stage) {
519 LifecycleStatus::Failed
520 } else {
521 LifecycleStatus::Completed
522 }
523 }
524 );
525 if failure == Some(stage) {
526 break;
527 }
528 }
529 assert_eq!(*adapter.calls.lock().unwrap(), expected);
530 assert!(receiver.try_recv().is_err());
531 }
532 }
533
534 #[tokio::test]
535 async fn runtime_events_adapter_preserves_attach_and_cancellation() {
536 let (observer, mut receiver) = LifecycleObserver::channel();
537 let adapter = Adapter {
538 fail: None,
539 calls: Mutex::new(Vec::new()),
540 };
541 let observed =
542 ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
543 let cancel = CancellationToken::new();
544 assert!(
545 !observed
546 .attach_only(&request(), &cancel)
547 .await
548 .unwrap()
549 .owned_by_us
550 );
551 assert_eq!(receiver.try_recv().unwrap().stage, LifecycleStage::Attach);
552 assert_eq!(
553 receiver.try_recv().unwrap().status,
554 LifecycleStatus::Completed
555 );
556 cancel.cancel();
557 assert_eq!(
558 observed.serve(&request(), &cancel).await.unwrap_err().0,
559 "cancelled"
560 );
561 assert_eq!(
562 receiver.try_recv().unwrap().scope,
563 LifecycleScope::AcquisitionDaemon
564 );
565 assert_eq!(receiver.try_recv().unwrap().status, LifecycleStatus::Failed);
566 assert!(receiver.try_recv().is_err());
567 }
568
569 #[tokio::test]
570 async fn saturated_or_closed_observation_never_retries_or_replaces_runtime_result() {
571 for closed in [false, true] {
572 let (observer, mut receiver) = LifecycleObserver::channel();
573 for _ in 0..EVENT_CAPACITY {
574 observer.emit(
575 LifecycleScope::Runtime,
576 LifecycleStage::Start,
577 LifecycleStatus::Started,
578 );
579 }
580 if closed {
581 receiver.close();
582 }
583 let calls = std::sync::atomic::AtomicUsize::new(0);
584 let result = tokio::time::timeout(
585 std::time::Duration::from_secs(1),
586 observe(
587 Some(&observer),
588 LifecycleScope::Runtime,
589 LifecycleStage::Readiness,
590 async {
591 calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
592 Err::<(), _>(BackendError("authoritative runtime failure".into()))
593 },
594 ),
595 )
596 .await
597 .unwrap();
598 assert_eq!(result.unwrap_err().0, "authoritative runtime failure");
599 assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
600 assert_eq!(receiver.len(), EVENT_CAPACITY);
601 let result = observe(
602 Some(&observer),
603 LifecycleScope::Runtime,
604 LifecycleStage::Stop,
605 async { Ok::<_, BackendError>("authoritative success") },
606 )
607 .await
608 .unwrap();
609 assert_eq!(result, "authoritative success");
610 assert_eq!(receiver.len(), EVENT_CAPACITY);
611 }
612 }
613
614 #[tokio::test]
615 async fn runtime_events_temporary_daemon_cleanup_does_not_claim_pull_success() {
616 let (observer, mut receiver) = LifecycleObserver::channel();
617 let adapter = Adapter {
618 fail: None,
619 calls: Mutex::new(Vec::new()),
620 };
621 let observed =
622 ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
623 let cancel = CancellationToken::new();
624 let operation = observe(
625 Some(&observer),
626 LifecycleScope::Runtime,
627 LifecycleStage::Acquisition,
628 async {
629 cancel.cancel();
630 Err::<(), _>(BackendError("pull cancelled".into()))
631 },
632 );
633 let result = crate::pull::with_ollama_daemon(
634 &observed,
635 "http://127.0.0.1:11434",
636 &cancel,
637 operation,
638 )
639 .await;
640 assert_eq!(result.unwrap_err().0, "pull cancelled");
641 assert_eq!(
642 *adapter.calls.lock().unwrap(),
643 [LifecycleStage::Start, LifecycleStage::Stop]
644 );
645 for (scope, stage, status) in [
646 (
647 LifecycleScope::AcquisitionDaemon,
648 LifecycleStage::Start,
649 LifecycleStatus::Started,
650 ),
651 (
652 LifecycleScope::AcquisitionDaemon,
653 LifecycleStage::Start,
654 LifecycleStatus::Completed,
655 ),
656 (
657 LifecycleScope::Runtime,
658 LifecycleStage::Acquisition,
659 LifecycleStatus::Started,
660 ),
661 (
662 LifecycleScope::Runtime,
663 LifecycleStage::Acquisition,
664 LifecycleStatus::Failed,
665 ),
666 (
667 LifecycleScope::AcquisitionDaemon,
668 LifecycleStage::Stop,
669 LifecycleStatus::Started,
670 ),
671 (
672 LifecycleScope::AcquisitionDaemon,
673 LifecycleStage::Stop,
674 LifecycleStatus::Completed,
675 ),
676 ] {
677 assert_eq!(
678 receiver.try_recv().unwrap(),
679 LifecycleEvent {
680 scope,
681 stage,
682 status
683 }
684 );
685 }
686 assert!(receiver.try_recv().is_err());
687 }
688
689 #[tokio::test]
690 async fn runtime_events_bracket_success_and_failure_without_false_completion() {
691 let (observer, mut receiver) = LifecycleObserver::channel();
692 let operation = async {
693 assert_eq!(
694 receiver.try_recv().unwrap(),
695 LifecycleEvent {
696 scope: LifecycleScope::Runtime,
697 stage: LifecycleStage::Verification,
698 status: LifecycleStatus::Started,
699 }
700 );
701 Ok::<_, BackendError>(42)
702 };
703 assert_eq!(
704 observe(
705 Some(&observer),
706 LifecycleScope::Runtime,
707 LifecycleStage::Verification,
708 operation
709 )
710 .await
711 .unwrap(),
712 42
713 );
714 assert_eq!(
715 receiver.try_recv().unwrap().status,
716 LifecycleStatus::Completed
717 );
718 let error = observe(
719 Some(&observer),
720 LifecycleScope::Runtime,
721 LifecycleStage::Start,
722 async { Err::<(), _>(BackendError("private diagnostic".into())) },
723 )
724 .await
725 .unwrap_err();
726 assert_eq!(error.0, "private diagnostic");
727 assert_eq!(
728 receiver.try_recv().unwrap().status,
729 LifecycleStatus::Started
730 );
731 let failed = receiver.try_recv().unwrap();
732 assert_eq!(failed.status, LifecycleStatus::Failed);
733 assert!(!format!("{failed:?}").contains("private diagnostic"));
734 assert!(receiver.try_recv().is_err());
735 }
736
737 #[tokio::test]
738 async fn runtime_events_full_closed_or_absent_observer_never_blocks_operations() {
739 let (observer, mut receiver) = LifecycleObserver::channel();
740 for _ in 0..EVENT_CAPACITY * 2 {
741 observe(
742 Some(&observer),
743 LifecycleScope::Runtime,
744 LifecycleStage::Readiness,
745 async { Ok::<_, BackendError>(()) },
746 )
747 .await
748 .unwrap();
749 }
750 assert_eq!(receiver.len(), EVENT_CAPACITY);
751 receiver.close();
752 for observer in [Some(&observer), None] {
753 assert_eq!(
754 observe(
755 observer,
756 LifecycleScope::Runtime,
757 LifecycleStage::Stop,
758 async { Ok::<_, BackendError>(7) }
759 )
760 .await
761 .unwrap(),
762 7
763 );
764 }
765 }
766
767 #[tokio::test]
768 async fn runtime_events_dropped_operation_has_no_completion() {
769 let (observer, mut receiver) = LifecycleObserver::channel();
770 {
771 let operation = observe(
772 Some(&observer),
773 LifecycleScope::Runtime,
774 LifecycleStage::Readiness,
775 std::future::pending::<Result<(), BackendError>>(),
776 );
777 tokio::pin!(operation);
778 tokio::select! {
779 biased;
780 _ = &mut operation => panic!("pending operation completed"),
781 event = receiver.recv() => assert_eq!(event.unwrap().status, LifecycleStatus::Started),
782 }
783 }
784 assert!(receiver.try_recv().is_err());
785 }
786}