1use crate::{
2 adapters::{BackendAdapter, BackendError, ServeRequest},
3 identity::ProcessIdentity,
4 state::ServerState,
5};
6use std::{
7 collections::VecDeque,
8 future::Future,
9 sync::{Arc, Mutex},
10};
11use tokio::sync::mpsc::{self, Receiver, Sender};
12use tokio_util::sync::CancellationToken;
13
14pub const EVENT_CAPACITY: usize = 64;
15
16#[derive(Clone, Copy)]
17pub(crate) enum LifecycleWarning {
18 InstalledBypass,
19 EstimatedFit,
20 SizeOnly,
21 DrafterSkipped,
22}
23
24impl LifecycleWarning {
25 fn message(self) -> &'static str {
26 match self {
27 Self::InstalledBypass => {
28 "up: bypassing estimated fit; local content integrity is not catalog verification; throughput may be unknown"
29 }
30 Self::EstimatedFit => {
31 "up: requested quantization may not fit this hardware; continuing because it was explicitly requested"
32 }
33 Self::SizeOnly => {
34 "up: weights passed a size-floor check; no catalog SHA-256 was available"
35 }
36 Self::DrafterSkipped => {
37 "up: skipped the speculative drafter because it does not fit beside the model"
38 }
39 }
40 }
41}
42
43#[derive(Clone, Default)]
44pub struct DiagnosticObserver {
45 state: Arc<Mutex<DiagnosticSnapshot>>,
46}
47
48#[derive(Clone, Default)]
49pub struct DiagnosticSnapshot {
50 warnings: [u64; 3],
51 progress: VecDeque<(u64, u64)>,
52 omitted_progress: u64,
53}
54
55impl DiagnosticObserver {
56 pub fn snapshot(&self) -> DiagnosticSnapshot {
57 self.state
58 .lock()
59 .unwrap_or_else(|error| error.into_inner())
60 .clone()
61 }
62
63 fn warning(&self, warning: LifecycleWarning) {
64 let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
65 let count = &mut state.warnings[warning as usize];
66 *count = count.saturating_add(1);
67 }
68
69 fn progress(&self, completed: u64, total: u64) {
70 let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
71 if state.progress.len() == EVENT_CAPACITY {
72 state.progress.pop_front();
73 state.omitted_progress = state.omitted_progress.saturating_add(1);
74 }
75 state.progress.push_back((completed, total));
76 }
77}
78
79impl DiagnosticSnapshot {
80 pub fn text(&self) -> String {
81 let mut lines = Vec::new();
82 for warning in [
83 LifecycleWarning::InstalledBypass,
84 LifecycleWarning::EstimatedFit,
85 LifecycleWarning::SizeOnly,
86 ] {
87 let count = self.warnings[warning as usize];
88 if count > 0 {
89 lines.push(format!("{} ({count} occurrences)", warning.message()));
90 }
91 }
92 if self.omitted_progress > 0 {
93 lines.push(format!(
94 "{} earlier progress diagnostics omitted",
95 self.omitted_progress
96 ));
97 }
98 for (completed, total) in &self.progress {
99 lines.push(format!(" [redacted filename]: {completed}/{total} bytes"));
100 }
101 if lines.is_empty() {
102 String::new()
103 } else {
104 format!("{}\n", lines.join("\n"))
105 }
106 }
107}
108
109pub(crate) fn warning(observer: Option<&LifecycleObserver>, warning: LifecycleWarning) {
110 match observer.and_then(|observer| observer.diagnostics.as_ref()) {
111 Some(diagnostics) => diagnostics.warning(warning),
112 None => eprintln!("{}", warning.message()),
113 }
114}
115
116pub(crate) fn download_progress(
117 diagnostics: Option<&DiagnosticObserver>,
118 completed: u64,
119 total: u64,
120 file: &str,
121) {
122 match diagnostics {
123 Some(diagnostics) => diagnostics.progress(completed, total),
124 None => eprintln!(
125 " {}: {completed}/{total} bytes",
126 rigspark_core::reports::strip_control(file)
127 ),
128 }
129}
130
131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
132pub enum LifecycleScope {
133 Runtime,
134 AcquisitionDaemon,
135}
136
137#[derive(Debug, Clone, Copy, PartialEq, Eq)]
138pub enum LifecycleStage {
139 Acquisition,
140 Verification,
141 AcquisitionVerification,
142 Start,
143 Attach,
144 Activation,
145 Readiness,
146 Stop,
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
150pub enum LifecycleStatus {
151 Started,
152 Completed,
153 Failed,
154}
155
156#[derive(Debug, Clone, Copy, PartialEq, Eq)]
162pub struct LifecycleEvent {
163 pub scope: LifecycleScope,
164 pub stage: LifecycleStage,
165 pub status: LifecycleStatus,
166}
167
168pub struct LifecycleObserver {
171 sender: Sender<LifecycleEvent>,
172 pub(crate) diagnostics: Option<DiagnosticObserver>,
173}
174
175impl LifecycleObserver {
176 pub fn channel() -> (Self, Receiver<LifecycleEvent>) {
177 let (sender, receiver) = mpsc::channel(EVENT_CAPACITY);
178 (
179 Self {
180 sender,
181 diagnostics: None,
182 },
183 receiver,
184 )
185 }
186
187 pub fn with_diagnostics(mut self, diagnostics: DiagnosticObserver) -> Self {
188 self.diagnostics = Some(diagnostics);
189 self
190 }
191
192 fn emit(&self, scope: LifecycleScope, stage: LifecycleStage, status: LifecycleStatus) {
193 let _ = self.sender.try_send(LifecycleEvent {
194 scope,
195 stage,
196 status,
197 });
198 }
199}
200
201pub(crate) async fn observe<T>(
202 observer: Option<&LifecycleObserver>,
203 scope: LifecycleScope,
204 stage: LifecycleStage,
205 operation: impl Future<Output = Result<T, BackendError>>,
206) -> Result<T, BackendError> {
207 if let Some(observer) = observer {
208 observer.emit(scope, stage, LifecycleStatus::Started);
209 }
210 let result = operation.await;
211 if let Some(observer) = observer {
212 observer.emit(
213 scope,
214 stage,
215 if result.is_ok() {
216 LifecycleStatus::Completed
217 } else {
218 LifecycleStatus::Failed
219 },
220 );
221 }
222 result
223}
224
225pub(super) struct ObservedAdapter<'runtime> {
226 inner: &'runtime dyn BackendAdapter,
227 observer: Option<&'runtime LifecycleObserver>,
228 scope: LifecycleScope,
229}
230
231impl<'runtime> ObservedAdapter<'runtime> {
232 pub(super) fn new(
233 inner: &'runtime dyn BackendAdapter,
234 observer: Option<&'runtime LifecycleObserver>,
235 scope: LifecycleScope,
236 ) -> Self {
237 Self {
238 inner,
239 observer,
240 scope,
241 }
242 }
243}
244
245#[async_trait::async_trait]
246impl BackendAdapter for ObservedAdapter<'_> {
247 fn name(&self) -> &'static str {
248 self.inner.name()
249 }
250 fn trusts(&self, identity: &ProcessIdentity) -> bool {
251 self.inner.trusts(identity)
252 }
253 fn can_embed(&self) -> bool {
254 self.inner.can_embed()
255 }
256 fn can_stream(&self) -> bool {
257 self.inner.can_stream()
258 }
259
260 async fn serve(
261 &self,
262 request: &ServeRequest,
263 cancel: &CancellationToken,
264 ) -> Result<ServerState, BackendError> {
265 observe(
266 self.observer,
267 self.scope,
268 LifecycleStage::Start,
269 self.inner.serve(request, cancel),
270 )
271 .await
272 }
273 async fn attach_only(
274 &self,
275 request: &ServeRequest,
276 cancel: &CancellationToken,
277 ) -> Result<ServerState, BackendError> {
278 observe(
279 self.observer,
280 self.scope,
281 LifecycleStage::Attach,
282 self.inner.attach_only(request, cancel),
283 )
284 .await
285 }
286 async fn ready(
287 &self,
288 request: &ServeRequest,
289 cancel: &CancellationToken,
290 ) -> Result<(), BackendError> {
291 observe(
292 self.observer,
293 self.scope,
294 LifecycleStage::Readiness,
295 self.inner.ready(request, cancel),
296 )
297 .await
298 }
299 async fn ready_handle(
300 &self,
301 request: &ServeRequest,
302 handle: &ServerState,
303 cancel: &CancellationToken,
304 ) -> Result<(), BackendError> {
305 observe(
306 self.observer,
307 self.scope,
308 LifecycleStage::Readiness,
309 self.inner.ready_handle(request, handle, cancel),
310 )
311 .await
312 }
313 async fn stop(
314 &self,
315 handle: &ServerState,
316 cancel: &CancellationToken,
317 ) -> Result<(), BackendError> {
318 observe(
319 self.observer,
320 self.scope,
321 LifecycleStage::Stop,
322 self.inner.stop(handle, cancel),
323 )
324 .await
325 }
326}
327
328#[cfg(test)]
329mod tests {
330 use super::*;
331 use crate::{
332 adapters::{BackendAdapter, ServeRequest},
333 identity::ProcessIdentity,
334 state::{RuntimeState, ServerState},
335 };
336 use std::sync::Mutex;
337 use tokio_util::sync::CancellationToken;
338
339 #[test]
340 fn diagnostic_sink_bounds_progress_without_evicting_security_warnings() {
341 let diagnostics = DiagnosticObserver::default();
342 diagnostics.warning(LifecycleWarning::InstalledBypass);
343 diagnostics.warning(LifecycleWarning::EstimatedFit);
344 diagnostics.warning(LifecycleWarning::SizeOnly);
345 for completed in 0..100 {
346 download_progress(
347 Some(&diagnostics),
348 completed,
349 100,
350 "\u{1b}]52;c;secret\u{7}/token=secret.bin",
351 );
352 }
353 let snapshot = diagnostics.snapshot();
354 assert_eq!(snapshot.progress.len(), EVENT_CAPACITY);
355 assert_eq!(snapshot.omitted_progress, 36);
356 let text = snapshot.text();
357 assert!(text.contains("no catalog SHA-256"));
358 assert!(text.contains("bypassing estimated fit"));
359 assert!(text.contains("may not fit"));
360 assert!(text.contains("99/100 bytes"));
361 assert!(text.contains("36 earlier progress diagnostics omitted"));
362 assert!(text.contains("[redacted filename]"));
363 assert!(!text.contains("secret"));
364 assert!(!text.contains('\u{1b}'));
365 }
366
367 #[test]
368 fn diagnostic_sink_is_injected_and_keeps_repeated_warning_counts() {
369 let (plain, _) = LifecycleObserver::channel();
370 assert!(plain.diagnostics.is_none());
371 let diagnostics = DiagnosticObserver::default();
372 let observed = plain.with_diagnostics(diagnostics.clone());
373 for _ in 0..100 {
374 warning(Some(&observed), LifecycleWarning::SizeOnly);
375 }
376 assert!(diagnostics.snapshot().text().contains("100 occurrences"));
377 assert!(DiagnosticObserver::default().snapshot().text().is_empty());
378 }
379
380 struct Adapter {
381 fail: Option<LifecycleStage>,
382 calls: Mutex<Vec<LifecycleStage>>,
383 }
384
385 impl Adapter {
386 fn attempt(
387 &self,
388 stage: LifecycleStage,
389 cancel: &CancellationToken,
390 ) -> Result<(), BackendError> {
391 self.calls.lock().unwrap().push(stage);
392 if cancel.is_cancelled() {
393 Err(BackendError("cancelled".into()))
394 } else if self.fail == Some(stage) {
395 Err(BackendError("mock operation failed".into()))
396 } else {
397 Ok(())
398 }
399 }
400 }
401
402 fn handle() -> ServerState {
403 RuntimeState::parse(r#"{"schemaVersion":2,"active":{"backend":"ollama","modelId":"model:latest","endpoint":"http://127.0.0.1:11434","port":11434,"ownedByUs":true,"pid":123}}"#)
404 .unwrap().active.unwrap()
405 }
406
407 fn request() -> ServeRequest {
408 ServeRequest {
409 draft_model_path: None,
410 model_id: "model:latest".into(),
411 endpoint: "http://127.0.0.1:11434".into(),
412 model_path: None,
413 context: Some(4096),
414 cache: None,
415 slot_save_path: None,
416 }
417 }
418
419 #[async_trait::async_trait]
420 impl BackendAdapter for Adapter {
421 fn name(&self) -> &'static str {
422 "ollama"
423 }
424 fn trusts(&self, _identity: &ProcessIdentity) -> bool {
425 false
426 }
427 async fn serve(
428 &self,
429 request: &ServeRequest,
430 cancel: &CancellationToken,
431 ) -> Result<ServerState, BackendError> {
432 assert_eq!(
433 request.context,
434 if request.model_id == "llmup-acquisition" {
435 None
436 } else {
437 Some(4096)
438 }
439 );
440 self.attempt(LifecycleStage::Start, cancel)?;
441 Ok(handle())
442 }
443 async fn attach_only(
444 &self,
445 _request: &ServeRequest,
446 cancel: &CancellationToken,
447 ) -> Result<ServerState, BackendError> {
448 self.attempt(LifecycleStage::Attach, cancel)?;
449 let mut handle = handle();
450 handle.owned_by_us = false;
451 Ok(handle)
452 }
453 async fn ready(
454 &self,
455 _request: &ServeRequest,
456 _cancel: &CancellationToken,
457 ) -> Result<(), BackendError> {
458 panic!("ready_handle override must be forwarded")
459 }
460 async fn ready_handle(
461 &self,
462 _request: &ServeRequest,
463 active: &ServerState,
464 cancel: &CancellationToken,
465 ) -> Result<(), BackendError> {
466 assert_eq!(active.pid, Some(123));
467 self.attempt(LifecycleStage::Readiness, cancel)
468 }
469 async fn stop(
470 &self,
471 active: &ServerState,
472 cancel: &CancellationToken,
473 ) -> Result<(), BackendError> {
474 assert!(active.owned_by_us);
475 self.attempt(LifecycleStage::Stop, cancel)
476 }
477 }
478
479 #[tokio::test]
480 async fn runtime_events_adapter_orders_real_calls_and_preserves_failures() {
481 for failure in [
482 None,
483 Some(LifecycleStage::Start),
484 Some(LifecycleStage::Readiness),
485 Some(LifecycleStage::Stop),
486 ] {
487 let (observer, mut receiver) = LifecycleObserver::channel();
488 let adapter = Adapter {
489 fail: failure,
490 calls: Mutex::new(Vec::new()),
491 };
492 let observed = ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::Runtime);
493 let cancel = CancellationToken::new();
494 let outcome = async {
495 let active = observed.serve(&request(), &cancel).await?;
496 observed.ready_handle(&request(), &active, &cancel).await?;
497 observed.stop(&active, &cancel).await
498 }
499 .await;
500 assert_eq!(outcome.is_err(), failure.is_some());
501 if let Err(error) = outcome {
502 assert_eq!(error.0, "mock operation failed");
503 }
504 let mut expected = Vec::new();
505 for stage in [
506 LifecycleStage::Start,
507 LifecycleStage::Readiness,
508 LifecycleStage::Stop,
509 ] {
510 expected.push(stage);
511 assert_eq!(
512 receiver.try_recv().unwrap(),
513 LifecycleEvent {
514 scope: LifecycleScope::Runtime,
515 stage,
516 status: LifecycleStatus::Started
517 }
518 );
519 assert_eq!(
520 receiver.try_recv().unwrap(),
521 LifecycleEvent {
522 scope: LifecycleScope::Runtime,
523 stage,
524 status: if failure == Some(stage) {
525 LifecycleStatus::Failed
526 } else {
527 LifecycleStatus::Completed
528 }
529 }
530 );
531 if failure == Some(stage) {
532 break;
533 }
534 }
535 assert_eq!(*adapter.calls.lock().unwrap(), expected);
536 assert!(receiver.try_recv().is_err());
537 }
538 }
539
540 #[tokio::test]
541 async fn runtime_events_adapter_preserves_attach_and_cancellation() {
542 let (observer, mut receiver) = LifecycleObserver::channel();
543 let adapter = Adapter {
544 fail: None,
545 calls: Mutex::new(Vec::new()),
546 };
547 let observed =
548 ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
549 let cancel = CancellationToken::new();
550 assert!(
551 !observed
552 .attach_only(&request(), &cancel)
553 .await
554 .unwrap()
555 .owned_by_us
556 );
557 assert_eq!(receiver.try_recv().unwrap().stage, LifecycleStage::Attach);
558 assert_eq!(
559 receiver.try_recv().unwrap().status,
560 LifecycleStatus::Completed
561 );
562 cancel.cancel();
563 assert_eq!(
564 observed.serve(&request(), &cancel).await.unwrap_err().0,
565 "cancelled"
566 );
567 assert_eq!(
568 receiver.try_recv().unwrap().scope,
569 LifecycleScope::AcquisitionDaemon
570 );
571 assert_eq!(receiver.try_recv().unwrap().status, LifecycleStatus::Failed);
572 assert!(receiver.try_recv().is_err());
573 }
574
575 #[tokio::test]
576 async fn saturated_or_closed_observation_never_retries_or_replaces_runtime_result() {
577 for closed in [false, true] {
578 let (observer, mut receiver) = LifecycleObserver::channel();
579 for _ in 0..EVENT_CAPACITY {
580 observer.emit(
581 LifecycleScope::Runtime,
582 LifecycleStage::Start,
583 LifecycleStatus::Started,
584 );
585 }
586 if closed {
587 receiver.close();
588 }
589 let calls = std::sync::atomic::AtomicUsize::new(0);
590 let result = tokio::time::timeout(
591 std::time::Duration::from_secs(1),
592 observe(
593 Some(&observer),
594 LifecycleScope::Runtime,
595 LifecycleStage::Readiness,
596 async {
597 calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
598 Err::<(), _>(BackendError("authoritative runtime failure".into()))
599 },
600 ),
601 )
602 .await
603 .unwrap();
604 assert_eq!(result.unwrap_err().0, "authoritative runtime failure");
605 assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
606 assert_eq!(receiver.len(), EVENT_CAPACITY);
607 let result = observe(
608 Some(&observer),
609 LifecycleScope::Runtime,
610 LifecycleStage::Stop,
611 async { Ok::<_, BackendError>("authoritative success") },
612 )
613 .await
614 .unwrap();
615 assert_eq!(result, "authoritative success");
616 assert_eq!(receiver.len(), EVENT_CAPACITY);
617 }
618 }
619
620 #[tokio::test]
621 async fn runtime_events_temporary_daemon_cleanup_does_not_claim_pull_success() {
622 let (observer, mut receiver) = LifecycleObserver::channel();
623 let adapter = Adapter {
624 fail: None,
625 calls: Mutex::new(Vec::new()),
626 };
627 let observed =
628 ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
629 let cancel = CancellationToken::new();
630 let operation = observe(
631 Some(&observer),
632 LifecycleScope::Runtime,
633 LifecycleStage::Acquisition,
634 async {
635 cancel.cancel();
636 Err::<(), _>(BackendError("pull cancelled".into()))
637 },
638 );
639 let result = crate::pull::with_ollama_daemon(
640 &observed,
641 "http://127.0.0.1:11434",
642 &cancel,
643 operation,
644 )
645 .await;
646 assert_eq!(result.unwrap_err().0, "pull cancelled");
647 assert_eq!(
648 *adapter.calls.lock().unwrap(),
649 [LifecycleStage::Start, LifecycleStage::Stop]
650 );
651 for (scope, stage, status) in [
652 (
653 LifecycleScope::AcquisitionDaemon,
654 LifecycleStage::Start,
655 LifecycleStatus::Started,
656 ),
657 (
658 LifecycleScope::AcquisitionDaemon,
659 LifecycleStage::Start,
660 LifecycleStatus::Completed,
661 ),
662 (
663 LifecycleScope::Runtime,
664 LifecycleStage::Acquisition,
665 LifecycleStatus::Started,
666 ),
667 (
668 LifecycleScope::Runtime,
669 LifecycleStage::Acquisition,
670 LifecycleStatus::Failed,
671 ),
672 (
673 LifecycleScope::AcquisitionDaemon,
674 LifecycleStage::Stop,
675 LifecycleStatus::Started,
676 ),
677 (
678 LifecycleScope::AcquisitionDaemon,
679 LifecycleStage::Stop,
680 LifecycleStatus::Completed,
681 ),
682 ] {
683 assert_eq!(
684 receiver.try_recv().unwrap(),
685 LifecycleEvent {
686 scope,
687 stage,
688 status
689 }
690 );
691 }
692 assert!(receiver.try_recv().is_err());
693 }
694
695 #[tokio::test]
696 async fn runtime_events_bracket_success_and_failure_without_false_completion() {
697 let (observer, mut receiver) = LifecycleObserver::channel();
698 let operation = async {
699 assert_eq!(
700 receiver.try_recv().unwrap(),
701 LifecycleEvent {
702 scope: LifecycleScope::Runtime,
703 stage: LifecycleStage::Verification,
704 status: LifecycleStatus::Started,
705 }
706 );
707 Ok::<_, BackendError>(42)
708 };
709 assert_eq!(
710 observe(
711 Some(&observer),
712 LifecycleScope::Runtime,
713 LifecycleStage::Verification,
714 operation
715 )
716 .await
717 .unwrap(),
718 42
719 );
720 assert_eq!(
721 receiver.try_recv().unwrap().status,
722 LifecycleStatus::Completed
723 );
724 let error = observe(
725 Some(&observer),
726 LifecycleScope::Runtime,
727 LifecycleStage::Start,
728 async { Err::<(), _>(BackendError("private diagnostic".into())) },
729 )
730 .await
731 .unwrap_err();
732 assert_eq!(error.0, "private diagnostic");
733 assert_eq!(
734 receiver.try_recv().unwrap().status,
735 LifecycleStatus::Started
736 );
737 let failed = receiver.try_recv().unwrap();
738 assert_eq!(failed.status, LifecycleStatus::Failed);
739 assert!(!format!("{failed:?}").contains("private diagnostic"));
740 assert!(receiver.try_recv().is_err());
741 }
742
743 #[tokio::test]
744 async fn runtime_events_full_closed_or_absent_observer_never_blocks_operations() {
745 let (observer, mut receiver) = LifecycleObserver::channel();
746 for _ in 0..EVENT_CAPACITY * 2 {
747 observe(
748 Some(&observer),
749 LifecycleScope::Runtime,
750 LifecycleStage::Readiness,
751 async { Ok::<_, BackendError>(()) },
752 )
753 .await
754 .unwrap();
755 }
756 assert_eq!(receiver.len(), EVENT_CAPACITY);
757 receiver.close();
758 for observer in [Some(&observer), None] {
759 assert_eq!(
760 observe(
761 observer,
762 LifecycleScope::Runtime,
763 LifecycleStage::Stop,
764 async { Ok::<_, BackendError>(7) }
765 )
766 .await
767 .unwrap(),
768 7
769 );
770 }
771 }
772
773 #[tokio::test]
774 async fn runtime_events_dropped_operation_has_no_completion() {
775 let (observer, mut receiver) = LifecycleObserver::channel();
776 {
777 let operation = observe(
778 Some(&observer),
779 LifecycleScope::Runtime,
780 LifecycleStage::Readiness,
781 std::future::pending::<Result<(), BackendError>>(),
782 );
783 tokio::pin!(operation);
784 tokio::select! {
785 biased;
786 _ = &mut operation => panic!("pending operation completed"),
787 event = receiver.recv() => assert_eq!(event.unwrap().status, LifecycleStatus::Started),
788 }
789 }
790 assert!(receiver.try_recv().is_err());
791 }
792}