ic_testkit/pic/
diagnostics.rs

1use std::{
2    fmt, io,
3    time::{Duration, Instant},
4};
5
6use ic_host_artifacts::batch::{BatchLabelError, validate_labels};
7use ic_platform_pocket_ic::{CanisterLogRecord, CanisterStatusResult, ClientError, PocketIc};
8use ic_platform_types::principal::ExternalPrincipal as Principal;
9
10/// Default maximum number of canister-log records retained in diagnostics.
11pub const DEFAULT_CANISTER_LOG_RECORD_LIMIT: usize = 32;
12
13/// Default maximum aggregate raw canister-log bytes retained in diagnostics.
14pub const DEFAULT_CANISTER_LOG_BYTE_LIMIT: usize = 16 * 1024;
15
16/// Explicit bounds applied when canister logs are converted to diagnostic text.
17#[derive(Clone, Copy, Debug, Eq, PartialEq)]
18pub struct CanisterLogRenderLimits {
19    /// Maximum retained records; zero retains none.
20    pub record_limit: usize,
21    /// Maximum retained raw-content bytes; zero retains none.
22    pub byte_limit: usize,
23}
24
25impl Default for CanisterLogRenderLimits {
26    fn default() -> Self {
27        Self {
28            record_limit: DEFAULT_CANISTER_LOG_RECORD_LIMIT,
29            byte_limit: DEFAULT_CANISTER_LOG_BYTE_LIMIT,
30        }
31    }
32}
33
34/// Exact controller-aware inputs for one best-effort diagnostic collection.
35#[derive(Clone, Copy, Debug, Eq, PartialEq)]
36pub struct CanisterDiagnosticsRequest {
37    /// Canister to inspect.
38    pub canister_id: Principal,
39    /// Exact sender for the status request.
40    pub status_sender: Principal,
41    /// Exact sender for the log request.
42    pub log_sender: Principal,
43    /// Caller-selected log rendering bounds.
44    pub log_limits: CanisterLogRenderLimits,
45}
46
47/// One caller-labeled exact diagnostic request in a collect-all batch.
48#[derive(Clone, Debug, Eq, PartialEq)]
49pub struct LabeledCanisterDiagnosticsRequest {
50    /// Caller label, checked for emptiness and uniqueness before collection.
51    pub label: String,
52    /// Exact controller-aware diagnostic inputs.
53    pub request: CanisterDiagnosticsRequest,
54}
55
56/// One canister-log record retained as bounded lossy UTF-8 text.
57#[derive(Clone, Debug, Eq, PartialEq)]
58pub struct CanisterDiagnosticLogRecord {
59    index: u64,
60    timestamp_nanos: u64,
61    content: String,
62    original_content_bytes: usize,
63    omitted_content_bytes: usize,
64}
65
66impl CanisterDiagnosticLogRecord {
67    /// Upstream record index.
68    #[must_use]
69    pub const fn index(&self) -> u64 {
70        self.index
71    }
72
73    /// Upstream record timestamp in nanoseconds.
74    #[must_use]
75    pub const fn timestamp_nanos(&self) -> u64 {
76        self.timestamp_nanos
77    }
78
79    /// Retained content converted with lossy UTF-8 decoding.
80    #[must_use]
81    pub fn content(&self) -> &str {
82        &self.content
83    }
84
85    /// Original raw content length before bounding and conversion.
86    #[must_use]
87    pub const fn original_content_bytes(&self) -> usize {
88        self.original_content_bytes
89    }
90
91    /// Raw content bytes omitted from this retained record.
92    #[must_use]
93    pub const fn omitted_content_bytes(&self) -> usize {
94        self.omitted_content_bytes
95    }
96
97    /// Whether this record's content was truncated.
98    #[must_use]
99    pub const fn was_truncated(&self) -> bool {
100        self.omitted_content_bytes != 0
101    }
102}
103
104/// Successfully fetched canister logs after bounded text conversion.
105#[derive(Clone, Debug, Eq, PartialEq)]
106pub struct CanisterDiagnosticLogs {
107    records: Vec<CanisterDiagnosticLogRecord>,
108    total_records: usize,
109    total_content_bytes: usize,
110    omitted_content_bytes: usize,
111}
112
113impl CanisterDiagnosticLogs {
114    /// Retained records in upstream order.
115    #[must_use]
116    pub fn records(&self) -> &[CanisterDiagnosticLogRecord] {
117        &self.records
118    }
119
120    /// Total number of records returned by `PocketIC` before bounding.
121    #[must_use]
122    pub const fn total_records(&self) -> usize {
123        self.total_records
124    }
125
126    /// Total raw content bytes returned by `PocketIC` before bounding.
127    #[must_use]
128    pub const fn total_content_bytes(&self) -> usize {
129        self.total_content_bytes
130    }
131
132    /// Number of whole records omitted by the configured bounds.
133    #[must_use]
134    pub const fn omitted_records(&self) -> usize {
135        self.total_records - self.records.len()
136    }
137
138    /// Aggregate raw content bytes omitted across retained and omitted records.
139    #[must_use]
140    pub const fn omitted_content_bytes(&self) -> usize {
141        self.omitted_content_bytes
142    }
143
144    /// Whether either whole records or record content were truncated.
145    #[must_use]
146    pub const fn was_truncated(&self) -> bool {
147        self.omitted_records() != 0 || self.omitted_content_bytes != 0
148    }
149}
150
151impl fmt::Display for CanisterDiagnosticLogs {
152    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
153        if self.records.is_empty() {
154            if self.total_records == 0 {
155                formatter.write_str("<empty>")?;
156            } else {
157                formatter.write_str("<no retained records>")?;
158            }
159        } else {
160            for (position, record) in self.records.iter().enumerate() {
161                if position != 0 {
162                    formatter.write_str(", ")?;
163                }
164                write!(
165                    formatter,
166                    "[{}@{}]={:?}",
167                    record.index, record.timestamp_nanos, record.content
168                )?;
169                if record.was_truncated() {
170                    write!(
171                        formatter,
172                        " (truncated {} bytes)",
173                        record.omitted_content_bytes
174                    )?;
175                }
176            }
177        }
178        if self.was_truncated() {
179            write!(
180                formatter,
181                "; truncated omitted_records={} omitted_content_bytes={}",
182                self.omitted_records(),
183                self.omitted_content_bytes
184            )?;
185        }
186        Ok(())
187    }
188}
189
190/// Independent status and log outcomes for one diagnostic collection.
191#[derive(Debug)]
192pub struct CanisterDiagnosticsReport {
193    /// Exact request used for this report.
194    pub request: CanisterDiagnosticsRequest,
195    /// Status result, independent of log retrieval.
196    pub status: Result<CanisterStatusResult, ClientError>,
197    /// Log result, independent of status retrieval.
198    pub logs: Result<CanisterDiagnosticLogs, ClientError>,
199}
200
201impl CanisterDiagnosticsReport {
202    /// Whether both status and log collection succeeded.
203    #[must_use]
204    pub const fn is_success(&self) -> bool {
205        self.status.is_ok() && self.logs.is_ok()
206    }
207}
208
209/// One ordered labeled entry in a collect-all diagnostics batch.
210#[derive(Debug)]
211pub struct CanisterDiagnosticsBatchEntry {
212    /// Caller-supplied label.
213    pub label: String,
214    /// Structured status and log outcomes for this entry.
215    pub report: CanisterDiagnosticsReport,
216    /// Complete wall-clock time for this target's diagnostic collection.
217    pub entry_elapsed: Duration,
218}
219
220impl CanisterDiagnosticsBatchEntry {
221    /// Whether both status and log collection succeeded for this entry.
222    #[must_use]
223    pub const fn is_success(&self) -> bool {
224        self.report.is_success()
225    }
226}
227
228/// Ordered entries from a sequential collect-all diagnostics batch.
229#[derive(Debug, Default)]
230pub struct CanisterDiagnosticsBatchReport {
231    /// Entries in the supplied request order.
232    pub entries: Vec<CanisterDiagnosticsBatchEntry>,
233    /// Complete wall-clock time for the sequential collect-all batch.
234    pub total: Duration,
235}
236
237impl CanisterDiagnosticsBatchReport {
238    /// Entries with at least one failed status or log operation.
239    pub fn failures(&self) -> impl Iterator<Item = &CanisterDiagnosticsBatchEntry> {
240        self.entries.iter().filter(|entry| !entry.is_success())
241    }
242
243    /// Whether every entry collected both status and logs successfully.
244    #[must_use]
245    pub fn is_success(&self) -> bool {
246        self.entries
247            .iter()
248            .all(CanisterDiagnosticsBatchEntry::is_success)
249    }
250}
251
252impl fmt::Display for CanisterDiagnosticsBatchReport {
253    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
254        write!(
255            formatter,
256            "diagnostics={} total={:?}",
257            self.entries.len(),
258            self.total
259        )?;
260        for entry in &self.entries {
261            write!(
262                formatter,
263                "; label={:?} elapsed={:?} {}",
264                entry.label, entry.entry_elapsed, entry.report
265            )?;
266        }
267        Ok(())
268    }
269}
270
271impl fmt::Display for CanisterDiagnosticsReport {
272    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
273        write!(
274            formatter,
275            "canister={} status_sender={} status=",
276            self.request.canister_id, self.request.status_sender
277        )?;
278        match &self.status {
279            Ok(status) => write!(
280                formatter,
281                "ok(state={:?} version={} controllers={} module_hash_bytes={} memory_bytes={} cycles={})",
282                status.status,
283                status.version,
284                status.settings.controllers.len(),
285                status.module_hash.as_ref().map_or(0, Vec::len),
286                status.memory_size,
287                status.cycles,
288            ),
289            Err(failure) => write!(formatter, "<{failure}>"),
290        }?;
291        write!(formatter, " log_sender={} logs=", self.request.log_sender)?;
292        match &self.logs {
293            Err(failure) => write!(formatter, "<{failure}>")?,
294            Ok(logs) => write!(formatter, "{logs}")?,
295        }
296        Ok(())
297    }
298}
299
300/// Reusable structured `PocketIC` failure diagnostics.
301pub trait PocketIcDiagnosticsExt {
302    /// Collect status and bounded logs using the request's exact senders.
303    ///
304    /// Both calls are attempted independently. Typed client failures remain in
305    /// the report so diagnostics can stay subordinate to the original failure.
306    fn collect_canister_diagnostics(
307        &self,
308        request: CanisterDiagnosticsRequest,
309    ) -> CanisterDiagnosticsReport;
310
311    /// Explicitly collect and write one contextual diagnostic report.
312    ///
313    /// Uses the request's exact senders and log limits, with no retry or output
314    /// destination selected implicitly. Unrelated collection/formatting/writer
315    /// panics propagate.
316    /// Ordinary writer errors are returned unchanged. Status/log failures remain
317    /// in the written report; successful output does not mean those calls passed.
318    /// Callers keep their primary failure authoritative and decide whether to
319    /// retain or ignore this secondary reporting result.
320    ///
321    /// # Errors
322    /// Returns the writer’s I/O error when diagnostics cannot be written.
323    fn write_canister_diagnostics(
324        &self,
325        request: CanisterDiagnosticsRequest,
326        context: &str,
327        writer: &mut dyn io::Write,
328    ) -> io::Result<()> {
329        let report = self.collect_canister_diagnostics(request);
330        writeln!(writer, "{context}: {report}")
331    }
332
333    /// Collect every labeled request sequentially in its supplied order.
334    ///
335    /// Every target is attempted even when an earlier entry has client failures.
336    /// Each entry preserves its exact request and independent status/log
337    /// outcomes; no anonymous retry or fallback is performed.
338    /// Invalid labels return Host's [`BatchLabelError`] before collection starts.
339    ///
340    /// # Errors
341    /// Rejects invalid or duplicate labels before collecting diagnostics.
342    fn collect_canister_diagnostics_batch(
343        &self,
344        requests: &[LabeledCanisterDiagnosticsRequest],
345    ) -> Result<CanisterDiagnosticsBatchReport, BatchLabelError> {
346        validate_labels(requests.iter().map(|request| request.label.as_str()))?;
347        let started = Instant::now();
348        let entries = requests
349            .iter()
350            .map(|labeled| {
351                let entry_started = Instant::now();
352                let request = labeled.request;
353                let report = self.collect_canister_diagnostics(request);
354                CanisterDiagnosticsBatchEntry {
355                    label: labeled.label.clone(),
356                    report,
357                    entry_elapsed: entry_started.elapsed(),
358                }
359            })
360            .collect();
361        Ok(CanisterDiagnosticsBatchReport {
362            entries,
363            total: started.elapsed(),
364        })
365    }
366}
367
368impl PocketIcDiagnosticsExt for PocketIc {
369    fn collect_canister_diagnostics(
370        &self,
371        request: CanisterDiagnosticsRequest,
372    ) -> CanisterDiagnosticsReport {
373        let status = self.canister_status(request.canister_id, Some(request.status_sender));
374        let logs = self
375            .fetch_canister_logs(request.canister_id, request.log_sender)
376            .map(|records| render_log_records(records, request.log_limits));
377
378        CanisterDiagnosticsReport {
379            request,
380            status,
381            logs,
382        }
383    }
384}
385
386fn render_log_records(
387    records: Vec<CanisterLogRecord>,
388    limits: CanisterLogRenderLimits,
389) -> CanisterDiagnosticLogs {
390    let total_records = records.len();
391    let mut total_content_bytes = 0usize;
392    let mut rendered = Vec::with_capacity(total_records.min(limits.record_limit));
393    let mut retained_bytes = 0usize;
394    let mut omitted_content_bytes = 0usize;
395
396    for record in records {
397        total_content_bytes = total_content_bytes.saturating_add(record.content.len());
398        let include = rendered.len() < limits.record_limit && retained_bytes < limits.byte_limit;
399        let retained = if include {
400            record
401                .content
402                .len()
403                .min(limits.byte_limit.saturating_sub(retained_bytes))
404        } else {
405            0
406        };
407        let omitted = record.content.len().saturating_sub(retained);
408        omitted_content_bytes = omitted_content_bytes.saturating_add(omitted);
409        if !include {
410            continue;
411        }
412
413        let content = String::from_utf8_lossy(&record.content[..retained]).into_owned();
414        retained_bytes = retained_bytes.saturating_add(retained);
415        rendered.push(CanisterDiagnosticLogRecord {
416            index: record.idx,
417            timestamp_nanos: record.timestamp_nanos,
418            content,
419            original_content_bytes: record.content.len(),
420            omitted_content_bytes: omitted,
421        });
422    }
423
424    CanisterDiagnosticLogs {
425        records: rendered,
426        total_records,
427        total_content_bytes,
428        omitted_content_bytes,
429    }
430}
431
432#[cfg(test)]
433mod tests {
434    use std::cell::Cell;
435
436    use ic_host_artifacts::batch::BatchLabelError;
437    use ic_platform_pocket_ic::CanisterLogRecord;
438    use ic_platform_types::principal::ExternalPrincipal as Principal;
439
440    use super::{
441        CanisterDiagnosticsReport, CanisterDiagnosticsRequest, CanisterLogRenderLimits,
442        LabeledCanisterDiagnosticsRequest, PocketIcDiagnosticsExt, render_log_records,
443    };
444
445    struct Reporting {
446        calls: Cell<usize>,
447        request: Cell<Option<CanisterDiagnosticsRequest>>,
448    }
449
450    impl PocketIcDiagnosticsExt for Reporting {
451        fn collect_canister_diagnostics(
452            &self,
453            request: CanisterDiagnosticsRequest,
454        ) -> CanisterDiagnosticsReport {
455            self.calls.set(self.calls.get() + 1);
456            self.request.set(Some(request));
457            CanisterDiagnosticsReport {
458                request,
459                status: Err(ic_platform_pocket_ic::ClientError::Protocol(
460                    "synthetic status failure".into(),
461                )),
462                logs: Err(ic_platform_pocket_ic::ClientError::Protocol(
463                    "synthetic log failure".into(),
464                )),
465            }
466        }
467    }
468
469    #[test]
470    fn explicit_reporting_preserves_requests_and_writer_failures() {
471        let collector = Reporting {
472            calls: Cell::new(0),
473            request: Cell::new(None),
474        };
475        let request = CanisterDiagnosticsRequest {
476            canister_id: Principal::from_slice(&[1]),
477            status_sender: Principal::from_slice(&[2]),
478            log_sender: Principal::from_slice(&[3]),
479            log_limits: CanisterLogRenderLimits::default(),
480        };
481        let mut output = Vec::new();
482        collector
483            .write_canister_diagnostics(request, "install", &mut output)
484            .unwrap();
485        let text = String::from_utf8(output).unwrap();
486        assert!(
487            text.contains("synthetic status failure") && text.contains("synthetic log failure")
488        );
489        let error = collector
490            .write_canister_diagnostics(request, "install", &mut std::io::Cursor::new([0_u8; 0]))
491            .unwrap_err();
492        assert_eq!(error.kind(), std::io::ErrorKind::WriteZero);
493        assert_eq!(collector.calls.get(), 2);
494        assert_eq!(collector.request.get(), Some(request));
495    }
496
497    #[test]
498    fn labeled_batch_retains_order_and_continues_after_entry_failure() {
499        let collector = Reporting {
500            calls: Cell::new(0),
501            request: Cell::new(None),
502        };
503        let first = CanisterDiagnosticsRequest {
504            canister_id: Principal::from_slice(&[1]),
505            status_sender: Principal::from_slice(&[2]),
506            log_sender: Principal::from_slice(&[3]),
507            log_limits: CanisterLogRenderLimits::default(),
508        };
509        let second = CanisterDiagnosticsRequest {
510            canister_id: Principal::from_slice(&[4]),
511            status_sender: Principal::from_slice(&[5]),
512            log_sender: Principal::from_slice(&[6]),
513            log_limits: CanisterLogRenderLimits::default(),
514        };
515        let report = collector
516            .collect_canister_diagnostics_batch(&[
517                LabeledCanisterDiagnosticsRequest {
518                    label: "root".into(),
519                    request: first,
520                },
521                LabeledCanisterDiagnosticsRequest {
522                    label: "worker".into(),
523                    request: second,
524                },
525            ])
526            .expect("valid labeled diagnostics batch");
527
528        assert_eq!(collector.calls.get(), 2);
529        assert_eq!(report.entries.len(), 2);
530        assert_eq!(report.entries[0].label, "root");
531        assert_eq!(report.entries[0].report.request, first);
532        assert_eq!(report.entries[1].label, "worker");
533        assert_eq!(report.entries[1].report.request, second);
534        assert!(
535            report
536                .entries
537                .iter()
538                .all(|entry| entry.entry_elapsed <= report.total)
539        );
540        assert_eq!(report.failures().count(), 2);
541        assert!(!report.is_success());
542        let compact = report.to_string();
543        assert!(compact.contains("label=\"root\""));
544        assert!(compact.contains("label=\"worker\""));
545        assert!(compact.contains("synthetic status failure"));
546    }
547
548    #[test]
549    fn diagnostic_batch_rejects_invalid_labels_before_collection() {
550        let collector = Reporting {
551            calls: Cell::new(0),
552            request: Cell::new(None),
553        };
554        let request = CanisterDiagnosticsRequest {
555            canister_id: Principal::from_slice(&[1]),
556            status_sender: Principal::from_slice(&[2]),
557            log_sender: Principal::from_slice(&[3]),
558            log_limits: CanisterLogRenderLimits::default(),
559        };
560        let empty = collector
561            .collect_canister_diagnostics_batch(&[LabeledCanisterDiagnosticsRequest {
562                label: String::new(),
563                request,
564            }])
565            .expect_err("empty label must reject diagnostics batch");
566        assert_eq!(empty, BatchLabelError::Empty { index: 0 });
567        assert_eq!(collector.calls.get(), 0);
568
569        let duplicate = collector
570            .collect_canister_diagnostics_batch(&[
571                LabeledCanisterDiagnosticsRequest {
572                    label: "same".into(),
573                    request,
574                },
575                LabeledCanisterDiagnosticsRequest {
576                    label: "same".into(),
577                    request,
578                },
579            ])
580            .expect_err("duplicate labels must reject diagnostics batch");
581        assert_eq!(
582            duplicate,
583            BatchLabelError::Duplicate {
584                label: "same".to_owned(),
585                first_index: 0,
586                duplicate_index: 1,
587            }
588        );
589        assert_eq!(collector.calls.get(), 0);
590    }
591
592    #[test]
593    fn log_rendering_is_bounded_lossy_utf8_and_reports_truncation() {
594        let logs = render_log_records(
595            vec![
596                CanisterLogRecord {
597                    idx: 7,
598                    timestamp_nanos: 11,
599                    content: vec![b'f', 0x80, b'o'],
600                },
601                CanisterLogRecord {
602                    idx: 8,
603                    timestamp_nanos: 12,
604                    content: b"bar".to_vec(),
605                },
606            ],
607            CanisterLogRenderLimits {
608                record_limit: 1,
609                byte_limit: 2,
610            },
611        );
612
613        assert_eq!(logs.total_records(), 2);
614        assert_eq!(logs.total_content_bytes(), 6);
615        assert_eq!(logs.omitted_records(), 1);
616        assert_eq!(logs.omitted_content_bytes(), 4);
617        assert!(logs.was_truncated());
618        assert_eq!(logs.records().len(), 1);
619        assert_eq!(logs.records()[0].content(), "f�");
620        assert_eq!(logs.records()[0].original_content_bytes(), 3);
621        assert_eq!(logs.records()[0].omitted_content_bytes(), 1);
622        assert!(logs.records()[0].was_truncated());
623        let rendered = logs.to_string();
624        assert!(rendered.contains("f�"));
625        assert!(rendered.contains("truncated omitted_records=1 omitted_content_bytes=4"));
626    }
627
628    #[test]
629    fn zero_log_bounds_retain_only_aggregate_truncation() {
630        for (record_limit, byte_limit) in [(0, 0), (0, 16), (16, 0)] {
631            let logs = render_log_records(
632                vec![CanisterLogRecord {
633                    idx: 1,
634                    timestamp_nanos: 2,
635                    content: b"hello".to_vec(),
636                }],
637                CanisterLogRenderLimits {
638                    record_limit,
639                    byte_limit,
640                },
641            );
642
643            assert_eq!(logs.records(), []);
644            assert_eq!(logs.total_records(), 1);
645            assert_eq!(logs.total_content_bytes(), 5);
646            assert_eq!(logs.omitted_records(), 1);
647            assert_eq!(logs.omitted_content_bytes(), 5);
648            assert!(logs.was_truncated());
649            assert_eq!(
650                logs.to_string(),
651                "<no retained records>; truncated omitted_records=1 omitted_content_bytes=5"
652            );
653        }
654    }
655
656    #[test]
657    fn logs_within_bounds_preserve_empty_records_and_raw_byte_totals() {
658        let limits = CanisterLogRenderLimits {
659            record_limit: 3,
660            byte_limit: 3,
661        };
662        let empty = render_log_records(vec![], limits);
663        assert_eq!(empty.total_records(), 0);
664        assert_eq!(empty.total_content_bytes(), 0);
665        assert_eq!(empty.omitted_records(), 0);
666        assert_eq!(empty.omitted_content_bytes(), 0);
667        assert!(!empty.was_truncated());
668        assert_eq!(empty.to_string(), "<empty>");
669
670        let logs = render_log_records(
671            vec![
672                CanisterLogRecord {
673                    idx: 1,
674                    timestamp_nanos: 2,
675                    content: vec![],
676                },
677                CanisterLogRecord {
678                    idx: 3,
679                    timestamp_nanos: 4,
680                    content: vec![0x80, b'a'],
681                },
682                CanisterLogRecord {
683                    idx: 5,
684                    timestamp_nanos: 6,
685                    content: vec![],
686                },
687            ],
688            limits,
689        );
690        assert_eq!(logs.total_records(), 3);
691        assert_eq!(logs.records().len(), 3);
692        assert_eq!(logs.total_content_bytes(), 2);
693        assert_eq!(logs.records()[0].content(), "");
694        assert_eq!(logs.records()[1].content(), "�a");
695        assert_eq!(logs.records()[2].content(), "");
696        assert_eq!(logs.omitted_records(), 0);
697        assert_eq!(logs.omitted_content_bytes(), 0);
698        assert!(!logs.was_truncated());
699    }
700}
ic_testkit/pic/diagnostics.rs

ic_testkit/pic/
diagnostics.rs
