ic_testkit/benchmark/
mod.rs

1//! Parse, pair, aggregate, compare, and report compact benchmark markers.
2//!
3//! The normal pipeline is [`parse_benchmark_events`],
4//! [`pair_benchmark_spans`], [`aggregate_benchmark_spans`], and optionally
5//! [`compare_benchmark_aggregates`]. Native report output lives in
6//! the repository's `benchmark_report` example. Marker
7//! producers can use [`format_marker`] on the host or
8//! `ic_testkit::performance::Performance` in canister code.
9//!
10//! Totals and run counts remain exact integers. Means use IC Metrics' exact
11//! scaled integer projection, rounded down. Callers choose the scale and handle
12//! projection errors; reports use scale one in the original counter units.
13//! Comparisons retain current and previous means instead of percentage estimates.
14
15use std::collections::{BTreeMap, btree_map::Entry};
16
17use ic_metrics::{MeasurementRatioError, checked_scaled_ratio};
18
19/// Default prefix for benchmark marker lines.
20pub const DEFAULT_PREFIX: &str = "ICTK";
21
22/// Structural identity of one suite or the cross-suite aggregate.
23#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
24pub enum BenchmarkScope {
25    /// One caller-authored suite, including a suite literally named `ALL`.
26    Suite(String),
27    /// Aggregate across every suite.
28    All,
29}
30
31#[derive(Clone, Debug, Eq, PartialEq)]
32pub struct BenchmarkParserConfig {
33    pub prefixes: Vec<String>,
34    pub suite_derivation: SuiteDerivation,
35    /// When enabled, non-empty lines without a configured marker prefix are
36    /// reported as malformed markers instead of ignored log noise.
37    pub strict: bool,
38}
39
40impl Default for BenchmarkParserConfig {
41    fn default() -> Self {
42        Self {
43            prefixes: vec![DEFAULT_PREFIX.to_string()],
44            suite_derivation: SuiteDerivation::FirstPathSegment,
45            strict: false,
46        }
47    }
48}
49
50#[derive(Clone, Debug, Eq, PartialEq)]
51pub enum SuiteDerivation {
52    FirstPathSegment,
53    Fixed(String),
54}
55
56impl SuiteDerivation {
57    #[must_use]
58    pub fn derive_suite(&self, span_label: &str) -> String {
59        match self {
60            Self::FirstPathSegment => span_label
61                .split('/')
62                .next()
63                .filter(|part| !part.is_empty())
64                .unwrap_or(span_label)
65                .to_string(),
66            Self::Fixed(suite) => suite.clone(),
67        }
68    }
69}
70
71#[derive(Clone, Copy, Debug, Eq, PartialEq)]
72pub enum BenchmarkEventKind {
73    Start,
74    End,
75}
76
77#[derive(Clone, Copy, Debug, Eq, PartialEq)]
78pub enum BenchmarkEventSource {
79    Unknown,
80    Stdout,
81    Stderr,
82    FetchedLog,
83}
84
85impl BenchmarkEventSource {
86    #[must_use]
87    pub const fn as_str(self) -> &'static str {
88        match self {
89            Self::Unknown => "unknown",
90            Self::Stdout => "stdout",
91            Self::Stderr => "stderr",
92            Self::FetchedLog => "fetched_log",
93        }
94    }
95}
96
97#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
98pub struct BenchmarkCounters {
99    pub instructions: u128,
100    pub heap_bytes: u128,
101    pub memory_bytes: u128,
102}
103
104impl BenchmarkCounters {
105    fn checked_delta(self, start: Self) -> Option<Self> {
106        Some(Self {
107            instructions: self.instructions.checked_sub(start.instructions)?,
108            heap_bytes: self.heap_bytes.checked_sub(start.heap_bytes)?,
109            memory_bytes: self.memory_bytes.checked_sub(start.memory_bytes)?,
110        })
111    }
112
113    fn checked_add(self, other: Self) -> Result<Self, BenchmarkCounter> {
114        Ok(Self {
115            instructions: self
116                .instructions
117                .checked_add(other.instructions)
118                .ok_or(BenchmarkCounter::Instructions)?,
119            heap_bytes: self
120                .heap_bytes
121                .checked_add(other.heap_bytes)
122                .ok_or(BenchmarkCounter::HeapBytes)?,
123            memory_bytes: self
124                .memory_bytes
125                .checked_add(other.memory_bytes)
126                .ok_or(BenchmarkCounter::MemoryBytes)?,
127        })
128    }
129
130    fn min_assign(&mut self, other: Self) {
131        self.instructions = self.instructions.min(other.instructions);
132        self.heap_bytes = self.heap_bytes.min(other.heap_bytes);
133        self.memory_bytes = self.memory_bytes.min(other.memory_bytes);
134    }
135
136    fn max_assign(&mut self, other: Self) {
137        self.instructions = self.instructions.max(other.instructions);
138        self.heap_bytes = self.heap_bytes.max(other.heap_bytes);
139        self.memory_bytes = self.memory_bytes.max(other.memory_bytes);
140    }
141}
142
143#[derive(Clone, Debug, Eq, PartialEq)]
144pub struct RawBenchmarkEvent {
145    pub prefix: String,
146    pub suite: String,
147    pub span_label: String,
148    pub kind: BenchmarkEventKind,
149    pub counters: BenchmarkCounters,
150    pub source_line: usize,
151    pub source: BenchmarkEventSource,
152}
153
154#[derive(Clone, Debug, Eq, PartialEq)]
155pub struct MalformedBenchmarkMarker {
156    pub source_line: usize,
157    pub source: BenchmarkEventSource,
158    pub line: String,
159    pub reason: String,
160}
161
162#[derive(Clone, Debug, Default, Eq, PartialEq)]
163pub struct BenchmarkParseReport {
164    pub events: Vec<RawBenchmarkEvent>,
165    pub malformed_markers: Vec<MalformedBenchmarkMarker>,
166    pub ignored_line_count: usize,
167}
168
169#[derive(Clone, Debug, Eq, PartialEq)]
170pub struct BenchmarkSpan {
171    pub suite: String,
172    pub span_label: String,
173    pub start_line: usize,
174    pub end_line: usize,
175    pub end: BenchmarkCounters,
176    pub delta: BenchmarkCounters,
177}
178
179#[derive(Clone, Debug, Eq, PartialEq)]
180pub struct InvalidBenchmarkSpan {
181    pub start: RawBenchmarkEvent,
182    pub end: RawBenchmarkEvent,
183}
184
185#[derive(Clone, Debug, Default, Eq, PartialEq)]
186pub struct BenchmarkSpanReport {
187    pub spans: Vec<BenchmarkSpan>,
188    pub unpaired_markers: Vec<RawBenchmarkEvent>,
189    pub invalid_spans: Vec<InvalidBenchmarkSpan>,
190}
191
192#[derive(Clone, Debug, Eq, PartialEq)]
193pub struct BenchmarkAggregateRow {
194    pub span_label: String,
195    pub runs: u64,
196    pub total: BenchmarkCounters,
197    pub min: BenchmarkCounters,
198    pub max: BenchmarkCounters,
199    pub peak_end: BenchmarkCounters,
200    /// Suite identity or the cross-suite aggregate.
201    pub scope: BenchmarkScope,
202}
203
204/// Floor-rounded scaled means of exact aggregate totals and run counts.
205///
206/// Fields are `floor(total * scale / runs)`; choose a positive scale to
207/// interpret them in the original counter unit. Scale zero projects zero.
208#[derive(Clone, Copy, Debug, Eq, PartialEq)]
209pub struct BenchmarkAverages {
210    pub scale: u64,
211    pub instructions: u128,
212    pub heap_bytes: u128,
213    pub memory_bytes: u128,
214}
215
216/// Counter whose aggregate cannot be represented.
217#[derive(Clone, Copy, Debug, Eq, PartialEq)]
218pub enum BenchmarkCounter {
219    Runs,
220    Instructions,
221    HeapBytes,
222    MemoryBytes,
223}
224
225/// Overflow rejects the complete aggregation, without returning partial totals.
226#[derive(Clone, Debug, Eq, PartialEq)]
227pub struct BenchmarkAggregateError {
228    pub span_label: String,
229    pub counter: BenchmarkCounter,
230    /// Suite identity or the cross-suite aggregate.
231    pub scope: BenchmarkScope,
232}
233
234impl std::fmt::Display for BenchmarkAggregateError {
235    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
236        write!(
237            formatter,
238            "benchmark aggregate overflow: scope={:?}, span={}, counter={:?}",
239            self.scope, self.span_label, self.counter
240        )
241    }
242}
243
244impl std::error::Error for BenchmarkAggregateError {}
245
246/// Run count and scaled means for one present comparison side.
247#[derive(Clone, Copy, Debug, Eq, PartialEq)]
248pub struct BenchmarkMeasurement {
249    pub runs: u64,
250    pub average: BenchmarkAverages,
251}
252
253/// Current and previous measurements, preserving missing rows and measured zero.
254#[derive(Clone, Debug, Eq, PartialEq)]
255pub struct BenchmarkComparisonRow {
256    pub span_label: String,
257    pub current: Option<BenchmarkMeasurement>,
258    pub previous: Option<BenchmarkMeasurement>,
259    /// Suite identity or the cross-suite aggregate.
260    pub scope: BenchmarkScope,
261}
262
263impl BenchmarkAggregateRow {
264    /// Project exact totals at the caller's scale, rounding down.
265    /// Zero runs and unrepresentable scaled results return IC Metrics' typed error.
266    pub fn average(&self, scale: u64) -> Result<BenchmarkAverages, MeasurementRatioError> {
267        Ok(BenchmarkAverages {
268            scale,
269            instructions: checked_scaled_ratio(self.total.instructions, self.runs, scale)?,
270            heap_bytes: checked_scaled_ratio(self.total.heap_bytes, self.runs, scale)?,
271            memory_bytes: checked_scaled_ratio(self.total.memory_bytes, self.runs, scale)?,
272        })
273    }
274}
275
276#[must_use]
277pub fn format_marker(prefix: &str, label: &str, counters: BenchmarkCounters) -> String {
278    format!(
279        "{}|{}|{}|{}|{}",
280        prefix, label, counters.instructions, counters.heap_bytes, counters.memory_bytes
281    )
282}
283
284#[must_use]
285pub fn parse_benchmark_events(input: &str, config: &BenchmarkParserConfig) -> BenchmarkParseReport {
286    parse_benchmark_events_from_source(input, config, BenchmarkEventSource::Unknown)
287}
288
289#[must_use]
290pub fn parse_benchmark_events_from_source(
291    input: &str,
292    config: &BenchmarkParserConfig,
293    source: BenchmarkEventSource,
294) -> BenchmarkParseReport {
295    let mut report = BenchmarkParseReport::default();
296    append_benchmark_events(&mut report, input, config, source);
297    report
298}
299
300fn append_benchmark_events(
301    report: &mut BenchmarkParseReport,
302    input: &str,
303    config: &BenchmarkParserConfig,
304    source: BenchmarkEventSource,
305) {
306    for (index, line) in input.lines().enumerate() {
307        let source_line = index + 1;
308        if !has_configured_prefix(line, &config.prefixes) {
309            if config.strict && !line.trim().is_empty() {
310                report.malformed_markers.push(malformed(
311                    source_line,
312                    source,
313                    line,
314                    "line does not use a configured marker prefix",
315                ));
316            } else {
317                report.ignored_line_count += 1;
318            }
319            continue;
320        }
321
322        match parse_marker_line(line, source_line, source, config) {
323            Ok(event) => report.events.push(event),
324            Err(marker) => report.malformed_markers.push(marker),
325        }
326    }
327}
328
329/// Parse separately captured stdout and stderr.
330///
331/// Separate streams do not carry global ordering, so events are returned in
332/// stdout-then-stderr order. If a benchmark span can start on one stream and end
333/// on the other, capture combined process output and use [`parse_benchmark_events`].
334#[must_use]
335pub fn parse_benchmark_events_from_captured_output(
336    stdout: &str,
337    stderr: &str,
338    config: &BenchmarkParserConfig,
339) -> BenchmarkParseReport {
340    let mut report = BenchmarkParseReport::default();
341    append_benchmark_events(&mut report, stdout, config, BenchmarkEventSource::Stdout);
342    append_benchmark_events(&mut report, stderr, config, BenchmarkEventSource::Stderr);
343    report
344}
345
346#[must_use]
347pub fn pair_benchmark_spans(events: &[RawBenchmarkEvent]) -> BenchmarkSpanReport {
348    let mut report = BenchmarkSpanReport::default();
349    let mut open_starts: BTreeMap<(&str, &str), Vec<&RawBenchmarkEvent>> = BTreeMap::new();
350
351    for event in events {
352        let key = (event.suite.as_str(), event.span_label.as_str());
353        match event.kind {
354            BenchmarkEventKind::Start => open_starts.entry(key).or_default().push(event),
355            BenchmarkEventKind::End => match open_starts.entry(key) {
356                Entry::Occupied(mut entry) => {
357                    let start = entry
358                        .get_mut()
359                        .pop()
360                        .expect("open marker stacks must remain nonempty");
361                    if entry.get().is_empty() {
362                        entry.remove();
363                    }
364                    push_paired_span(&mut report, start, event);
365                }
366                Entry::Vacant(_) => report.unpaired_markers.push(event.clone()),
367            },
368        }
369    }
370
371    for starts in open_starts.into_values() {
372        for event in starts {
373            report.unpaired_markers.push(event.clone());
374        }
375    }
376
377    report
378}
379
380/// Aggregate spans by named suite and across all suites.
381///
382/// Returns an error if any counter total or run count overflows. No partial
383/// report is returned, and totals are never wrapped or saturated.
384pub fn aggregate_benchmark_spans(
385    spans: &[BenchmarkSpan],
386) -> Result<Vec<BenchmarkAggregateRow>, BenchmarkAggregateError> {
387    let mut rows = BTreeMap::new();
388
389    for span in spans {
390        add_span_to_aggregate(&mut rows, AggregateScope::Suite(&span.suite), span)?;
391        add_span_to_aggregate(&mut rows, AggregateScope::All, span)?;
392    }
393
394    Ok(rows
395        .into_iter()
396        .map(|((scope, span_label), builder)| {
397            builder.finish(scope.into_owned(), span_label.to_owned())
398        })
399        .collect())
400}
401
402/// Pair current and previous means at the same caller-selected scale.
403/// Returns a typed error if either side cannot supply the selected projection.
404pub fn compare_benchmark_aggregates(
405    current: &[BenchmarkAggregateRow],
406    previous: &[BenchmarkAggregateRow],
407    scale: u64,
408) -> Result<Vec<BenchmarkComparisonRow>, MeasurementRatioError> {
409    let mut rows = BTreeMap::new();
410    for row in current {
411        rows.insert((&row.scope, row.span_label.as_str()), (Some(row), None));
412    }
413    for row in previous {
414        rows.entry((&row.scope, row.span_label.as_str()))
415            .or_insert((None, None))
416            .1 = Some(row);
417    }
418    let measurement = |row: &BenchmarkAggregateRow| -> Result<_, MeasurementRatioError> {
419        Ok(BenchmarkMeasurement {
420            runs: row.runs,
421            average: row.average(scale)?,
422        })
423    };
424
425    rows.into_iter()
426        .map(|((scope, span_label), (current_row, previous_row))| {
427            Ok(BenchmarkComparisonRow {
428                span_label: span_label.to_owned(),
429                current: current_row.map(measurement).transpose()?,
430                previous: previous_row.map(measurement).transpose()?,
431                scope: scope.clone(),
432            })
433        })
434        .collect()
435}
436
437fn parse_marker_line(
438    line: &str,
439    source_line: usize,
440    source: BenchmarkEventSource,
441    config: &BenchmarkParserConfig,
442) -> Result<RawBenchmarkEvent, MalformedBenchmarkMarker> {
443    let mut parts = line.split('|');
444    // Five fields and an absent sixth establish the complete marker shape
445    // without collecting an arbitrary number of columns from malformed logs.
446    let columns: [Option<&str>; 6] = std::array::from_fn(|_| parts.next());
447    let [
448        Some(prefix),
449        Some(label),
450        Some(instructions),
451        Some(heap_bytes),
452        Some(memory_bytes),
453        None,
454    ] = columns
455    else {
456        return Err(malformed(
457            source_line,
458            source,
459            line,
460            "expected five pipe-separated columns",
461        ));
462    };
463
464    if !config.prefixes.iter().any(|known| known == prefix) {
465        return Err(malformed(
466            source_line,
467            source,
468            line,
469            "prefix is not configured",
470        ));
471    }
472
473    if label.is_empty() {
474        return Err(malformed(source_line, source, line, "label is empty"));
475    }
476
477    let (span_label, kind) = split_label_kind(label).ok_or_else(|| {
478        malformed(
479            source_line,
480            source,
481            line,
482            "label must end in :start or :end",
483        )
484    })?;
485
486    let counters = BenchmarkCounters {
487        instructions: parse_counter(instructions, source_line, source, line, "instructions")?,
488        heap_bytes: parse_counter(heap_bytes, source_line, source, line, "heap_bytes")?,
489        memory_bytes: parse_counter(memory_bytes, source_line, source, line, "memory_bytes")?,
490    };
491    let suite = config.suite_derivation.derive_suite(span_label);
492
493    Ok(RawBenchmarkEvent {
494        prefix: prefix.to_string(),
495        suite,
496        span_label: span_label.to_string(),
497        kind,
498        counters,
499        source_line,
500        source,
501    })
502}
503
504fn parse_counter(
505    value: &str,
506    source_line: usize,
507    source: BenchmarkEventSource,
508    line: &str,
509    name: &str,
510) -> Result<u128, MalformedBenchmarkMarker> {
511    if value.is_empty() {
512        return Err(malformed(
513            source_line,
514            source,
515            line,
516            format!("{name} counter is empty"),
517        ));
518    }
519
520    value.parse::<u128>().map_err(|_| {
521        malformed(
522            source_line,
523            source,
524            line,
525            format!("{name} counter is not an unsigned integer"),
526        )
527    })
528}
529
530fn split_label_kind(label: &str) -> Option<(&str, BenchmarkEventKind)> {
531    let start = label.strip_suffix(":start");
532    let end = label.strip_suffix(":end");
533
534    match (start, end) {
535        (Some(span_label), None) if !span_label.is_empty() => {
536            Some((span_label, BenchmarkEventKind::Start))
537        }
538        (None, Some(span_label)) if !span_label.is_empty() => {
539            Some((span_label, BenchmarkEventKind::End))
540        }
541        _ => None,
542    }
543}
544
545fn has_configured_prefix(line: &str, prefixes: &[String]) -> bool {
546    prefixes.iter().any(|prefix| {
547        line.strip_prefix(prefix)
548            .is_some_and(|rest| rest.starts_with('|'))
549    })
550}
551
552fn malformed(
553    source_line: usize,
554    source: BenchmarkEventSource,
555    line: &str,
556    reason: impl Into<String>,
557) -> MalformedBenchmarkMarker {
558    MalformedBenchmarkMarker {
559        source_line,
560        source,
561        line: line.to_string(),
562        reason: reason.into(),
563    }
564}
565
566fn push_paired_span(
567    report: &mut BenchmarkSpanReport,
568    start: &RawBenchmarkEvent,
569    end: &RawBenchmarkEvent,
570) {
571    if let Some(delta) = end.counters.checked_delta(start.counters) {
572        report.spans.push(BenchmarkSpan {
573            suite: start.suite.clone(),
574            span_label: start.span_label.clone(),
575            start_line: start.source_line,
576            end_line: end.source_line,
577            end: end.counters,
578            delta,
579        });
580    } else {
581        report.invalid_spans.push(InvalidBenchmarkSpan {
582            start: start.clone(),
583            end: end.clone(),
584        });
585    }
586}
587
588// Borrow grouping identities until a row or error must own them. Keep the same
589// suite-before-all ordering as the public report scope.
590#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)]
591enum AggregateScope<'a> {
592    Suite(&'a str),
593    All,
594}
595
596impl AggregateScope<'_> {
597    fn into_owned(self) -> BenchmarkScope {
598        match self {
599            Self::Suite(suite) => BenchmarkScope::Suite(suite.to_owned()),
600            Self::All => BenchmarkScope::All,
601        }
602    }
603}
604
605#[derive(Clone, Debug)]
606struct AggregateBuilder {
607    runs: u64,
608    total: BenchmarkCounters,
609    min: BenchmarkCounters,
610    max: BenchmarkCounters,
611    peak_end: BenchmarkCounters,
612}
613
614impl AggregateBuilder {
615    const fn new(span: &BenchmarkSpan) -> Self {
616        Self {
617            runs: 1,
618            total: span.delta,
619            min: span.delta,
620            max: span.delta,
621            peak_end: span.end,
622        }
623    }
624
625    fn push(&mut self, span: &BenchmarkSpan) -> Result<(), BenchmarkCounter> {
626        let runs = self.runs.checked_add(1).ok_or(BenchmarkCounter::Runs)?;
627        let total = self.total.checked_add(span.delta)?;
628        self.runs = runs;
629        self.total = total;
630        self.min.min_assign(span.delta);
631        self.max.max_assign(span.delta);
632        self.peak_end.max_assign(span.end);
633        Ok(())
634    }
635
636    const fn finish(self, scope: BenchmarkScope, span_label: String) -> BenchmarkAggregateRow {
637        BenchmarkAggregateRow {
638            span_label,
639            runs: self.runs,
640            total: self.total,
641            min: self.min,
642            max: self.max,
643            peak_end: self.peak_end,
644            scope,
645        }
646    }
647}
648
649fn add_span_to_aggregate<'a>(
650    rows: &mut BTreeMap<(AggregateScope<'a>, &'a str), AggregateBuilder>,
651    scope: AggregateScope<'a>,
652    span: &'a BenchmarkSpan,
653) -> Result<(), BenchmarkAggregateError> {
654    match rows.entry((scope, span.span_label.as_str())) {
655        Entry::Occupied(mut entry) => entry.get_mut().push(span).map_err(|counter| {
656            let (scope, span_label) = entry.key();
657            BenchmarkAggregateError {
658                span_label: (*span_label).to_owned(),
659                counter,
660                scope: scope.into_owned(),
661            }
662        }),
663        Entry::Vacant(entry) => {
664            entry.insert(AggregateBuilder::new(span));
665            Ok(())
666        }
667    }
668}
669
670#[cfg(test)]
671mod tests {
672    use super::{AggregateBuilder, BenchmarkCounter, BenchmarkCounters, BenchmarkSpan};
673
674    #[test]
675    fn aggregate_run_count_overflow_is_rejected_without_changing_totals() {
676        let span = BenchmarkSpan {
677            suite: "probe".into(),
678            span_label: "probe/run".into(),
679            start_line: 1,
680            end_line: 2,
681            end: BenchmarkCounters::default(),
682            delta: BenchmarkCounters::default(),
683        };
684        let mut builder = AggregateBuilder::new(&span);
685        builder.runs = u64::MAX;
686        assert_eq!(builder.push(&span), Err(BenchmarkCounter::Runs));
687        assert_eq!(builder.runs, u64::MAX);
688        assert_eq!(builder.total, BenchmarkCounters::default());
689    }
690}
ic_testkit/benchmark/mod.rs

ic_testkit/benchmark/
mod.rs
