ic_testkit/benchmark/
mod.rs

1//! Parse, pair, aggregate, compare, and report compact benchmark markers.
2//!
3//! The normal pipeline is [`parse_benchmark_events`],
4//! [`pair_benchmark_spans`], [`aggregate_benchmark_spans`], and optionally
5//! [`compare_benchmark_aggregates`]. Native report output lives in
6//! `benchmark::report`. Marker
7//! producers can use [`format_marker`] on the host or
8//! `ic_testkit::performance::Performance` in canister code.
9//!
10//! Totals and run counts remain exact integers. Means use IC Metrics' exact
11//! scaled integer projection, rounded down. Callers choose the scale and handle
12//! projection errors; reports use scale one in the original counter units.
13//! Comparisons retain current and previous means instead of percentage estimates.
14
15use std::collections::{BTreeMap, btree_map::Entry};
16
17use ic_metrics::{MeasurementRatioError, checked_scaled_ratio};
18
19#[cfg(not(target_arch = "wasm32"))]
20pub mod report;
21
22/// Default prefix for benchmark marker lines.
23pub const DEFAULT_PREFIX: &str = "ICTK";
24
25/// Structural identity of one suite or the cross-suite aggregate.
26#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
27pub enum BenchmarkScope {
28    /// One caller-authored suite, including a suite literally named `ALL`.
29    Suite(String),
30    /// Aggregate across every suite.
31    All,
32}
33
34#[derive(Clone, Debug, Eq, PartialEq)]
35pub struct BenchmarkParserConfig {
36    pub prefixes: Vec<String>,
37    pub suite_derivation: SuiteDerivation,
38    /// When enabled, non-empty lines without a configured marker prefix are
39    /// reported as malformed markers instead of ignored log noise.
40    pub strict: bool,
41}
42
43impl Default for BenchmarkParserConfig {
44    fn default() -> Self {
45        Self {
46            prefixes: vec![DEFAULT_PREFIX.to_string()],
47            suite_derivation: SuiteDerivation::FirstPathSegment,
48            strict: false,
49        }
50    }
51}
52
53#[derive(Clone, Debug, Eq, PartialEq)]
54pub enum SuiteDerivation {
55    FirstPathSegment,
56    Fixed(String),
57}
58
59impl SuiteDerivation {
60    #[must_use]
61    pub fn derive_suite(&self, span_label: &str) -> String {
62        match self {
63            Self::FirstPathSegment => span_label
64                .split('/')
65                .next()
66                .filter(|part| !part.is_empty())
67                .unwrap_or(span_label)
68                .to_string(),
69            Self::Fixed(suite) => suite.clone(),
70        }
71    }
72}
73
74#[derive(Clone, Copy, Debug, Eq, PartialEq)]
75pub enum BenchmarkEventKind {
76    Start,
77    End,
78}
79
80#[derive(Clone, Copy, Debug, Eq, PartialEq)]
81pub enum BenchmarkEventSource {
82    Unknown,
83    Stdout,
84    Stderr,
85    FetchedLog,
86}
87
88impl BenchmarkEventSource {
89    #[must_use]
90    pub const fn as_str(self) -> &'static str {
91        match self {
92            Self::Unknown => "unknown",
93            Self::Stdout => "stdout",
94            Self::Stderr => "stderr",
95            Self::FetchedLog => "fetched_log",
96        }
97    }
98}
99
100#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
101pub struct BenchmarkCounters {
102    pub instructions: u128,
103    pub heap_bytes: u128,
104    pub memory_bytes: u128,
105    pub total_allocation: u128,
106}
107
108impl BenchmarkCounters {
109    fn checked_delta(self, start: Self) -> Option<Self> {
110        Some(Self {
111            instructions: self.instructions.checked_sub(start.instructions)?,
112            heap_bytes: self.heap_bytes.checked_sub(start.heap_bytes)?,
113            memory_bytes: self.memory_bytes.checked_sub(start.memory_bytes)?,
114            total_allocation: self.total_allocation.checked_sub(start.total_allocation)?,
115        })
116    }
117
118    fn checked_add(self, other: Self) -> Result<Self, BenchmarkCounter> {
119        Ok(Self {
120            instructions: self
121                .instructions
122                .checked_add(other.instructions)
123                .ok_or(BenchmarkCounter::Instructions)?,
124            heap_bytes: self
125                .heap_bytes
126                .checked_add(other.heap_bytes)
127                .ok_or(BenchmarkCounter::HeapBytes)?,
128            memory_bytes: self
129                .memory_bytes
130                .checked_add(other.memory_bytes)
131                .ok_or(BenchmarkCounter::MemoryBytes)?,
132            total_allocation: self
133                .total_allocation
134                .checked_add(other.total_allocation)
135                .ok_or(BenchmarkCounter::TotalAllocation)?,
136        })
137    }
138
139    fn min_assign(&mut self, other: Self) {
140        self.instructions = self.instructions.min(other.instructions);
141        self.heap_bytes = self.heap_bytes.min(other.heap_bytes);
142        self.memory_bytes = self.memory_bytes.min(other.memory_bytes);
143        self.total_allocation = self.total_allocation.min(other.total_allocation);
144    }
145
146    fn max_assign(&mut self, other: Self) {
147        self.instructions = self.instructions.max(other.instructions);
148        self.heap_bytes = self.heap_bytes.max(other.heap_bytes);
149        self.memory_bytes = self.memory_bytes.max(other.memory_bytes);
150        self.total_allocation = self.total_allocation.max(other.total_allocation);
151    }
152}
153
154#[derive(Clone, Debug, Eq, PartialEq)]
155pub struct RawBenchmarkEvent {
156    pub prefix: String,
157    pub suite: String,
158    pub span_label: String,
159    pub kind: BenchmarkEventKind,
160    pub counters: BenchmarkCounters,
161    pub source_line: usize,
162    pub source: BenchmarkEventSource,
163}
164
165#[derive(Clone, Debug, Eq, PartialEq)]
166pub struct MalformedBenchmarkMarker {
167    pub source_line: usize,
168    pub source: BenchmarkEventSource,
169    pub line: String,
170    pub reason: String,
171}
172
173#[derive(Clone, Debug, Default, Eq, PartialEq)]
174pub struct BenchmarkParseReport {
175    pub events: Vec<RawBenchmarkEvent>,
176    pub malformed_markers: Vec<MalformedBenchmarkMarker>,
177    pub ignored_line_count: usize,
178}
179
180#[derive(Clone, Debug, Eq, PartialEq)]
181pub struct BenchmarkSpan {
182    pub suite: String,
183    pub span_label: String,
184    pub start_line: usize,
185    pub end_line: usize,
186    pub end: BenchmarkCounters,
187    pub delta: BenchmarkCounters,
188}
189
190#[derive(Clone, Debug, Eq, PartialEq)]
191pub struct InvalidBenchmarkSpan {
192    pub start: RawBenchmarkEvent,
193    pub end: RawBenchmarkEvent,
194}
195
196#[derive(Clone, Debug, Default, Eq, PartialEq)]
197pub struct BenchmarkSpanReport {
198    pub spans: Vec<BenchmarkSpan>,
199    pub unpaired_markers: Vec<RawBenchmarkEvent>,
200    pub invalid_spans: Vec<InvalidBenchmarkSpan>,
201}
202
203#[derive(Clone, Debug, Eq, PartialEq)]
204pub struct BenchmarkAggregateRow {
205    pub span_label: String,
206    pub runs: u64,
207    pub total: BenchmarkCounters,
208    pub min: BenchmarkCounters,
209    pub max: BenchmarkCounters,
210    pub peak_end: BenchmarkCounters,
211    /// Suite identity or the cross-suite aggregate.
212    pub scope: BenchmarkScope,
213}
214
215/// Floor-rounded scaled means of exact aggregate totals and run counts.
216///
217/// Fields are `floor(total * scale / runs)`; choose a positive scale to
218/// interpret them in the original counter unit. Scale zero projects zero.
219#[derive(Clone, Copy, Debug, Eq, PartialEq)]
220pub struct BenchmarkAverages {
221    pub scale: u64,
222    pub instructions: u128,
223    pub heap_bytes: u128,
224    pub memory_bytes: u128,
225    pub total_allocation: u128,
226}
227
228/// Counter whose aggregate cannot be represented.
229#[derive(Clone, Copy, Debug, Eq, PartialEq)]
230pub enum BenchmarkCounter {
231    Runs,
232    Instructions,
233    HeapBytes,
234    MemoryBytes,
235    TotalAllocation,
236}
237
238/// Overflow rejects the complete aggregation, without returning partial totals.
239#[derive(Clone, Debug, Eq, PartialEq)]
240pub struct BenchmarkAggregateError {
241    pub span_label: String,
242    pub counter: BenchmarkCounter,
243    /// Suite identity or the cross-suite aggregate.
244    pub scope: BenchmarkScope,
245}
246
247impl std::fmt::Display for BenchmarkAggregateError {
248    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
249        write!(
250            formatter,
251            "benchmark aggregate overflow: scope={:?}, span={}, counter={:?}",
252            self.scope, self.span_label, self.counter
253        )
254    }
255}
256
257impl std::error::Error for BenchmarkAggregateError {}
258
259/// Run count and scaled means for one present comparison side.
260#[derive(Clone, Copy, Debug, Eq, PartialEq)]
261pub struct BenchmarkMeasurement {
262    pub runs: u64,
263    pub average: BenchmarkAverages,
264}
265
266/// Current and previous measurements, preserving missing rows and measured zero.
267#[derive(Clone, Debug, Eq, PartialEq)]
268pub struct BenchmarkComparisonRow {
269    pub span_label: String,
270    pub current: Option<BenchmarkMeasurement>,
271    pub previous: Option<BenchmarkMeasurement>,
272    /// Suite identity or the cross-suite aggregate.
273    pub scope: BenchmarkScope,
274}
275
276impl BenchmarkAggregateRow {
277    /// Project exact totals at the caller's scale, rounding down.
278    /// Zero runs and unrepresentable scaled results return IC Metrics' typed error.
279    pub fn average(&self, scale: u64) -> Result<BenchmarkAverages, MeasurementRatioError> {
280        Ok(BenchmarkAverages {
281            scale,
282            instructions: checked_scaled_ratio(self.total.instructions, self.runs, scale)?,
283            heap_bytes: checked_scaled_ratio(self.total.heap_bytes, self.runs, scale)?,
284            memory_bytes: checked_scaled_ratio(self.total.memory_bytes, self.runs, scale)?,
285            total_allocation: checked_scaled_ratio(self.total.total_allocation, self.runs, scale)?,
286        })
287    }
288}
289
290#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
291pub struct BenchmarkRunMetadata {
292    pub timestamp: String,
293    pub run_directory_name: String,
294    pub run_index: u32,
295    pub git_commit_hash: Option<String>,
296    pub git_commit_short_hash: Option<String>,
297    pub ic_testkit_version: String,
298    pub pocket_ic_version: String,
299    pub rustc_version: String,
300    pub benchmark_command: Option<String>,
301    pub selected_previous_run: Option<String>,
302}
303
304#[derive(Clone, Debug, Eq, PartialEq)]
305pub struct BenchmarkRunReport {
306    pub parse: BenchmarkParseReport,
307    pub spans: BenchmarkSpanReport,
308    pub aggregates: Vec<BenchmarkAggregateRow>,
309    pub comparison: Option<Vec<BenchmarkComparisonRow>>,
310    pub metadata: BenchmarkRunMetadata,
311}
312
313#[must_use]
314pub fn format_marker(prefix: &str, label: &str, counters: BenchmarkCounters) -> String {
315    format!(
316        "{}|{}|{}|{}|{}|{}",
317        prefix,
318        label,
319        counters.instructions,
320        counters.heap_bytes,
321        counters.memory_bytes,
322        counters.total_allocation
323    )
324}
325
326#[must_use]
327pub fn parse_benchmark_events(input: &str, config: &BenchmarkParserConfig) -> BenchmarkParseReport {
328    parse_benchmark_events_from_source(input, config, BenchmarkEventSource::Unknown)
329}
330
331#[must_use]
332pub fn parse_benchmark_events_from_source(
333    input: &str,
334    config: &BenchmarkParserConfig,
335    source: BenchmarkEventSource,
336) -> BenchmarkParseReport {
337    let mut report = BenchmarkParseReport::default();
338    append_benchmark_events(&mut report, input, config, source);
339    report
340}
341
342fn append_benchmark_events(
343    report: &mut BenchmarkParseReport,
344    input: &str,
345    config: &BenchmarkParserConfig,
346    source: BenchmarkEventSource,
347) {
348    for (index, line) in input.lines().enumerate() {
349        let source_line = index + 1;
350        if !has_configured_prefix(line, &config.prefixes) {
351            if config.strict && !line.trim().is_empty() {
352                report.malformed_markers.push(malformed(
353                    source_line,
354                    source,
355                    line,
356                    "line does not use a configured marker prefix",
357                ));
358            } else {
359                report.ignored_line_count += 1;
360            }
361            continue;
362        }
363
364        match parse_marker_line(line, source_line, source, config) {
365            Ok(event) => report.events.push(event),
366            Err(marker) => report.malformed_markers.push(marker),
367        }
368    }
369}
370
371/// Parse separately captured stdout and stderr.
372///
373/// Separate streams do not carry global ordering, so events are returned in
374/// stdout-then-stderr order. If a benchmark span can start on one stream and end
375/// on the other, capture combined process output and use [`parse_benchmark_events`].
376#[must_use]
377pub fn parse_benchmark_events_from_captured_output(
378    stdout: &str,
379    stderr: &str,
380    config: &BenchmarkParserConfig,
381) -> BenchmarkParseReport {
382    let mut report = BenchmarkParseReport::default();
383    append_benchmark_events(&mut report, stdout, config, BenchmarkEventSource::Stdout);
384    append_benchmark_events(&mut report, stderr, config, BenchmarkEventSource::Stderr);
385    report
386}
387
388#[must_use]
389pub fn pair_benchmark_spans(events: &[RawBenchmarkEvent]) -> BenchmarkSpanReport {
390    let mut report = BenchmarkSpanReport::default();
391    let mut open_starts: BTreeMap<(&str, &str), Vec<&RawBenchmarkEvent>> = BTreeMap::new();
392
393    for event in events {
394        let key = (event.suite.as_str(), event.span_label.as_str());
395        match event.kind {
396            BenchmarkEventKind::Start => open_starts.entry(key).or_default().push(event),
397            BenchmarkEventKind::End => match open_starts.entry(key) {
398                Entry::Occupied(mut entry) => {
399                    let start = entry
400                        .get_mut()
401                        .pop()
402                        .expect("open marker stacks must remain nonempty");
403                    if entry.get().is_empty() {
404                        entry.remove();
405                    }
406                    push_paired_span(&mut report, start, event);
407                }
408                Entry::Vacant(_) => report.unpaired_markers.push(event.clone()),
409            },
410        }
411    }
412
413    for starts in open_starts.into_values() {
414        for event in starts {
415            report.unpaired_markers.push(event.clone());
416        }
417    }
418
419    report
420}
421
422/// Aggregate spans by named suite and across all suites.
423///
424/// Returns an error if any counter total or run count overflows. No partial
425/// report is returned, and totals are never wrapped or saturated.
426pub fn aggregate_benchmark_spans(
427    spans: &[BenchmarkSpan],
428) -> Result<Vec<BenchmarkAggregateRow>, BenchmarkAggregateError> {
429    let mut rows: BTreeMap<(BenchmarkScope, &str), AggregateBuilder> = BTreeMap::new();
430
431    for span in spans {
432        add_span_to_aggregate(&mut rows, BenchmarkScope::Suite(span.suite.clone()), span)?;
433        add_span_to_aggregate(&mut rows, BenchmarkScope::All, span)?;
434    }
435
436    Ok(rows
437        .into_iter()
438        .map(|((scope, span_label), builder)| builder.finish(scope, span_label.to_owned()))
439        .collect())
440}
441
442/// Pair current and previous means at the same caller-selected scale.
443/// Returns a typed error if either side cannot supply the selected projection.
444pub fn compare_benchmark_aggregates(
445    current: &[BenchmarkAggregateRow],
446    previous: &[BenchmarkAggregateRow],
447    scale: u64,
448) -> Result<Vec<BenchmarkComparisonRow>, MeasurementRatioError> {
449    let mut rows = BTreeMap::new();
450    for row in current {
451        rows.insert((&row.scope, row.span_label.as_str()), (Some(row), None));
452    }
453    for row in previous {
454        rows.entry((&row.scope, row.span_label.as_str()))
455            .or_insert((None, None))
456            .1 = Some(row);
457    }
458    let measurement = |row: &BenchmarkAggregateRow| -> Result<_, MeasurementRatioError> {
459        Ok(BenchmarkMeasurement {
460            runs: row.runs,
461            average: row.average(scale)?,
462        })
463    };
464
465    rows.into_iter()
466        .map(|((scope, span_label), (current_row, previous_row))| {
467            Ok(BenchmarkComparisonRow {
468                span_label: span_label.to_owned(),
469                current: current_row.map(measurement).transpose()?,
470                previous: previous_row.map(measurement).transpose()?,
471                scope: scope.clone(),
472            })
473        })
474        .collect()
475}
476
477fn parse_marker_line(
478    line: &str,
479    source_line: usize,
480    source: BenchmarkEventSource,
481    config: &BenchmarkParserConfig,
482) -> Result<RawBenchmarkEvent, MalformedBenchmarkMarker> {
483    let mut parts = line.split('|');
484    // Six fields and an absent seventh establish the complete marker shape
485    // without collecting an arbitrary number of columns from malformed logs.
486    let columns: [Option<&str>; 7] = std::array::from_fn(|_| parts.next());
487    let [
488        Some(prefix),
489        Some(label),
490        Some(instructions),
491        Some(heap_bytes),
492        Some(memory_bytes),
493        Some(total_allocation),
494        None,
495    ] = columns
496    else {
497        return Err(malformed(
498            source_line,
499            source,
500            line,
501            "expected six pipe-separated columns",
502        ));
503    };
504
505    if !config.prefixes.iter().any(|known| known == prefix) {
506        return Err(malformed(
507            source_line,
508            source,
509            line,
510            "prefix is not configured",
511        ));
512    }
513
514    if label.is_empty() {
515        return Err(malformed(source_line, source, line, "label is empty"));
516    }
517
518    let (span_label, kind) = split_label_kind(label).ok_or_else(|| {
519        malformed(
520            source_line,
521            source,
522            line,
523            "label must end in :start or :end",
524        )
525    })?;
526
527    let counters = BenchmarkCounters {
528        instructions: parse_counter(instructions, source_line, source, line, "instructions")?,
529        heap_bytes: parse_counter(heap_bytes, source_line, source, line, "heap_bytes")?,
530        memory_bytes: parse_counter(memory_bytes, source_line, source, line, "memory_bytes")?,
531        total_allocation: parse_counter(
532            total_allocation,
533            source_line,
534            source,
535            line,
536            "total_allocation",
537        )?,
538    };
539    let suite = config.suite_derivation.derive_suite(span_label);
540
541    Ok(RawBenchmarkEvent {
542        prefix: prefix.to_string(),
543        suite,
544        span_label: span_label.to_string(),
545        kind,
546        counters,
547        source_line,
548        source,
549    })
550}
551
552fn parse_counter(
553    value: &str,
554    source_line: usize,
555    source: BenchmarkEventSource,
556    line: &str,
557    name: &str,
558) -> Result<u128, MalformedBenchmarkMarker> {
559    if value.is_empty() {
560        return Err(malformed(
561            source_line,
562            source,
563            line,
564            format!("{name} counter is empty"),
565        ));
566    }
567
568    value.parse::<u128>().map_err(|_| {
569        malformed(
570            source_line,
571            source,
572            line,
573            format!("{name} counter is not an unsigned integer"),
574        )
575    })
576}
577
578fn split_label_kind(label: &str) -> Option<(&str, BenchmarkEventKind)> {
579    let start = label.strip_suffix(":start");
580    let end = label.strip_suffix(":end");
581
582    match (start, end) {
583        (Some(span_label), None) if !span_label.is_empty() => {
584            Some((span_label, BenchmarkEventKind::Start))
585        }
586        (None, Some(span_label)) if !span_label.is_empty() => {
587            Some((span_label, BenchmarkEventKind::End))
588        }
589        _ => None,
590    }
591}
592
593fn has_configured_prefix(line: &str, prefixes: &[String]) -> bool {
594    prefixes.iter().any(|prefix| {
595        line.strip_prefix(prefix)
596            .is_some_and(|rest| rest.starts_with('|'))
597    })
598}
599
600fn malformed(
601    source_line: usize,
602    source: BenchmarkEventSource,
603    line: &str,
604    reason: impl Into<String>,
605) -> MalformedBenchmarkMarker {
606    MalformedBenchmarkMarker {
607        source_line,
608        source,
609        line: line.to_string(),
610        reason: reason.into(),
611    }
612}
613
614fn push_paired_span(
615    report: &mut BenchmarkSpanReport,
616    start: &RawBenchmarkEvent,
617    end: &RawBenchmarkEvent,
618) {
619    if let Some(delta) = end.counters.checked_delta(start.counters) {
620        report.spans.push(BenchmarkSpan {
621            suite: start.suite.clone(),
622            span_label: start.span_label.clone(),
623            start_line: start.source_line,
624            end_line: end.source_line,
625            end: end.counters,
626            delta,
627        });
628    } else {
629        report.invalid_spans.push(InvalidBenchmarkSpan {
630            start: start.clone(),
631            end: end.clone(),
632        });
633    }
634}
635
636#[derive(Clone, Debug)]
637struct AggregateBuilder {
638    runs: u64,
639    total: BenchmarkCounters,
640    min: BenchmarkCounters,
641    max: BenchmarkCounters,
642    peak_end: BenchmarkCounters,
643}
644
645impl AggregateBuilder {
646    const fn new(span: &BenchmarkSpan) -> Self {
647        Self {
648            runs: 1,
649            total: span.delta,
650            min: span.delta,
651            max: span.delta,
652            peak_end: span.end,
653        }
654    }
655
656    fn push(&mut self, span: &BenchmarkSpan) -> Result<(), BenchmarkCounter> {
657        let runs = self.runs.checked_add(1).ok_or(BenchmarkCounter::Runs)?;
658        let total = self.total.checked_add(span.delta)?;
659        self.runs = runs;
660        self.total = total;
661        self.min.min_assign(span.delta);
662        self.max.max_assign(span.delta);
663        self.peak_end.max_assign(span.end);
664        Ok(())
665    }
666
667    const fn finish(self, scope: BenchmarkScope, span_label: String) -> BenchmarkAggregateRow {
668        BenchmarkAggregateRow {
669            span_label,
670            runs: self.runs,
671            total: self.total,
672            min: self.min,
673            max: self.max,
674            peak_end: self.peak_end,
675            scope,
676        }
677    }
678}
679
680fn add_span_to_aggregate<'a>(
681    rows: &mut BTreeMap<(BenchmarkScope, &'a str), AggregateBuilder>,
682    scope: BenchmarkScope,
683    span: &'a BenchmarkSpan,
684) -> Result<(), BenchmarkAggregateError> {
685    match rows.entry((scope, span.span_label.as_str())) {
686        Entry::Occupied(mut entry) => entry.get_mut().push(span).map_err(|counter| {
687            let (scope, span_label) = entry.key();
688            BenchmarkAggregateError {
689                span_label: (*span_label).to_owned(),
690                counter,
691                scope: scope.clone(),
692            }
693        }),
694        Entry::Vacant(entry) => {
695            entry.insert(AggregateBuilder::new(span));
696            Ok(())
697        }
698    }
699}
700
701#[cfg(test)]
702mod tests {
703    use super::{AggregateBuilder, BenchmarkCounter, BenchmarkCounters, BenchmarkSpan};
704
705    #[test]
706    fn aggregate_run_count_overflow_is_rejected_without_changing_totals() {
707        let span = BenchmarkSpan {
708            suite: "probe".into(),
709            span_label: "probe/run".into(),
710            start_line: 1,
711            end_line: 2,
712            end: BenchmarkCounters::default(),
713            delta: BenchmarkCounters::default(),
714        };
715        let mut builder = AggregateBuilder::new(&span);
716        builder.runs = u64::MAX;
717        assert_eq!(builder.push(&span), Err(BenchmarkCounter::Runs));
718        assert_eq!(builder.runs, u64::MAX);
719        assert_eq!(builder.total, BenchmarkCounters::default());
720    }
721}
ic_testkit/benchmark/mod.rs

ic_testkit/benchmark/
mod.rs
