balter 0.6.0

A load/stress testing framework.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

use crate::controllers::Controller;
use crate::data::SampleSet;
use balter_core::BASE_TPS;
use std::num::NonZeroU32;
#[allow(unused_imports)]
use tracing::{debug, error, info, instrument, trace, warn, Instrument};

const ERROR_RATE_TOLERANCE: f64 = 0.03;
const DEFAULT_SMALL_STEP_SIZE: f64 = 0.5;

pub(crate) struct ErrorRateController {
    base_label: String,
    goal_tps: NonZeroU32,
    error_rate: f64,
    state: State,
}

impl ErrorRateController {
    pub fn new(name: &str, error_rate: f64) -> Self {
        Self {
            base_label: format!("balter_{name}"),
            goal_tps: BASE_TPS,
            error_rate,
            state: State::BigStep,
        }
    }

    fn check_bounds(&self, sample_error_rate: f64) -> Bounds {
        let bounds = (
            self.error_rate - ERROR_RATE_TOLERANCE,
            self.error_rate + ERROR_RATE_TOLERANCE,
        );
        let bounds = (bounds.0.max(0.), bounds.1.min(0.99));

        match sample_error_rate {
            // NOTE: Special case for 0. error rate since that is the inflection point
            x if x == 0. => Bounds::Under,
            x if x >= bounds.0 && x <= bounds.1 => Bounds::At,
            x if x > bounds.1 => Bounds::Over,
            _ => Bounds::Under,
        }
    }
}

impl Controller for ErrorRateController {
    fn initial_tps(&self) -> NonZeroU32 {
        BASE_TPS
    }

    fn limit(&mut self, samples: &SampleSet, stable: bool) -> NonZeroU32 {
        // TODO: Remove panic; this can be a type-safe check
        let sample_error_rate = samples.mean_err();

        let (new_goal_tps, new_state) = match self.check_bounds(sample_error_rate) {
            Bounds::Under => match self.state {
                s @ State::BigStep => {
                    trace!("Under bounds w/ BigStep");
                    (NonZeroU32::new(self.goal_tps.get() * 2).unwrap(), s)
                }
                s @ State::SmallStep(step_ratio) => {
                    trace!("Under bounds w/ SmallStep.");
                    // TODO: Better handling of conversions
                    let step = (self.goal_tps.get() as f64 * step_ratio).max(1.);
                    (
                        NonZeroU32::new(self.goal_tps.get() + step as u32).unwrap(),
                        s,
                    )
                }
                State::Stable => {
                    trace!("Under bounds w/ Stable.");
                    (self.goal_tps, State::SmallStep(DEFAULT_SMALL_STEP_SIZE))
                }
            },
            Bounds::At => {
                match self.state {
                    State::BigStep | State::SmallStep(_) => {
                        trace!("At bounds w/ BigStep|SmallStep.");
                        // TODO: Remove unwraps
                        let samples_tps = samples.mean_tps();
                        (convert_to_nonzerou32(samples_tps).unwrap(), State::Stable)
                    }
                    s @ State::Stable => {
                        trace!("At bounds w/ Stable.");
                        (self.goal_tps, s)
                    }
                }
            }
            Bounds::Over => {
                match self.state {
                    State::BigStep => {
                        trace!("Over bounds w/ BigStep.");
                        // TODO: Remove unwrap
                        (
                            NonZeroU32::new(self.goal_tps.get() / 2).unwrap(),
                            State::SmallStep(DEFAULT_SMALL_STEP_SIZE),
                        )
                    }
                    State::SmallStep(step_ratio) => {
                        trace!("Over bounds w/ SmallStep({step_ratio}).");

                        let rev_goal = (self.goal_tps.get() as f64 / (step_ratio + 1.)).max(1.);
                        (
                            NonZeroU32::new(rev_goal as u32).unwrap(),
                            State::SmallStep(step_ratio / 2.),
                        )
                    }
                    State::Stable => {
                        trace!("Over bounds w/ Stable.");
                        (self.goal_tps, State::SmallStep(DEFAULT_SMALL_STEP_SIZE))
                    }
                }
            }
        };

        if new_goal_tps < self.goal_tps || stable {
            self.goal_tps = new_goal_tps;
            self.state = new_state;
        } else {
            debug!("TPS not stabalized; holding off on increasing Goal TPS");
        }

        if cfg!(feature = "metrics") {
            metrics::gauge!(format!("{}_erc_goal_tps", &self.base_label)).set(self.goal_tps.get());
            metrics::gauge!(format!("{}_erc_state", &self.base_label)).set(match self.state {
                State::BigStep => 2,
                State::SmallStep(_) => 1,
                State::Stable => 0,
            });
        }

        self.goal_tps
    }
}

#[derive(Debug, Clone, Copy)]
enum State {
    BigStep,
    SmallStep(f64),
    Stable,
}

enum Bounds {
    Under,
    At,
    Over,
}

fn convert_to_nonzerou32(val: f64) -> Option<NonZeroU32> {
    let val = val as u32;
    NonZeroU32::new(val)
}