Skip to main content

memseek_test/
runner.rs

1use std::{
2    borrow::Cow,
3    marker::PhantomData,
4    time::{Duration, Instant},
5};
6
7#[cfg(feature = "tui")]
8use crate::progress::Progress;
9use crate::{
10    error::ScenarioError,
11    recorder::Recorder,
12    scenario::{Scenario, SetupMode, TeardownMode},
13    shutdown::Shutdown,
14};
15
16#[derive(Clone, Copy, Debug)]
17pub enum RunMode {
18    Times(u64),
19    Duration(Duration),
20}
21
22/// 超时退避配置:请求超时后等待 `initial` 起,按 `factor` 指数增长,封顶 `max`;
23/// 任意非超时结果(成功或普通失败)复位到 `initial`。
24///
25/// 目的:被测系统过载时自动减速,避免超时风暴雪上加霜;恢复后自动回到满速。
26#[derive(Clone, Copy, Debug)]
27pub struct BackoffConfig {
28    /// 首次退避等待(默认 100ms)
29    pub initial: Duration,
30    /// 退避最大值(默认 5s)
31    pub max: Duration,
32    /// 增长倍数(默认 2.0)
33    pub factor: f64,
34}
35
36impl Default for BackoffConfig {
37    fn default() -> Self {
38        Self {
39            initial: Duration::from_millis(100),
40            max: Duration::from_secs(3),
41            factor: 2.0,
42        }
43    }
44}
45
46impl BackoffConfig {
47    pub const fn new(initial: Duration, max: Duration, factor: f64) -> Self {
48        Self {
49            initial,
50            max,
51            factor,
52        }
53    }
54}
55
56/// 并发任务身份,由 Manager 分片时分配,使用者只读。
57///
58/// `index` 为该任务编号(范围 `0..total`);每个并发任务拥有唯一的
59/// `index`,可用它做账号等参数化(`format!("loadtest_{}", index + 1)`)。
60/// `round` 为全局运行编号:每次执行(含 `setup`)从共享计数器取号,
61/// 同一轮内的 `setup`/`run`/`validate` 携带同一编号,
62/// 可用于全局唯一命名(`format!("data_{}", round)`)或日志定位。
63#[derive(Clone, Copy, Debug)]
64pub struct TaskIndex {
65    /// 当前任务编号,范围 `0..total`
66    pub index: usize,
67    /// 并发任务总数
68    pub total: usize,
69    /// 全局运行编号(本轮编号,每轮递增)
70    pub round: usize,
71}
72
73impl TaskIndex {
74    pub const fn new(index: usize, total: usize, round: usize) -> Self {
75        Self {
76            index,
77            total,
78            round,
79        }
80    }
81}
82
83pub struct RunnerConfig {
84    pub mode: RunMode,
85    /// 任务编号(Manager 分片时分配)
86    pub task_index: usize,
87    /// 并发任务总数
88    pub task_total: usize,
89    /// 优雅关闭信号;触发后当前轮次完成即停止(不再启动新轮次)
90    pub shutdown: Option<Shutdown>,
91    /// 超时退避;`None` 表示超时后立即进入下一轮(默认)
92    pub backoff: Option<BackoffConfig>,
93    /// 实时进度上报句柄(feature `tui`,由 Manager 注入);`None` 表示不上报
94    #[cfg(feature = "tui")]
95    pub progress: Option<Progress>,
96}
97
98pub struct ScenarioRunner<S> {
99    config: RunnerConfig,
100    _marker: PhantomData<S>,
101}
102
103/// setup 执行结果。
104enum SetupDone<S> {
105    /// setup 成功(已复位退避)
106    Ready(S),
107    /// 普通失败(已记录),任务应中止
108    Failed,
109    /// 超时退避等待中被停止信号打断,任务应中止
110    Cancelled,
111}
112
113/// 轮次预算:`Times` 按次数消耗,`Duration` 按墙钟消耗。
114///
115/// 两种执行模式共用同一套轮次循环骨架(取消检查 → 轮级 setup →
116/// 轮体 run/validate/teardown → 退避),预算只决定"是否还有下一轮"。
117#[derive(Clone, Copy, Debug)]
118enum RoundBudget {
119    Times {
120        left: u64,
121    },
122    /// 剩余时长,每轮按该轮实际耗时(含 setup 与 teardown)扣减
123    Duration {
124        remaining: Duration,
125    },
126}
127
128impl RoundBudget {
129    const fn new(mode: RunMode) -> Self {
130        match mode {
131            RunMode::Times(times) => Self::Times { left: times },
132            RunMode::Duration(duration) => Self::Duration {
133                remaining: duration,
134            },
135        }
136    }
137
138    /// 是否还有预算执行下一轮
139    fn has_next(&self) -> bool {
140        match self {
141            Self::Times { left } => *left > 0,
142            Self::Duration { remaining } => !remaining.is_zero(),
143        }
144    }
145
146    /// 记一轮的消耗
147    fn spend(&mut self, elapsed: Duration) {
148        match self {
149            Self::Times { left } => *left = left.saturating_sub(1),
150            Self::Duration { remaining } => *remaining = remaining.saturating_sub(elapsed),
151        }
152    }
153}
154
155impl<S> ScenarioRunner<S>
156where
157    S: Scenario,
158    S::Error: ScenarioError,
159{
160    pub fn new(config: RunnerConfig) -> Self {
161        Self {
162            config,
163            _marker: PhantomData,
164        }
165    }
166
167    pub async fn run(&self, ctx: &S::Ctx) -> Recorder {
168        let mut recorder = Recorder::from_config(&self.config);
169
170        // 优雅关闭:信号到达时已完成当前轮次,不再启动新轮次。
171        // 各分支在 setup 后/每轮开始前检查。
172        // 退避状态在 setup 与各轮之间共享:任意非超时结果(系统恢复)后复位。
173        let mut backoff_wait = Self::backoff_initial(self.config.backoff);
174
175        match S::SETUP_MODE {
176            SetupMode::Task => {
177                if self.is_cancelled() {
178                    recorder.record_interrupted();
179                    return recorder;
180                }
181                // setup 任务级一次(round 无轮次含义,固定 0),产出供各轮复用
182                let task = TaskIndex::new(self.config.task_index, self.config.task_total, 0);
183                #[cfg(feature = "tui")]
184                recorder.begin_setup();
185                let setup = match self
186                    .setup_retrying(ctx, &task, &mut recorder, &mut backoff_wait)
187                    .await
188                {
189                    SetupDone::Ready(setup) => setup,
190                    SetupDone::Failed => return recorder,
191                    SetupDone::Cancelled => {
192                        recorder.record_interrupted();
193                        return recorder;
194                    }
195                };
196                #[cfg(feature = "tui")]
197                recorder.end_setup();
198                // setup 期间收到信号:等 setup 完成后直接退出,不进入 run 循环
199                if self.is_cancelled() {
200                    recorder.record_interrupted();
201                } else {
202                    self.drive_rounds(ctx, &setup, &mut recorder, &mut backoff_wait)
203                        .await;
204                }
205                // 任务级收尾:finally 语义(取消后也执行);setup 成功过才会走到这里
206                if S::TEARDOWN_MODE == TeardownMode::Task {
207                    Self::teardown(ctx, &task, &setup, None, &mut recorder).await;
208                }
209            }
210            SetupMode::Round => {
211                let mut budget = RoundBudget::new(self.config.mode);
212                let mut round = 0usize;
213                // 轮级 setup 的产出仅本轮有效;任务级收尾(TEARDOWN_MODE = Task)
214                // 需要一个 setup 借用,这里保留最近一次成功的产出供其使用。
215                let mut last: Option<(TaskIndex, S::Setup)> = None;
216
217                while budget.has_next() {
218                    if self.is_cancelled() {
219                        recorder.record_interrupted();
220                        break;
221                    }
222                    let start = Instant::now();
223                    let task =
224                        TaskIndex::new(self.config.task_index, self.config.task_total, round);
225                    round += 1;
226
227                    let setup = match self
228                        .setup_retrying(ctx, &task, &mut recorder, &mut backoff_wait)
229                        .await
230                    {
231                        SetupDone::Ready(setup) => setup,
232                        // 普通失败:中止任务(不再启动新轮次)
233                        SetupDone::Failed => break,
234                        SetupDone::Cancelled => {
235                            recorder.record_interrupted();
236                            break;
237                        }
238                    };
239
240                    let entry = (task, setup);
241                    let stopped = self
242                        .round(ctx, &entry.0, &entry.1, &mut recorder, &mut backoff_wait)
243                        .await;
244                    budget.spend(start.elapsed());
245                    last = Some(entry);
246                    if stopped {
247                        recorder.record_interrupted();
248                        break;
249                    }
250                }
251
252                if S::TEARDOWN_MODE == TeardownMode::Task
253                    && let Some((task, setup)) = last.as_ref()
254                {
255                    Self::teardown(ctx, task, setup, None, &mut recorder).await;
256                }
257            }
258        }
259
260        recorder
261    }
262
263    /// `SetupMode::Task` 下的轮次循环:每轮复用同一份 setup 产出。
264    async fn drive_rounds(
265        &self,
266        ctx: &S::Ctx,
267        setup: &S::Setup,
268        recorder: &mut Recorder,
269        wait: &mut Duration,
270    ) {
271        let mut budget = RoundBudget::new(self.config.mode);
272        let mut round = 0usize;
273
274        while budget.has_next() {
275            if self.is_cancelled() {
276                recorder.record_interrupted();
277                return;
278            }
279            let task = TaskIndex::new(self.config.task_index, self.config.task_total, round);
280            round += 1;
281
282            let start = Instant::now();
283            if self.round(ctx, &task, setup, recorder, wait).await {
284                recorder.record_interrupted();
285                return;
286            }
287            budget.spend(start.elapsed());
288        }
289    }
290
291    /// 退避初始值:未配置退避时用零占位(不产生等待)。
292    fn backoff_initial(backoff: Option<BackoffConfig>) -> Duration {
293        backoff.map(|b| b.initial).unwrap_or(Duration::ZERO)
294    }
295
296    /// 执行 setup,失败时处理同轮体的退避语义:
297    /// 超时 → 退避后重试(可被停止信号打断);普通失败或未配置退避时中止。
298    /// 每次尝试的耗时计入 setup 统计。
299    async fn setup_retrying(
300        &self,
301        ctx: &S::Ctx,
302        task: &TaskIndex,
303        recorder: &mut Recorder,
304        wait: &mut Duration,
305    ) -> SetupDone<S::Setup> {
306        let Some(cfg) = self.config.backoff else {
307            // 未配置退避:保持旧语义,失败即中止
308            let start = Instant::now();
309            let ret = S::setup(ctx, task).await;
310            recorder.record_setup_duration(start.elapsed());
311            return match ret {
312                Ok(setup) => SetupDone::Ready(setup),
313                Err(err) => {
314                    recorder.record_result(&Err::<(), S::Error>(err));
315                    SetupDone::Failed
316                }
317            };
318        };
319        loop {
320            let start = Instant::now();
321            let ret = S::setup(ctx, task).await;
322            recorder.record_setup_duration(start.elapsed());
323            match ret {
324                Ok(setup) => {
325                    if *wait != cfg.initial {
326                        *wait = cfg.initial;
327                    }
328                    return SetupDone::Ready(setup);
329                }
330                Err(err) => {
331                    let timed_out = err.is_timeout();
332                    recorder.record_result(&Err::<(), S::Error>(err));
333                    if !timed_out {
334                        return SetupDone::Failed;
335                    }
336                    if self.wait_or_cancel(*wait).await {
337                        return SetupDone::Cancelled;
338                    }
339                    // 指数增长,封顶 max,不低于 initial
340                    let next = wait.as_secs_f64() * cfg.factor;
341                    *wait = Duration::from_secs_f64(next.min(cfg.max.as_secs_f64()));
342                    *wait = (*wait).max(cfg.initial);
343                }
344            }
345        }
346    }
347
348    /// 执行一轮:`run → validate(仅成功时) → teardown(finally)`。
349    /// 返回 `true` 表示本轮超时后的退避等待被停止信号打断。
350    ///
351    /// 退避只由 `run` 的超时驱动:`teardown` 的耗时与失败都不参与退避。
352    async fn round(
353        &self,
354        ctx: &S::Ctx,
355        task: &TaskIndex,
356        setup: &S::Setup,
357        recorder: &mut Recorder,
358        wait: &mut Duration,
359    ) -> bool {
360        let result = Self::once(ctx, task, setup, recorder).await;
361        let timed_out = result.as_ref().is_err_and(|e| e.is_timeout());
362        // run 失败已记录,不再进入 validate
363        if let Ok(output) = &result {
364            Self::validate(ctx, task, setup, output, recorder).await;
365        }
366        // 轮级收尾:finally 语义——run/validate 无论成败都会执行
367        if S::TEARDOWN_MODE == TeardownMode::Round {
368            Self::teardown(ctx, task, setup, Some(result.as_ref()), recorder).await;
369        }
370
371        let Some(cfg) = self.config.backoff else {
372            return false;
373        };
374        if timed_out {
375            if self.wait_or_cancel(*wait).await {
376                return true;
377            }
378            // 指数增长,封顶 max,不低于 initial
379            let next = wait.as_secs_f64() * cfg.factor;
380            *wait = Duration::from_secs_f64(next.min(cfg.max.as_secs_f64()));
381            *wait = (*wait).max(cfg.initial);
382        } else if *wait != cfg.initial {
383            *wait = cfg.initial;
384        }
385        false
386    }
387
388    /// 退避等待;绑定停止信号时,信号到达立即返回 `true`。
389    async fn wait_or_cancel(&self, duration: Duration) -> bool {
390        match self.config.shutdown.as_ref() {
391            Some(s) => tokio::select! {
392                _ = tokio::time::sleep(duration) => false,
393                _ = s.wait_cancelled() => true,
394            },
395            None => {
396                tokio::time::sleep(duration).await;
397                false
398            }
399        }
400    }
401
402    /// 是否已收到优雅关闭信号。
403    fn is_cancelled(&self) -> bool {
404        self.config
405            .shutdown
406            .as_ref()
407            .is_some_and(|s| s.is_cancelled())
408    }
409
410    /// 执行一次 `run` 并记录耗时/结果,返回本轮产出。
411    /// `Err` 已在记录时计为失败,调用方不再进入 validate。
412    async fn once(
413        ctx: &S::Ctx,
414        task: &TaskIndex,
415        setup: &S::Setup,
416        recorder: &mut Recorder,
417    ) -> Result<S::Output, S::Error> {
418        let start = Instant::now();
419        #[cfg(feature = "tui")]
420        recorder.begin_round();
421        let result = S::run(ctx, task, setup).await;
422        #[cfg(feature = "tui")]
423        recorder.end_round();
424
425        recorder.record_duration(start.elapsed());
426        recorder.record_result(&result);
427        result
428    }
429
430    /// 校验 run 的成功产出;`Err` 同样记为一次 validate 失败。
431    async fn validate(
432        ctx: &S::Ctx,
433        task: &TaskIndex,
434        setup: &S::Setup,
435        output: &S::Output,
436        recorder: &mut Recorder,
437    ) {
438        let start = Instant::now();
439        let ret = S::validate(ctx, task, setup, output).await;
440        recorder.record_validate_duration(start.elapsed());
441        match ret {
442            Ok(true) => {
443                recorder.record_validate(true);
444            }
445            Ok(false) => {
446                recorder.record_validate_failure(Cow::Borrowed("validate"));
447            }
448            Err(err) => {
449                recorder.log_internal(format!("{err:?}"));
450                recorder.record_validate_failure_with_status(err.kind(), err.status_code());
451            }
452        }
453    }
454
455    /// 收尾阶段:记录耗时与结果。
456    ///
457    /// finally 语义——`run`/`validate` 无论成败都会调用,且不可被停止信号打断。
458    /// 失败只记账(`teardown_failures` + `teardown:{kind}` 错误明细),
459    /// 不加剧 `failures`/`timeouts`,不触发退避,也不中止任务。
460    /// `result` 为该轮 `run` 的结果(`TEARDOWN_MODE = Round`);任务级收尾传 `None`。
461    async fn teardown(
462        ctx: &S::Ctx,
463        task: &TaskIndex,
464        setup: &S::Setup,
465        result: Option<Result<&S::Output, &S::Error>>,
466        recorder: &mut Recorder,
467    ) {
468        let start = Instant::now();
469        let ret = S::teardown(ctx, task, setup, result).await;
470        recorder.record_teardown_duration(start.elapsed());
471        match ret {
472            Ok(()) => recorder.record_teardown(true),
473            Err(err) => {
474                recorder.log_internal(format!("{err:?}"));
475                recorder.record_teardown(false);
476                recorder.record_teardown_error_with_status(err.kind(), err.status_code());
477            }
478        }
479    }
480}