1use std::collections::{BTreeMap, BTreeSet};
9
10use crate::api::{
11 self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
12 RunLink, Severity, WaitSpread, WeekWaits,
13};
14use crate::config::{Admission, ByteSize, Config};
15use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
16use crate::sizing::{HELD, Throttling, spread};
17
18const SECOND: u64 = 1000;
19const DAY: u64 = 24 * 3600 * SECOND;
20const WEEK: u64 = 7 * DAY;
21
22const RECENT: usize = 5;
24const SLOWER_PERCENT: u64 = 20;
26const SLOWER_SECONDS: u64 = 30;
28
29const CAP_RUNS: usize = 10;
31const CAP_MIN_RUNS: usize = 5;
32const NEAR_CAP: u64 = 85;
34const NEAR_CAP_WARNING: usize = 2;
36const FLAKY_WARNING: u32 = 2;
38
39const WAITS_PERCENT: u64 = 50;
42const WAITS_SECONDS: u64 = 60;
43const WAITS_MIN_JOBS: u32 = 5;
44const WAITS_ONE_IN_20: u32 = 20;
46
47const THROTTLED_MIN_RUNS: u32 = 2;
49
50#[derive(Debug, Clone, PartialEq, Eq)]
52pub struct Run {
53 pub id: u64,
55 pub repository: String,
57 pub workflow: Option<String>,
59 pub name: String,
61 pub branch: Option<String>,
63 pub url: Option<String>,
65 pub run_id: u64,
67 pub run_attempt: u32,
69 pub state: JobState,
71 pub conclusion: Option<String>,
73 pub queued_at: u64,
75 pub started_at: Option<u64>,
77 pub finished_at: Option<u64>,
79 pub runner_started_at: Option<u64>,
81 pub profile: Option<String>,
83 pub memory_peak: Option<u64>,
85 pub build_cache: Option<api::BuildCacheCounts>,
87 pub sock_throttled: Option<u64>,
89 pub cpu_cap: Option<u64>,
91 pub cpu_periods: Option<u64>,
93 pub cpu_throttled: Option<u64>,
95 pub unseen: u64,
97}
98
99impl Run {
100 fn finished(&self) -> u64 {
101 self.finished_at.unwrap_or(0)
102 }
103
104 fn succeeded(&self) -> bool {
105 self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
106 }
107
108 fn failed(&self) -> bool {
110 self.state == JobState::Lost || self.failed_itself()
111 }
112
113 fn failed_itself(&self) -> bool {
115 self.state == JobState::Completed
116 && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
117 }
118
119 fn took(&self) -> Option<u64> {
121 Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
122 }
123
124 fn waited(&self, kept_since: u64) -> Option<u64> {
126 let wait = crate::sizing::Wait {
127 queued_at: self.queued_at,
128 started_at: self.started_at?,
129 unseen: self.unseen,
130 };
131 Some(wait.counted(kept_since) / SECOND)
132 }
133
134 fn throttling(&self) -> Throttling {
135 Throttling {
136 started_at: self.started_at,
137 finished_at: self.finished_at,
138 periods: self.cpu_periods,
139 throttled: self.cpu_throttled,
140 }
141 }
142
143 fn link(&self) -> RunLink {
144 RunLink {
145 job: self.id,
146 run_id: self.run_id,
147 run_attempt: self.run_attempt,
148 branch: self.branch.clone(),
149 url: self.url.clone(),
150 at: self.finished() / SECOND,
151 }
152 }
153}
154
155#[derive(Debug, Clone, Default, PartialEq, Eq)]
157pub struct History {
158 pub runs: Vec<Run>,
160 pub kept_since: u64,
163}
164
165type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
167
168fn key_of(run: &Run) -> JobKey<'_> {
169 (&run.repository, run.workflow.as_deref(), &run.name)
170}
171
172pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
175 let runs = &history.runs;
176 let since = now.saturating_sub(u64::from(days) * DAY);
177 let window: Vec<&Run> = runs
178 .iter()
179 .filter(|run| {
180 matches!(run.state, JobState::Completed | JobState::Lost)
181 && run
182 .finished_at
183 .is_some_and(|at| (since..=now).contains(&at))
184 })
185 .collect();
186 let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
187 for run in &window {
188 jobs.entry(key_of(run)).or_default().push(run);
189 }
190 let mut found = Vec::new();
191 for (key, runs) in &mut jobs {
192 runs.sort_by_key(|run| (run.finished(), run.id));
193 found.extend(slower(*key, runs));
194 found.extend(flaky(*key, runs));
195 found.extend(near_cap(*key, runs, config));
196 found.extend(cold_cache(*key, runs));
197 found.extend(cpu_held(*key, runs, config));
198 }
199 found.extend(throttled(&window));
200 let waits = waits(history, now);
201 found.extend(waits.iter().filter_map(waits_insight));
202 found.sort_by(|a, b| {
203 (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
204 b.severity,
205 &b.repository,
206 &b.workflow,
207 &b.job,
208 &b.key,
209 ))
210 });
211 Insights {
212 since: since / SECOND,
213 until: now / SECOND,
214 insights: found,
215 waits,
216 }
217}
218
219pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
222 let runs = &history.runs;
223 let since = now.saturating_sub(DAY);
224 let finished: Vec<&Run> = runs
225 .iter()
226 .filter(|run| {
227 matches!(run.state, JobState::Completed | JobState::Lost)
228 && run.finished_at.is_some_and(|at| at > since && at <= now)
229 })
230 .collect();
231 let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
232 .insights
233 .into_iter()
234 .map(|insight| insight.key)
235 .collect();
236 let new = insights(history, config, api::INSIGHT_DAYS, now)
237 .insights
238 .into_iter()
239 .filter(|insight| !before.contains(&insight.key))
240 .collect();
241 let mut started: Vec<&Run> = runs
242 .iter()
243 .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
244 .collect();
245 started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
246 Digest {
247 since: since / SECOND,
248 until: now / SECOND,
249 finished: finished.len() as u64,
250 failed: finished.iter().filter(|run| run.failed()).count() as u64,
251 new,
252 waits: started
253 .into_iter()
254 .take(3)
255 .map(|run| LongWait {
256 job: run.id,
257 repository: run.repository.clone(),
258 workflow: run.workflow.clone(),
259 name: run.name.clone(),
260 waited: run.waited(history.kept_since).unwrap_or(0),
261 url: run.url.clone(),
262 })
263 .collect(),
264 savings: None,
265 }
266}
267
268fn subject((repository, workflow, name): JobKey<'_>) -> String {
270 match workflow {
271 Some(workflow) => format!("{repository}: {workflow} / {name}"),
272 None => format!("{repository}: {name}"),
273 }
274}
275
276fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
277 format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
278}
279
280fn insight(
281 key: JobKey<'_>,
282 kind: &str,
283 severity: Severity,
284 sentence: String,
285 found: InsightKind,
286 runs: Vec<RunLink>,
287) -> Insight {
288 Insight {
289 key: key_text(kind, key),
290 severity,
291 repository: key.0.to_owned(),
292 workflow: key.1.map(str::to_owned),
293 job: Some(key.2.to_owned()),
294 sentence,
295 kind: found,
296 runs,
297 }
298}
299
300fn median(values: &[u64]) -> u64 {
302 let mut values = values.to_vec();
303 values.sort_unstable();
304 let middle = values.len() / 2;
305 match values.len() {
306 0 => 0,
307 len if len % 2 == 1 => values[middle],
308 _ => u64::midpoint(values[middle - 1], values[middle]),
309 }
310}
311
312fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
314 let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
315 runs.iter()
316 .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
317 .filter_map(|run| Some((*run, run.took()?)))
318 .collect()
319 };
320 let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
321 for run in runs.iter().filter(|run| run.succeeded()) {
322 if let Some(branch) = &run.branch {
323 *branches.entry(branch).or_default() += 1;
324 }
325 }
326 let busiest = branches
329 .iter()
330 .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
331 .map(|(branch, _)| *branch)?;
332 let (branch, timed) = ["main", "master"]
333 .into_iter()
334 .map(|branch| (branch, timed_on(branch)))
335 .find(|(_, timed)| timed.len() >= 2 * RECENT)
336 .unwrap_or_else(|| (busiest, timed_on(busiest)));
337 if timed.len() < 2 * RECENT {
338 return None;
339 }
340 let (baseline, recent) = timed.split_at(timed.len() - RECENT);
341 let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
342 let before = median(&took(baseline));
343 let now = median(&took(recent));
344 let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
345 if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
348 return None;
349 }
350 let mut since = None;
352 let mut fast_in_a_row = 0;
353 for (run, took) in &timed {
354 if *took >= threshold {
355 fast_in_a_row = 0;
356 since = since.or(Some(*run));
357 } else {
358 fast_in_a_row += 1;
359 if fast_in_a_row >= 2 {
360 since = None;
361 }
362 }
363 }
364 let since = since?;
365 let sentence = format!(
366 "{} on {branch} takes {} now, up from {}.",
367 subject(key),
368 span(now),
369 span(before)
370 );
371 let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
372 Some(insight(
373 key,
374 "slower",
375 Severity::Warning,
376 sentence,
377 InsightKind::Slower {
378 branch: Some(branch.to_owned()),
379 before,
380 now,
381 since: Some(since.link()),
382 },
383 links,
384 ))
385}
386
387fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
389 let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
390 for run in runs {
391 by_run.entry(run.run_id).or_default().push(run);
392 }
393 let mut failed_first: Vec<&Run> = by_run
394 .values()
395 .filter_map(|attempts| {
396 attempts.iter().copied().find(|failed| {
397 failed.failed_itself()
398 && attempts
399 .iter()
400 .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
401 })
402 })
403 .collect();
404 if failed_first.is_empty() {
405 return None;
406 }
407 failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
408 let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
409 let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
410 let sentence = format!(
411 "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
412 subject(key)
413 );
414 let severity = if flaky >= FLAKY_WARNING {
416 Severity::Warning
417 } else {
418 Severity::Notice
419 };
420 Some(insight(
421 key,
422 "flaky",
423 severity,
424 sentence,
425 InsightKind::Flaky { flaky, total },
426 failed_first.iter().take(5).map(|run| run.link()).collect(),
427 ))
428}
429
430fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
433 let measured = runs
434 .iter()
435 .rev()
436 .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
437 let name = measured.clone().next()?.profile.as_deref()?;
438 let measured: Vec<&Run> = measured
439 .filter(|run| run.profile.as_deref() == Some(name))
440 .take(CAP_RUNS)
441 .copied()
442 .collect();
443 if measured.len() < CAP_MIN_RUNS {
444 return None;
445 }
446 let profile = config.profiles.get(name)?;
447 let cap = profile.memory.0;
448 let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
449 let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
450 let peak = spread(peaks.clone()).p95;
451 if cap == 0 || !near(&peak) {
452 return None;
453 }
454 let near_runs = peaks.iter().filter(|peak| near(peak)).count();
455 let auto = if !config.sizing.auto {
456 AutoNote::Off
457 } else if profile
458 .max_memory
459 .is_some_and(|max| profile.memory.0 >= max.0)
460 {
461 AutoNote::AtMax
462 } else {
463 AutoNote::Raises
464 };
465 let what = match auto {
466 AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
467 AutoNote::AtMax => {
468 format!("{name} is at its max_memory, so auto sizing cannot raise it.")
469 }
470 AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
471 };
472 let sentence = format!(
473 "{} peaks at {}, {}% of {name}'s {}. {what}",
474 subject(key),
475 ByteSize(peak),
476 peak.saturating_mul(100) / cap,
477 ByteSize(cap)
478 );
479 let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
481 Severity::Notice
482 } else {
483 Severity::Warning
484 };
485 Some(insight(
486 key,
487 "near-cap",
488 severity,
489 sentence,
490 InsightKind::NearCap {
491 profile: name.to_owned(),
492 peak,
493 cap,
494 auto,
495 },
496 measured.iter().take(5).map(|run| run.link()).collect(),
497 ))
498}
499
500const COLD_RUNS: usize = 5;
502const COLD_MIN_RUNS: usize = 3;
503const COLD_MIN_LOOKUPS: u64 = 10;
504const COLD_PERCENT: u64 = 10;
507
508fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
509 let writers: Vec<&Run> = runs
510 .iter()
511 .rev()
512 .filter(|run| {
513 run.build_cache
514 .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
515 })
516 .take(COLD_RUNS)
517 .copied()
518 .collect();
519 if writers.len() < COLD_MIN_RUNS {
520 return None;
521 }
522 let mut total = api::BuildCacheCounts::default();
523 for run in &writers {
524 total.add(&run.build_cache.unwrap_or_default());
525 }
526 let percent = total.hit_percent()?;
527 if percent > COLD_PERCENT {
528 return None;
529 }
530 let branch = writers.first()?.branch.clone();
531 let sentence = format!(
532 "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
533 differ from run to run.",
534 subject(key),
535 branch
536 .as_deref()
537 .map(|branch| format!(" on {branch}"))
538 .unwrap_or_default(),
539 writers.len()
540 );
541 Some(insight(
542 key,
543 "cold-cache",
544 Severity::Notice,
545 sentence,
546 InsightKind::ColdCache {
547 branch,
548 hits: total.hits,
549 lookups: total.lookups(),
550 },
551 writers.iter().map(|run| run.link()).collect(),
552 ))
553}
554
555fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
562 let measured = runs.iter().rev().filter(|run| {
563 run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
564 });
565 let last = measured.clone().next()?;
566 let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
567 let profile = config.profiles.get(name)?;
568 if u64::from(profile.cpus) * 1000 != cap {
570 return None;
571 }
572 let at_cap: Vec<&Run> = measured
573 .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
574 .take(CAP_RUNS)
575 .copied()
576 .collect();
577 let held: Vec<&Run> = at_cap
578 .iter()
579 .filter(|run| run.throttling().held() == Some(true))
580 .copied()
581 .collect();
582 if (held.len() as u64) < RAISE_RUNS {
583 return None;
584 }
585 let cpus = profile.cpus;
586 let auto = config.host.admission == Admission::Measured && config.sizing.auto;
587 let raised_from = auto
588 .then(|| {
589 let ran: Vec<CpuRun> = runs
590 .iter()
591 .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
592 .map(|run| CpuRun {
593 cap: run.cpu_cap,
594 queued_at: run.queued_at,
595 throttling: run.throttling(),
596 })
597 .collect();
598 let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
599 unhelped_raise(&ran, cap, &now)
600 })
601 .flatten()
602 .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
603 let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
604 let what = match (raised_from, set) {
605 (Some(from), _) => format!(
606 "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
607 tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
608 sizing does next."
609 ),
610 (None, Some(set)) if config.host.admission == Admission::Measured => {
611 format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
612 }
613 (None, Some(set)) => format!(
614 "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
615 cpus to {set} if runnerctl sizing shows the budget has room."
616 ),
617 (None, None) => format!(
618 "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
619 room; runnerctl sizing says when it cannot."
620 ),
621 };
622 let sentence = format!(
623 "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
624 its run or more. {what}",
625 subject(key),
626 held.len(),
627 at_cap.len()
628 );
629 let severity = if auto && raised_from.is_none() {
631 Severity::Notice
632 } else {
633 Severity::Warning
634 };
635 Some(insight(
636 key,
637 "cpu-held",
638 severity,
639 sentence,
640 InsightKind::CpuHeld {
641 profile: name.to_owned(),
642 cpus,
643 held: u32::try_from(held.len()).unwrap_or(u32::MAX),
644 total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
645 raised_from,
646 set,
647 },
648 held.iter().take(5).map(|run| run.link()).collect(),
649 ))
650}
651
652fn throttled(window: &[&Run]) -> Vec<Insight> {
654 let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
655 for run in window {
656 repositories.entry(&run.repository).or_default().push(run);
657 }
658 repositories
659 .into_iter()
660 .filter_map(|(repository, mut runs)| {
661 runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
662 let hit: Vec<&Run> = runs
663 .iter()
664 .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
665 .copied()
666 .collect();
667 let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
668 if throttled < THROTTLED_MIN_RUNS {
669 return None;
670 }
671 let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
672 Some(Insight {
673 key: format!("throttled:{repository}"),
674 severity: Severity::Warning,
675 repository: repository.to_owned(),
676 workflow: None,
677 job: None,
678 sentence: format!(
679 "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
680 host, or a job at its cap, was short of memory."
681 ),
682 kind: InsightKind::Throttled { throttled, total },
683 runs: hit.iter().take(5).map(|run| run.link()).collect(),
684 })
685 })
686 .collect()
687}
688
689fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
693 let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
694 let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
695 for run in &history.runs {
696 let Some(started) = run.started_at.filter(|at| *at <= now) else {
697 continue;
698 };
699 let week = match now - started {
700 back if back < WEEK => 0,
701 back if back < 2 * WEEK && !before_kept => 1,
702 _ => continue,
703 };
704 weeks.entry(&run.repository).or_default()[week].push(run);
705 }
706 weeks
707 .into_iter()
708 .map(|(repository, [now, before])| RepositoryWaits {
709 repository: repository.to_owned(),
710 now: week_waits(&now, history.kept_since),
711 before: week_waits(&before, history.kept_since),
712 })
713 .collect()
714}
715
716fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
717 let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
718 let of = |runs: Vec<&&Run>| {
719 let waited: Vec<u64> = runs
720 .iter()
721 .filter_map(|run| run.waited(kept_since))
722 .collect();
723 let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
724 let spread = (!waited.is_empty()).then(|| spread(waited));
725 WaitSpread {
726 jobs,
727 p50: spread.as_ref().map(|spread| spread.p50),
728 p95: spread.as_ref().map(|spread| spread.p95),
729 }
730 };
731 WeekWaits {
732 all: of(runs.iter().collect()),
733 warm: of(runs.iter().filter(warm).collect()),
734 cold: of(runs.iter().filter(|run| !warm(run)).collect()),
735 }
736}
737
738fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
739 if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
740 return None;
741 }
742 let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
743 let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
744 if now < threshold {
745 return None;
746 }
747 Some(Insight {
748 key: format!("waits:{}", waits.repository),
749 severity: Severity::Notice,
750 repository: waits.repository.clone(),
751 workflow: None,
752 job: None,
753 sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
754 format!(
755 "{}: the longest wait for a runner this week is {}, up from {} the week before.",
756 waits.repository,
757 span(now),
758 span(before)
759 )
760 } else {
761 format!(
762 "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
763 waits.repository,
764 span(now),
765 span(before)
766 )
767 },
768 kind: InsightKind::Waits { before, now },
769 runs: Vec::new(),
770 })
771}
772
773fn span(seconds: u64) -> String {
775 match seconds {
776 0..60 => format!("{seconds}s"),
777 60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
778 60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
779 _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
780 }
781}
782
783#[cfg(test)]
784mod tests;