kaizen/core_loop/
alert_checks.rs

1// SPDX-License-Identifier: AGPL-3.0-or-later
2use crate::core_loop::{AlertEvent, AlertSeverity};
3use crate::store::Store;
4use anyhow::Result;
5
6type EventRow = (crate::core::event::SessionRecord, crate::core::event::Event);
7
8pub fn check_builtin(
9    store: &Store,
10    workspace: &str,
11    start_ms: u64,
12    now_ms: u64,
13) -> Result<Vec<AlertEvent>> {
14    let rows = store.workspace_events(workspace)?;
15    let mut out = crate::core_loop::alert_cost::cost_spike(store, workspace, start_ms, now_ms)?;
16    out.extend(error_rate(store, &rows, start_ms, now_ms)?);
17    out.extend(context_pressure(store, &rows, start_ms, now_ms)?);
18    out.extend(retry_cascade(store, &rows, start_ms, now_ms)?);
19    out.extend(truncation(store, &rows, start_ms, now_ms)?);
20    out.extend(feedback_drop(store, start_ms, now_ms)?);
21    out.extend(eval_regression(store, start_ms, now_ms)?);
22    Ok(out)
23}
24
25fn error_rate(
26    store: &Store,
27    rows: &[EventRow],
28    start_ms: u64,
29    now_ms: u64,
30) -> Result<Vec<AlertEvent>> {
31    let n = rows
32        .iter()
33        .filter(|(_, e)| {
34            e.ts_ms >= start_ms && matches!(e.kind, crate::core::event::EventKind::Error)
35        })
36        .count();
37    alert_if(
38        store,
39        n >= 3,
40        "error_rate",
41        "three or more error events in window",
42        start_ms,
43        now_ms,
44    )
45}
46
47fn context_pressure(
48    store: &Store,
49    rows: &[EventRow],
50    start_ms: u64,
51    now_ms: u64,
52) -> Result<Vec<AlertEvent>> {
53    let n = rows
54        .iter()
55        .filter(|(_, e)| e.ts_ms >= start_ms && pressure(e))
56        .count();
57    alert_if(
58        store,
59        n > 0,
60        "context_pressure",
61        "one or more events used at least 80% context",
62        start_ms,
63        now_ms,
64    )
65}
66
67fn retry_cascade(
68    store: &Store,
69    rows: &[EventRow],
70    start_ms: u64,
71    now_ms: u64,
72) -> Result<Vec<AlertEvent>> {
73    let n: u16 = rows
74        .iter()
75        .filter(|(_, e)| e.ts_ms >= start_ms)
76        .filter_map(|(_, e)| e.retry_count)
77        .sum();
78    alert_if(
79        store,
80        n >= 3,
81        "rate_limit_cascade",
82        "retry count crossed cascade threshold",
83        start_ms,
84        now_ms,
85    )
86}
87
88fn truncation(
89    store: &Store,
90    rows: &[EventRow],
91    start_ms: u64,
92    now_ms: u64,
93) -> Result<Vec<AlertEvent>> {
94    let scoped = rows
95        .iter()
96        .filter(|(_, e)| e.ts_ms >= start_ms)
97        .collect::<Vec<_>>();
98    let n = scoped
99        .iter()
100        .filter(|(_, e)| e.stop_reason.as_deref() == Some("max_tokens"))
101        .count();
102    alert_if(
103        store,
104        !scoped.is_empty() && n * 10 >= scoped.len(),
105        "truncation_rate",
106        "max-token stops crossed 10%",
107        start_ms,
108        now_ms,
109    )
110}
111
112fn feedback_drop(store: &Store, start_ms: u64, now_ms: u64) -> Result<Vec<AlertEvent>> {
113    let n = store
114        .list_feedback_in_window(start_ms, now_ms)?
115        .into_iter()
116        .filter(|r| {
117            r.label.as_ref().is_some_and(|l| {
118                matches!(
119                    l,
120                    crate::feedback::types::FeedbackLabel::Bad
121                        | crate::feedback::types::FeedbackLabel::Regression
122                )
123            })
124        })
125        .count();
126    alert_if(
127        store,
128        n >= 2,
129        "feedback_score_drop",
130        "bad/regression feedback crossed threshold",
131        start_ms,
132        now_ms,
133    )
134}
135
136fn eval_regression(store: &Store, start_ms: u64, now_ms: u64) -> Result<Vec<AlertEvent>> {
137    let rows = store.list_evals_in_window(start_ms, now_ms)?;
138    let mean = rows.iter().map(|r| r.score).sum::<f64>() / rows.len().max(1) as f64;
139    alert_if(
140        store,
141        rows.len() >= 3 && mean < 0.4,
142        "eval_regression",
143        "mean eval score below 0.40",
144        start_ms,
145        now_ms,
146    )
147}
148
149fn alert_if(
150    store: &Store,
151    ok: bool,
152    name: &str,
153    msg: &str,
154    start_ms: u64,
155    now_ms: u64,
156) -> Result<Vec<AlertEvent>> {
157    if !ok {
158        return Ok(vec![]);
159    }
160    crate::core_loop::alerts::emit(
161        store,
162        &format!("builtin:{name}:{start_ms}"),
163        name,
164        AlertSeverity::Warning,
165        msg,
166        None,
167        now_ms,
168    )
169    .map(|a| vec![a])
170}
171
172fn pressure(e: &crate::core::event::Event) -> bool {
173    matches!((e.context_used_tokens, e.context_max_tokens), (Some(u), Some(m)) if m > 0 && u * 100 >= m * 80)
174}
kaizen/core_loop/alert_checks.rs

kaizen/core_loop/
alert_checks.rs
