Skip to main content

everruns_durable/persistence/postgres/
circuit_breakers.rs

1//! Implementation of [`crate::CircuitBreakers`].
2
3use super::*;
4
5#[async_trait]
6impl CircuitBreakers for PostgresWorkflowEventStore {
7    #[instrument(skip(self, config))]
8    async fn create_circuit_breaker(
9        &self,
10        key: &str,
11        config: &CircuitBreakerConfig,
12    ) -> Result<(), StoreError> {
13        // Use INSERT ... ON CONFLICT DO NOTHING to handle race conditions
14        // when multiple workers try to create the same circuit breaker
15        sqlx::query(
16            r#"
17            INSERT INTO durable_circuit_breaker_state (key, state, failure_count, success_count, updated_at)
18            VALUES ($1, 'closed', 0, 0, NOW())
19            ON CONFLICT (key) DO NOTHING
20            "#,
21        )
22        .bind(key)
23        .execute(&self.pool)
24        .await
25        .map_err(|e| {
26            error!(error = %e, "Failed to create circuit breaker");
27            StoreError::Database(e.to_string())
28        })?;
29
30        debug!(
31            key,
32            failure_threshold = config.failure_threshold,
33            "created circuit breaker"
34        );
35        Ok(())
36    }
37
38    #[instrument(skip(self))]
39    async fn get_circuit_breaker(
40        &self,
41        key: &str,
42    ) -> Result<Option<CircuitBreakerState>, StoreError> {
43        let row = sqlx::query(
44            r#"
45            SELECT key, state, failure_count, success_count,
46                   last_failure_at, opened_at, half_open_at, updated_at
47            FROM durable_circuit_breaker_state
48            WHERE key = $1
49            "#,
50        )
51        .bind(key)
52        .fetch_optional(&self.pool)
53        .await
54        .map_err(|e| {
55            error!(error = %e, "Failed to get circuit breaker");
56            StoreError::Database(e.to_string())
57        })?;
58
59        match row {
60            Some(row) => {
61                let state_str: String = row.get("state");
62                let state = parse_circuit_state(&state_str)?;
63
64                Ok(Some(CircuitBreakerState {
65                    key: row.get("key"),
66                    state,
67                    failure_count: row.get::<i32, _>("failure_count") as u32,
68                    success_count: row.get::<i32, _>("success_count") as u32,
69                    last_failure_at: row.get("last_failure_at"),
70                    opened_at: row.get("opened_at"),
71                    half_open_at: row.get("half_open_at"),
72                    updated_at: row.get("updated_at"),
73                }))
74            }
75            None => Ok(None),
76        }
77    }
78
79    #[instrument(skip(self))]
80    async fn update_circuit_breaker(
81        &self,
82        key: &str,
83        state: CircuitState,
84        failure_count: u32,
85        success_count: u32,
86    ) -> Result<(), StoreError> {
87        let state_str = state.to_string();
88
89        // Build the query dynamically based on state transitions
90        let (opened_at_update, half_open_at_update, last_failure_update) = match state {
91            CircuitState::Open => (
92                "NOW()",
93                "NULL",
94                if failure_count > 0 {
95                    "NOW()"
96                } else {
97                    "last_failure_at"
98                },
99            ),
100            CircuitState::HalfOpen => ("opened_at", "NOW()", "last_failure_at"),
101            CircuitState::Closed => ("NULL", "NULL", "NULL"),
102        };
103
104        let query = format!(
105            r#"
106            UPDATE durable_circuit_breaker_state
107            SET state = $2,
108                failure_count = $3,
109                success_count = $4,
110                opened_at = {},
111                half_open_at = {},
112                last_failure_at = {},
113                updated_at = NOW()
114            WHERE key = $1
115            "#,
116            opened_at_update, half_open_at_update, last_failure_update
117        );
118
119        // SAFETY: opened_at_update, half_open_at_update, last_failure_update are
120        // server-chosen literals from a closed set (`"NOW()"`, `"NULL"`,
121        // `"last_failure_at"`, `"opened_at"`); no caller input flows into the
122        // format string.
123        sqlx::query(sqlx::AssertSqlSafe(query))
124            .bind(key)
125            .bind(&state_str)
126            .bind(failure_count as i32)
127            .bind(success_count as i32)
128            .execute(&self.pool)
129            .await
130            .map_err(|e| {
131                error!(error = %e, "Failed to update circuit breaker");
132                StoreError::Database(e.to_string())
133            })?;
134
135        debug!(key, %state_str, failure_count, success_count, "updated circuit breaker");
136        Ok(())
137    }
138
139    #[instrument(skip(self))]
140    async fn list_circuit_breakers(&self) -> Result<Vec<CircuitBreakerState>, StoreError> {
141        let rows = sqlx::query(
142            r#"
143            SELECT key, state, failure_count, success_count,
144                   last_failure_at, opened_at, half_open_at, updated_at
145            FROM durable_circuit_breaker_state
146            ORDER BY key
147            "#,
148        )
149        .fetch_all(&self.pool)
150        .await
151        .map_err(|e| {
152            error!(error = %e, "Failed to list circuit breakers");
153            StoreError::Database(e.to_string())
154        })?;
155
156        let mut breakers = Vec::with_capacity(rows.len());
157        for row in rows {
158            let state_str: String = row.get("state");
159            breakers.push(CircuitBreakerState {
160                key: row.get("key"),
161                state: parse_circuit_state(&state_str)?,
162                failure_count: row.get::<i32, _>("failure_count") as u32,
163                success_count: row.get::<i32, _>("success_count") as u32,
164                last_failure_at: row.get("last_failure_at"),
165                opened_at: row.get("opened_at"),
166                half_open_at: row.get("half_open_at"),
167                updated_at: row.get("updated_at"),
168            });
169        }
170
171        Ok(breakers)
172    }
173
174    #[instrument(skip(self))]
175    async fn force_open_circuit_breaker(&self, key: &str) -> Result<(), StoreError> {
176        let result = sqlx::query(
177            r#"
178            UPDATE durable_circuit_breaker_state
179            SET state = 'open',
180                failure_count = 0,
181                success_count = 0,
182                opened_at = NOW(),
183                updated_at = NOW()
184            WHERE key = $1
185            "#,
186        )
187        .bind(key)
188        .execute(&self.pool)
189        .await
190        .map_err(|e| {
191            error!(error = %e, "Failed to force open circuit breaker");
192            StoreError::Database(e.to_string())
193        })?;
194
195        if result.rows_affected() == 0 {
196            // Circuit breaker doesn't exist, create it in open state
197            sqlx::query(
198                r#"
199                INSERT INTO durable_circuit_breaker_state
200                    (key, state, failure_count, success_count, opened_at, updated_at)
201                VALUES ($1, 'open', 0, 0, NOW(), NOW())
202                "#,
203            )
204            .bind(key)
205            .execute(&self.pool)
206            .await
207            .map_err(|e| {
208                error!(error = %e, "Failed to create circuit breaker in open state");
209                StoreError::Database(e.to_string())
210            })?;
211        }
212
213        info!(key, "force opened circuit breaker");
214        Ok(())
215    }
216
217    #[instrument(skip(self))]
218    async fn force_close_circuit_breaker(&self, key: &str) -> Result<(), StoreError> {
219        let result = sqlx::query(
220            r#"
221            UPDATE durable_circuit_breaker_state
222            SET state = 'closed',
223                failure_count = 0,
224                success_count = 0,
225                opened_at = NULL,
226                half_open_at = NULL,
227                updated_at = NOW()
228            WHERE key = $1
229            "#,
230        )
231        .bind(key)
232        .execute(&self.pool)
233        .await
234        .map_err(|e| {
235            error!(error = %e, "Failed to force close circuit breaker");
236            StoreError::Database(e.to_string())
237        })?;
238
239        if result.rows_affected() == 0 {
240            return Err(StoreError::CircuitBreakerNotFound(key.to_string()));
241        }
242
243        info!(key, "force closed circuit breaker");
244        Ok(())
245    }
246
247    #[instrument(skip(self))]
248    async fn delete_circuit_breaker(&self, key: &str) -> Result<(), StoreError> {
249        let result = sqlx::query(
250            r#"
251            DELETE FROM durable_circuit_breaker_state
252            WHERE key = $1
253            "#,
254        )
255        .bind(key)
256        .execute(&self.pool)
257        .await
258        .map_err(|e| {
259            error!(error = %e, "Failed to delete circuit breaker");
260            StoreError::Database(e.to_string())
261        })?;
262
263        if result.rows_affected() == 0 {
264            return Err(StoreError::CircuitBreakerNotFound(key.to_string()));
265        }
266
267        info!(key, "deleted circuit breaker");
268        Ok(())
269    }
270}