everruns_durable/persistence/postgres/
circuit_breakers.rs1use super::*;
4
5#[async_trait]
6impl CircuitBreakers for PostgresWorkflowEventStore {
7 #[instrument(skip(self, config))]
8 async fn create_circuit_breaker(
9 &self,
10 key: &str,
11 config: &CircuitBreakerConfig,
12 ) -> Result<(), StoreError> {
13 sqlx::query(
16 r#"
17 INSERT INTO durable_circuit_breaker_state (key, state, failure_count, success_count, updated_at)
18 VALUES ($1, 'closed', 0, 0, NOW())
19 ON CONFLICT (key) DO NOTHING
20 "#,
21 )
22 .bind(key)
23 .execute(&self.pool)
24 .await
25 .map_err(|e| {
26 error!(error = %e, "Failed to create circuit breaker");
27 StoreError::Database(e.to_string())
28 })?;
29
30 debug!(
31 key,
32 failure_threshold = config.failure_threshold,
33 "created circuit breaker"
34 );
35 Ok(())
36 }
37
38 #[instrument(skip(self))]
39 async fn get_circuit_breaker(
40 &self,
41 key: &str,
42 ) -> Result<Option<CircuitBreakerState>, StoreError> {
43 let row = sqlx::query(
44 r#"
45 SELECT key, state, failure_count, success_count,
46 last_failure_at, opened_at, half_open_at, updated_at
47 FROM durable_circuit_breaker_state
48 WHERE key = $1
49 "#,
50 )
51 .bind(key)
52 .fetch_optional(&self.pool)
53 .await
54 .map_err(|e| {
55 error!(error = %e, "Failed to get circuit breaker");
56 StoreError::Database(e.to_string())
57 })?;
58
59 match row {
60 Some(row) => {
61 let state_str: String = row.get("state");
62 let state = parse_circuit_state(&state_str)?;
63
64 Ok(Some(CircuitBreakerState {
65 key: row.get("key"),
66 state,
67 failure_count: row.get::<i32, _>("failure_count") as u32,
68 success_count: row.get::<i32, _>("success_count") as u32,
69 last_failure_at: row.get("last_failure_at"),
70 opened_at: row.get("opened_at"),
71 half_open_at: row.get("half_open_at"),
72 updated_at: row.get("updated_at"),
73 }))
74 }
75 None => Ok(None),
76 }
77 }
78
79 #[instrument(skip(self))]
80 async fn update_circuit_breaker(
81 &self,
82 key: &str,
83 state: CircuitState,
84 failure_count: u32,
85 success_count: u32,
86 ) -> Result<(), StoreError> {
87 let state_str = state.to_string();
88
89 let (opened_at_update, half_open_at_update, last_failure_update) = match state {
91 CircuitState::Open => (
92 "NOW()",
93 "NULL",
94 if failure_count > 0 {
95 "NOW()"
96 } else {
97 "last_failure_at"
98 },
99 ),
100 CircuitState::HalfOpen => ("opened_at", "NOW()", "last_failure_at"),
101 CircuitState::Closed => ("NULL", "NULL", "NULL"),
102 };
103
104 let query = format!(
105 r#"
106 UPDATE durable_circuit_breaker_state
107 SET state = $2,
108 failure_count = $3,
109 success_count = $4,
110 opened_at = {},
111 half_open_at = {},
112 last_failure_at = {},
113 updated_at = NOW()
114 WHERE key = $1
115 "#,
116 opened_at_update, half_open_at_update, last_failure_update
117 );
118
119 sqlx::query(sqlx::AssertSqlSafe(query))
124 .bind(key)
125 .bind(&state_str)
126 .bind(failure_count as i32)
127 .bind(success_count as i32)
128 .execute(&self.pool)
129 .await
130 .map_err(|e| {
131 error!(error = %e, "Failed to update circuit breaker");
132 StoreError::Database(e.to_string())
133 })?;
134
135 debug!(key, %state_str, failure_count, success_count, "updated circuit breaker");
136 Ok(())
137 }
138
139 #[instrument(skip(self))]
140 async fn list_circuit_breakers(&self) -> Result<Vec<CircuitBreakerState>, StoreError> {
141 let rows = sqlx::query(
142 r#"
143 SELECT key, state, failure_count, success_count,
144 last_failure_at, opened_at, half_open_at, updated_at
145 FROM durable_circuit_breaker_state
146 ORDER BY key
147 "#,
148 )
149 .fetch_all(&self.pool)
150 .await
151 .map_err(|e| {
152 error!(error = %e, "Failed to list circuit breakers");
153 StoreError::Database(e.to_string())
154 })?;
155
156 let mut breakers = Vec::with_capacity(rows.len());
157 for row in rows {
158 let state_str: String = row.get("state");
159 breakers.push(CircuitBreakerState {
160 key: row.get("key"),
161 state: parse_circuit_state(&state_str)?,
162 failure_count: row.get::<i32, _>("failure_count") as u32,
163 success_count: row.get::<i32, _>("success_count") as u32,
164 last_failure_at: row.get("last_failure_at"),
165 opened_at: row.get("opened_at"),
166 half_open_at: row.get("half_open_at"),
167 updated_at: row.get("updated_at"),
168 });
169 }
170
171 Ok(breakers)
172 }
173
174 #[instrument(skip(self))]
175 async fn force_open_circuit_breaker(&self, key: &str) -> Result<(), StoreError> {
176 let result = sqlx::query(
177 r#"
178 UPDATE durable_circuit_breaker_state
179 SET state = 'open',
180 failure_count = 0,
181 success_count = 0,
182 opened_at = NOW(),
183 updated_at = NOW()
184 WHERE key = $1
185 "#,
186 )
187 .bind(key)
188 .execute(&self.pool)
189 .await
190 .map_err(|e| {
191 error!(error = %e, "Failed to force open circuit breaker");
192 StoreError::Database(e.to_string())
193 })?;
194
195 if result.rows_affected() == 0 {
196 sqlx::query(
198 r#"
199 INSERT INTO durable_circuit_breaker_state
200 (key, state, failure_count, success_count, opened_at, updated_at)
201 VALUES ($1, 'open', 0, 0, NOW(), NOW())
202 "#,
203 )
204 .bind(key)
205 .execute(&self.pool)
206 .await
207 .map_err(|e| {
208 error!(error = %e, "Failed to create circuit breaker in open state");
209 StoreError::Database(e.to_string())
210 })?;
211 }
212
213 info!(key, "force opened circuit breaker");
214 Ok(())
215 }
216
217 #[instrument(skip(self))]
218 async fn force_close_circuit_breaker(&self, key: &str) -> Result<(), StoreError> {
219 let result = sqlx::query(
220 r#"
221 UPDATE durable_circuit_breaker_state
222 SET state = 'closed',
223 failure_count = 0,
224 success_count = 0,
225 opened_at = NULL,
226 half_open_at = NULL,
227 updated_at = NOW()
228 WHERE key = $1
229 "#,
230 )
231 .bind(key)
232 .execute(&self.pool)
233 .await
234 .map_err(|e| {
235 error!(error = %e, "Failed to force close circuit breaker");
236 StoreError::Database(e.to_string())
237 })?;
238
239 if result.rows_affected() == 0 {
240 return Err(StoreError::CircuitBreakerNotFound(key.to_string()));
241 }
242
243 info!(key, "force closed circuit breaker");
244 Ok(())
245 }
246
247 #[instrument(skip(self))]
248 async fn delete_circuit_breaker(&self, key: &str) -> Result<(), StoreError> {
249 let result = sqlx::query(
250 r#"
251 DELETE FROM durable_circuit_breaker_state
252 WHERE key = $1
253 "#,
254 )
255 .bind(key)
256 .execute(&self.pool)
257 .await
258 .map_err(|e| {
259 error!(error = %e, "Failed to delete circuit breaker");
260 StoreError::Database(e.to_string())
261 })?;
262
263 if result.rows_affected() == 0 {
264 return Err(StoreError::CircuitBreakerNotFound(key.to_string()));
265 }
266
267 info!(key, "deleted circuit breaker");
268 Ok(())
269 }
270}