forest/health/
mod.rs

1// Copyright 2019-2026 ChainSafe Systems
2// SPDX-License-Identifier: Apache-2.0, MIT
3use std::sync::Arc;
4
5use axum::{
6    Router,
7    response::{IntoResponse, Response},
8    routing::get,
9};
10
11use crate::{Config, chain_sync::SyncStatus, libp2p::PeerManager, networks::ChainConfig};
12
13mod endpoints;
14
15/// Default listening port for the healthcheck server.
16pub const DEFAULT_HEALTHCHECK_PORT: u16 = 2346;
17
18/// State shared between the healthcheck server and the main application.
19pub(crate) struct ForestState {
20    pub config: Config,
21    pub chain_config: Arc<ChainConfig>,
22    pub genesis_timestamp: u64,
23    pub sync_status: SyncStatus,
24    pub peer_manager: Arc<PeerManager>,
25}
26
27/// Initializes the healthcheck server. The server listens on the address specified in the
28/// configuration (passed via state) and responds to the following endpoints:
29/// - `[endpoints::healthz]`
30/// - `[endpoints::readyz]`
31/// - `[endpoints::livez]`
32///
33/// All endpoints accept an optional `verbose` query parameter. If present, the response will include detailed information about the checks performed.
34pub(crate) async fn init_healthcheck_server(
35    forest_state: ForestState,
36    tcp_listener: tokio::net::TcpListener,
37) -> anyhow::Result<()> {
38    let healthcheck_service = Router::new()
39        .route("/healthz", get(endpoints::healthz))
40        .route("/readyz", get(endpoints::readyz))
41        .route("/livez", get(endpoints::livez))
42        .with_state(forest_state.into());
43
44    axum::serve(tcp_listener, healthcheck_service).await?;
45    Ok(())
46}
47
48/// Simple error wrapper for the healthcheck server
49struct AppError(anyhow::Error);
50
51impl IntoResponse for AppError {
52    fn into_response(self) -> Response {
53        (http::StatusCode::SERVICE_UNAVAILABLE, self.0.to_string()).into_response()
54    }
55}
56
57#[cfg(test)]
58mod test {
59    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
60
61    use super::*;
62    use crate::Client;
63    use crate::chain_sync::{NodeSyncStatus, SyncStatusReport};
64    use crate::cli_shared::cli::ChainIndexerConfig;
65    use arc_swap::ArcSwap;
66    use reqwest::StatusCode;
67
68    #[tokio::test]
69    async fn test_check_readyz() {
70        let healthcheck_address = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0);
71        let rpc_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
72
73        let sync_status = Arc::new(ArcSwap::from_pointee(SyncStatusReport::init()));
74
75        let forest_state = ForestState {
76            config: Config {
77                chain_indexer: ChainIndexerConfig {
78                    enable_indexer: true,
79                    gc_retention_epochs: None,
80                },
81                client: Client {
82                    healthcheck_address,
83                    rpc_address: rpc_listener.local_addr().unwrap(),
84                    ..Default::default()
85                },
86                ..Default::default()
87            },
88            chain_config: Default::default(),
89            genesis_timestamp: 0,
90            sync_status: sync_status.clone(),
91            peer_manager: Default::default(),
92        };
93
94        let listener =
95            tokio::net::TcpListener::bind(forest_state.config.client.healthcheck_address)
96                .await
97                .unwrap();
98        let healthcheck_port = listener.local_addr().unwrap().port();
99
100        tokio::spawn(async move {
101            init_healthcheck_server(forest_state, listener)
102                .await
103                .unwrap();
104        });
105
106        let call_healthcheck = |verbose| {
107            reqwest::get(format!(
108                "http://localhost:{}/readyz{}",
109                healthcheck_port,
110                if verbose { "?verbose" } else { "" }
111            ))
112        };
113
114        // instrument the state so that the ready requirements are met
115        sync_status.store(
116            sync_status
117                .load()
118                .as_ref()
119                .clone()
120                .with_status(NodeSyncStatus::Synced)
121                .with_current_head_epoch(i64::MAX)
122                .into(),
123        );
124
125        assert_eq!(
126            call_healthcheck(false).await.unwrap().status(),
127            StatusCode::OK
128        );
129        let response = call_healthcheck(true).await.unwrap();
130        assert_eq!(response.status(), StatusCode::OK);
131        let text = response.text().await.unwrap();
132        assert!(text.contains("[+] sync complete"));
133        assert!(text.contains("[+] epoch up to date"));
134        assert!(text.contains("[+] rpc server running"));
135
136        // instrument the state so that the ready requirements are not met
137        drop(rpc_listener);
138        sync_status.store(
139            sync_status
140                .load()
141                .as_ref()
142                .clone()
143                .with_status(NodeSyncStatus::Error)
144                .with_current_head_epoch(0)
145                .into(),
146        );
147
148        assert_eq!(
149            call_healthcheck(false).await.unwrap().status(),
150            StatusCode::SERVICE_UNAVAILABLE
151        );
152        let response = call_healthcheck(true).await.unwrap();
153        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
154
155        let text = response.text().await.unwrap();
156        assert!(text.contains("[!] sync incomplete"));
157        assert!(text.contains("[!] epoch outdated"));
158        assert!(text.contains("[!] rpc server not running"));
159    }
160
161    #[tokio::test]
162    async fn test_check_livez() {
163        let healthcheck_address = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0);
164        let rpc_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
165
166        let sync_status = Arc::new(ArcSwap::from_pointee(SyncStatusReport::default()));
167        let peer_manager = Arc::new(PeerManager::default());
168        let forest_state = ForestState {
169            config: Config {
170                client: Client {
171                    healthcheck_address,
172                    rpc_address: rpc_listener.local_addr().unwrap(),
173                    ..Default::default()
174                },
175                ..Default::default()
176            },
177            chain_config: Arc::new(ChainConfig::default()),
178            genesis_timestamp: 0,
179            sync_status: sync_status.clone(),
180            peer_manager: peer_manager.clone(),
181        };
182
183        let listener =
184            tokio::net::TcpListener::bind(forest_state.config.client.healthcheck_address)
185                .await
186                .unwrap();
187        let healthcheck_port = listener.local_addr().unwrap().port();
188
189        tokio::spawn(async move {
190            init_healthcheck_server(forest_state, listener)
191                .await
192                .unwrap();
193        });
194
195        let call_healthcheck = |verbose| {
196            reqwest::get(format!(
197                "http://localhost:{}/livez{}",
198                healthcheck_port,
199                if verbose { "?verbose" } else { "" }
200            ))
201        };
202
203        // instrument the state so that the live requirements are met
204        sync_status.store(
205            sync_status
206                .load()
207                .as_ref()
208                .clone()
209                .with_status(NodeSyncStatus::Syncing)
210                .into(),
211        );
212        let peer = libp2p::PeerId::random();
213        peer_manager.touch_peer(&peer);
214
215        assert_eq!(
216            call_healthcheck(false).await.unwrap().status(),
217            StatusCode::OK
218        );
219
220        let response = call_healthcheck(true).await.unwrap();
221        assert_eq!(response.status(), StatusCode::OK);
222        let text = response.text().await.unwrap();
223        assert!(text.contains("[+] sync ok"));
224        assert!(text.contains("[+] peers connected"));
225
226        // instrument the state so that the live requirements are not met
227        sync_status.store(
228            sync_status
229                .load()
230                .as_ref()
231                .clone()
232                .with_status(NodeSyncStatus::Error)
233                .into(),
234        );
235        peer_manager.remove_peer(&peer);
236
237        assert_eq!(
238            call_healthcheck(false).await.unwrap().status(),
239            StatusCode::SERVICE_UNAVAILABLE
240        );
241
242        let response = call_healthcheck(true).await.unwrap();
243        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
244        let text = response.text().await.unwrap();
245        assert!(text.contains("[!] sync error"));
246        assert!(text.contains("[!] no peers connected"));
247    }
248
249    #[tokio::test]
250    async fn test_check_healthz() {
251        let healthcheck_address = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0);
252        let rpc_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
253        let peer_manager = Arc::new(PeerManager::default());
254
255        let sync_status = Arc::new(ArcSwap::from_pointee(SyncStatusReport::default()));
256        let forest_state = ForestState {
257            config: Config {
258                client: Client {
259                    healthcheck_address,
260                    rpc_address: rpc_listener.local_addr().unwrap(),
261                    ..Default::default()
262                },
263                ..Default::default()
264            },
265            chain_config: Arc::new(ChainConfig::default()),
266            genesis_timestamp: 0,
267            sync_status: sync_status.clone(),
268            peer_manager: peer_manager.clone(),
269        };
270
271        let listener =
272            tokio::net::TcpListener::bind(forest_state.config.client.healthcheck_address)
273                .await
274                .unwrap();
275        let healthcheck_port = listener.local_addr().unwrap().port();
276
277        tokio::spawn(async move {
278            init_healthcheck_server(forest_state, listener)
279                .await
280                .unwrap();
281        });
282
283        let call_healthcheck = |verbose| {
284            reqwest::get(format!(
285                "http://localhost:{}/healthz{}",
286                healthcheck_port,
287                if verbose { "?verbose" } else { "" }
288            ))
289        };
290
291        // instrument the state so that the health requirements are met
292        sync_status.store(
293            sync_status
294                .load()
295                .as_ref()
296                .clone()
297                .with_status(NodeSyncStatus::Syncing)
298                .with_current_head_epoch(i64::MAX)
299                .into(),
300        );
301        let peer = libp2p::PeerId::random();
302        peer_manager.touch_peer(&peer);
303
304        assert_eq!(
305            call_healthcheck(false).await.unwrap().status(),
306            StatusCode::OK
307        );
308        let response = call_healthcheck(true).await.unwrap();
309        assert_eq!(response.status(), StatusCode::OK);
310        let text = response.text().await.unwrap();
311        assert!(text.contains("[+] sync ok"));
312        assert!(text.contains("[+] epoch up to date"));
313        assert!(text.contains("[+] rpc server running"));
314        assert!(text.contains("[+] peers connected"));
315
316        // instrument the state so that the health requirements are not met
317        drop(rpc_listener);
318        sync_status.store(
319            sync_status
320                .load()
321                .as_ref()
322                .clone()
323                .with_status(NodeSyncStatus::Error)
324                .with_current_head_epoch(0)
325                .into(),
326        );
327        peer_manager.remove_peer(&peer);
328
329        assert_eq!(
330            call_healthcheck(false).await.unwrap().status(),
331            StatusCode::SERVICE_UNAVAILABLE
332        );
333        let response = call_healthcheck(true).await.unwrap();
334        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
335
336        let text = response.text().await.unwrap();
337        assert!(text.contains("[!] sync error"));
338        assert!(text.contains("[!] epoch outdated"));
339        assert!(text.contains("[!] rpc server not running"));
340        assert!(text.contains("[!] no peers connected"));
341    }
342
343    #[tokio::test]
344    async fn test_check_unknown_healthcheck_endpoint() {
345        let healthcheck_address = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0);
346        let forest_state = ForestState {
347            config: Config {
348                client: Client {
349                    healthcheck_address,
350                    ..Default::default()
351                },
352                ..Default::default()
353            },
354            chain_config: Arc::default(),
355            genesis_timestamp: 0,
356            sync_status: Arc::new(ArcSwap::from_pointee(SyncStatusReport::default())),
357            peer_manager: Arc::default(),
358        };
359        let listener =
360            tokio::net::TcpListener::bind(forest_state.config.client.healthcheck_address)
361                .await
362                .unwrap();
363        let healthcheck_port = listener.local_addr().unwrap().port();
364
365        tokio::spawn(async move {
366            init_healthcheck_server(forest_state, listener)
367                .await
368                .unwrap();
369        });
370
371        let response = reqwest::get(format!(
372            "http://localhost:{healthcheck_port}/phngluimglwnafhcthulhurlyehwgahnaglfhtagn"
373        ))
374        .await
375        .unwrap();
376
377        assert_eq!(response.status(), StatusCode::NOT_FOUND);
378    }
379}
forest/health/mod.rs

forest/health/
mod.rs
