use deadpool::managed;
use std::sync::atomic::{AtomicU64, Ordering};
use thiserror::Error;
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
use tokio::time::timeout;
use crate::constants::pool::{
DATE_COMMAND, EXPECTED_DATE_RESPONSE_PREFIX, HEALTH_CHECK_BUFFER_SIZE, HEALTH_CHECK_TIMEOUT,
TCP_PEEK_BUFFER_SIZE,
};
use crate::stream::ConnectionStream;
#[allow(clippy::cast_precision_loss)] const fn count_as_f64_for_rate(value: u64) -> f64 {
value as f64
}
#[derive(Debug, Error)]
pub enum HealthCheckError {
#[error("TCP connection closed")]
TcpClosed,
#[error("Unexpected data in buffer")]
UnexpectedData,
#[error("TCP error: {0}")]
TcpError(std::io::Error),
#[error("Failed to write health check: {0}")]
WriteError(std::io::Error),
#[error("Failed to read health check response: {0}")]
ReadError(std::io::Error),
#[error("Health check timeout")]
Timeout,
#[error("Unexpected health check response: {0}")]
UnexpectedResponse(String),
#[error("Connection closed during health check")]
ConnectionClosedDuringCheck,
}
impl From<HealthCheckError> for managed::RecycleError<crate::connection_error::ConnectionError> {
fn from(err: HealthCheckError) -> Self {
Self::Message(err.to_string().into())
}
}
#[derive(Debug, Default)]
pub struct HealthCheckMetrics {
cycles_run: AtomicU64,
connections_checked: AtomicU64,
connections_failed: AtomicU64,
}
impl HealthCheckMetrics {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn record_cycle(&self, checked: u64, failed: u64) {
self.cycles_run.fetch_add(1, Ordering::Relaxed);
self.connections_checked
.fetch_add(checked, Ordering::Relaxed);
self.connections_failed.fetch_add(failed, Ordering::Relaxed);
}
pub fn failure_rate(&self) -> f64 {
let checked = self.connections_checked.load(Ordering::Relaxed);
if checked == 0 {
0.0
} else {
let failed = self.connections_failed.load(Ordering::Relaxed);
count_as_f64_for_rate(failed) / count_as_f64_for_rate(checked)
}
}
pub fn cycles_run(&self) -> u64 {
self.cycles_run.load(Ordering::Relaxed)
}
pub fn connections_checked(&self) -> u64 {
self.connections_checked.load(Ordering::Relaxed)
}
pub fn connections_failed(&self) -> u64 {
self.connections_failed.load(Ordering::Relaxed)
}
}
pub fn check_tcp_alive(
conn: &mut ConnectionStream,
) -> managed::RecycleResult<crate::connection_error::ConnectionError> {
if conn.has_pending_bytes() {
return Err(HealthCheckError::UnexpectedData.into());
}
let mut peek_buf = [0u8; TCP_PEEK_BUFFER_SIZE];
let tcp_stream = conn.underlying_tcp_stream();
match tcp_stream.try_read(&mut peek_buf) {
Ok(0) => return Err(HealthCheckError::TcpClosed.into()),
Ok(_) => {
return Err(HealthCheckError::UnexpectedData.into());
}
Err(e) if e.kind() != std::io::ErrorKind::WouldBlock => {
return Err(
HealthCheckError::TcpError(std::io::Error::new(e.kind(), e.to_string())).into(),
);
}
Err(_) => {}
}
Ok(())
}
#[inline]
pub(crate) fn validate_date_response(response: &str) -> Result<(), HealthCheckError> {
if response.starts_with(EXPECTED_DATE_RESPONSE_PREFIX) {
Ok(())
} else {
Err(HealthCheckError::UnexpectedResponse(response.to_string()))
}
}
async fn read_date_response<C>(conn: &mut C) -> Result<String, HealthCheckError>
where
C: AsyncRead + Unpin,
{
let mut response_buf = [0u8; HEALTH_CHECK_BUFFER_SIZE];
let request = crate::protocol::RequestContext::from_verb_args(b"DATE", b"");
crate::session::backend::read_single_line_reply(conn, &request, &mut response_buf)
.await
.map_err(|err| match err {
crate::session::backend::SingleLineReplyReadError::Full { bytes_read }
| crate::session::backend::SingleLineReplyReadError::Invalid { bytes_read } => {
HealthCheckError::UnexpectedResponse(
String::from_utf8_lossy(&response_buf[..bytes_read]).into_owned(),
)
}
crate::session::backend::SingleLineReplyReadError::Io(err) => {
HealthCheckError::ReadError(err)
}
crate::session::backend::SingleLineReplyReadError::Closed => {
HealthCheckError::ConnectionClosedDuringCheck
}
})
}
pub async fn check_date_response<C>(conn: &mut C) -> Result<(), HealthCheckError>
where
C: AsyncRead + AsyncWrite + Unpin,
{
let health_check = async {
conn.write_all(DATE_COMMAND)
.await
.map_err(HealthCheckError::WriteError)?;
let response = read_date_response(conn).await?;
validate_date_response(&response)
};
timeout(HEALTH_CHECK_TIMEOUT, health_check)
.await
.map_err(|_| HealthCheckError::Timeout)?
}
#[cfg(test)]
#[allow(clippy::float_cmp)] mod tests {
use super::*;
use std::collections::VecDeque;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::AsyncWrite;
struct ChunkedStream {
chunks: VecDeque<Vec<u8>>,
written: Vec<u8>,
}
impl ChunkedStream {
fn new(chunks: Vec<Vec<u8>>) -> Self {
Self {
chunks: chunks.into(),
written: Vec::new(),
}
}
}
impl tokio::io::AsyncRead for ChunkedStream {
fn poll_read(
mut self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
if let Some(chunk) = self.chunks.pop_front() {
let len = chunk.len().min(buf.remaining());
buf.put_slice(&chunk[..len]);
if len < chunk.len() {
self.chunks.push_front(chunk[len..].to_vec());
}
}
Poll::Ready(Ok(()))
}
}
impl AsyncWrite for ChunkedStream {
fn poll_write(
mut self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<std::io::Result<usize>> {
self.written.extend_from_slice(buf);
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[test]
fn test_health_check_metrics_new() {
let metrics = HealthCheckMetrics::new();
assert_eq!(metrics.cycles_run(), 0);
assert_eq!(metrics.connections_checked(), 0);
assert_eq!(metrics.connections_failed(), 0);
assert_eq!(metrics.failure_rate(), 0.0);
}
#[test]
fn test_health_check_metrics_record_cycle() {
let metrics = HealthCheckMetrics::new();
metrics.record_cycle(10, 2);
assert_eq!(metrics.cycles_run(), 1);
assert_eq!(metrics.connections_checked(), 10);
assert_eq!(metrics.connections_failed(), 2);
assert_eq!(metrics.failure_rate(), 0.2);
metrics.record_cycle(5, 1);
assert_eq!(metrics.cycles_run(), 2);
assert_eq!(metrics.connections_checked(), 15);
assert_eq!(metrics.connections_failed(), 3);
assert_eq!(metrics.failure_rate(), 0.2);
}
#[test]
fn test_health_check_metrics_failure_rate() {
let metrics = HealthCheckMetrics::new();
metrics.record_cycle(10, 0);
assert_eq!(metrics.failure_rate(), 0.0);
metrics.record_cycle(10, 5);
assert!((metrics.failure_rate() - 0.25).abs() < 0.01);
metrics.record_cycle(10, 10);
assert!((metrics.failure_rate() - 0.5).abs() < 0.01);
}
#[test]
fn test_health_check_metrics_zero_checked() {
let metrics = HealthCheckMetrics::new();
assert_eq!(metrics.failure_rate(), 0.0);
}
#[test]
fn test_health_check_metrics_multiple_cycles() {
let metrics = HealthCheckMetrics::new();
for i in 1..=5 {
metrics.record_cycle(10, 1);
assert_eq!(metrics.cycles_run(), i);
}
assert_eq!(metrics.connections_checked(), 50);
assert_eq!(metrics.connections_failed(), 5);
assert_eq!(metrics.failure_rate(), 0.1);
}
#[tokio::test]
async fn test_tcp_alive_check_rejects_queued_backend_bytes() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let client_handle =
tokio::spawn(async move { tokio::net::TcpStream::connect(addr).await.unwrap() });
let (server_stream, _) = listener.accept().await.unwrap();
let _client = client_handle.await.unwrap();
let mut conn = ConnectionStream::plain(server_stream);
conn.queue_pending_bytes(b"430 stale response\r\n").unwrap();
let result = check_tcp_alive(&mut conn);
assert!(
result.is_err(),
"connections with queued backend bytes must not recycle"
);
}
#[test]
fn test_health_check_error_display() {
assert_eq!(
HealthCheckError::TcpClosed.to_string(),
"TCP connection closed"
);
assert_eq!(
HealthCheckError::UnexpectedData.to_string(),
"Unexpected data in buffer"
);
assert_eq!(
HealthCheckError::Timeout.to_string(),
"Health check timeout"
);
assert_eq!(
HealthCheckError::ConnectionClosedDuringCheck.to_string(),
"Connection closed during health check"
);
}
#[test]
fn test_health_check_error_unexpected_response() {
let err = HealthCheckError::UnexpectedResponse("500 Error".to_string());
assert_eq!(
err.to_string(),
"Unexpected health check response: 500 Error"
);
}
#[test]
fn test_health_check_error_tcp_error() {
let io_err = std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset");
let err = HealthCheckError::TcpError(io_err);
assert!(err.to_string().contains("TCP error"));
assert!(err.to_string().contains("reset"));
}
#[test]
fn test_health_check_error_write_error() {
let io_err = std::io::Error::new(std::io::ErrorKind::BrokenPipe, "pipe");
let err = HealthCheckError::WriteError(io_err);
assert!(err.to_string().contains("Failed to write health check"));
}
#[test]
fn test_health_check_error_read_error() {
let io_err = std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout");
let err = HealthCheckError::ReadError(io_err);
assert!(
err.to_string()
.contains("Failed to read health check response")
);
}
#[test]
fn test_validate_date_response_success() {
assert!(validate_date_response("111 20231215120000\r\n").is_ok());
}
#[test]
fn test_validate_date_response_success_minimal() {
assert!(validate_date_response("111 \r\n").is_ok());
}
#[test]
fn test_validate_date_response_success_with_extra() {
assert!(validate_date_response("111 20231215120000 extra info\r\n").is_ok());
}
#[test]
fn test_validate_date_response_wrong_code() {
let result = validate_date_response("200 OK\r\n");
assert!(result.is_err());
match result {
Err(HealthCheckError::UnexpectedResponse(msg)) => {
assert_eq!(msg, "200 OK\r\n");
}
_ => panic!("Expected UnexpectedResponse error"),
}
}
#[test]
fn test_validate_date_response_error_code() {
assert!(validate_date_response("400 Bad Request\r\n").is_err());
assert!(validate_date_response("500 Server Error\r\n").is_err());
}
#[test]
fn test_validate_date_response_empty() {
let result = validate_date_response("");
assert!(result.is_err());
}
#[test]
fn test_validate_date_response_malformed() {
assert!(validate_date_response("not a valid response").is_err());
assert!(validate_date_response("1\r\n").is_err());
assert!(validate_date_response("11 \r\n").is_err()); }
#[test]
fn test_validate_date_response_partial_match() {
assert!(validate_date_response("110 Info\r\n").is_err());
assert!(validate_date_response("112 Other\r\n").is_err());
}
#[test]
fn test_validate_date_response_no_space() {
assert!(validate_date_response("11120231215120000\r\n").is_err());
}
#[test]
fn test_validate_date_response_whitespace_prefix() {
assert!(validate_date_response(" 111 20231215120000\r\n").is_err());
assert!(validate_date_response("\r\n111 20231215120000\r\n").is_err());
}
#[test]
fn test_validate_date_response_case_sensitivity() {
assert!(validate_date_response("111 lowercase\r\n").is_ok());
assert!(validate_date_response("111 UPPERCASE\r\n").is_ok());
}
#[test]
fn test_validate_date_response_unicode() {
assert!(validate_date_response("111 日本語\r\n").is_ok());
}
#[test]
fn test_validate_date_response_realistic_examples() {
assert!(validate_date_response("111 20231215120530\r\n").is_ok());
assert!(validate_date_response("111 19700101000000\r\n").is_ok());
assert!(validate_date_response("111 20991231235959\r\n").is_ok());
}
#[tokio::test]
async fn test_check_date_response_reads_split_reply() {
let mut stream = ChunkedStream::new(vec![b"111 20231215".to_vec(), b"120000\r\n".to_vec()]);
let result = check_date_response(&mut stream).await;
assert!(
result.is_ok(),
"split DATE responses should be consumed fully"
);
assert_eq!(stream.written, DATE_COMMAND);
}
#[tokio::test]
async fn test_check_date_response_rejects_invalid_reply_bytes() {
let mut stream = ChunkedStream::new(vec![b"abc\r\n".to_vec()]);
let result = check_date_response(&mut stream).await;
match result {
Err(HealthCheckError::UnexpectedResponse(response)) => {
assert_eq!(response, "abc\r\n");
}
other => panic!("Expected invalid DATE response, got {other:?}"),
}
}
}