use std::{fmt, future::Future, sync::Arc, time::Duration};
use async_trait::async_trait;
use futures_util::{StreamExt, stream};
use tokio::time::{self, Instant};
use crate::result::BoxError;
use super::{
error::ResourceRegistryError,
model::{
DEFAULT_RESOURCE_PROBE_TIMEOUT, ResourceDescriptor, ResourceDiagnostic,
ResourceDiagnosticsReport, normalize_diagnostics_service_name,
},
};
#[async_trait]
pub trait ResourceProbe: Send + Sync + 'static {
fn descriptor(&self) -> &ResourceDescriptor;
async fn probe(&self) -> Result<ResourceDiagnostic, BoxError>;
}
#[async_trait]
impl ResourceProbe for ResourceDiagnostic {
fn descriptor(&self) -> &ResourceDescriptor {
self.resource()
}
async fn probe(&self) -> Result<ResourceDiagnostic, BoxError> {
Ok(self.clone())
}
}
#[must_use]
pub fn resource_probe<F, Fut>(descriptor: ResourceDescriptor, probe: F) -> Arc<dyn ResourceProbe>
where
F: Fn(ResourceDescriptor) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<ResourceDiagnostic, BoxError>> + Send + 'static,
{
Arc::new(FnResourceProbe { descriptor, probe })
}
#[derive(Clone)]
pub struct ResourceRegistry {
service_name: String,
probes: Vec<Arc<dyn ResourceProbe>>,
probe_timeout: Duration,
run_timeout: Option<Duration>,
max_concurrency: usize,
}
impl ResourceRegistry {
pub fn new(service_name: impl Into<String>) -> Result<Self, ResourceRegistryError> {
Ok(Self {
service_name: normalize_diagnostics_service_name(service_name)?,
probes: Vec::new(),
probe_timeout: DEFAULT_RESOURCE_PROBE_TIMEOUT,
run_timeout: None,
max_concurrency: 1,
})
}
pub fn with_probe_timeout(
mut self,
probe_timeout: Duration,
) -> Result<Self, ResourceRegistryError> {
if probe_timeout.is_zero()
|| tokio::time::Instant::now()
.checked_add(probe_timeout)
.is_none()
{
return Err(ResourceRegistryError::InvalidProbeTimeout);
}
self.probe_timeout = probe_timeout;
Ok(self)
}
pub fn with_run_timeout(mut self, timeout: Duration) -> Result<Self, ResourceRegistryError> {
if timeout.is_zero() || Instant::now().checked_add(timeout).is_none() {
return Err(ResourceRegistryError::InvalidRunTimeout);
}
self.run_timeout = Some(timeout);
Ok(self)
}
pub fn with_max_concurrency(mut self, limit: usize) -> Result<Self, ResourceRegistryError> {
if limit == 0 {
return Err(ResourceRegistryError::InvalidMaxConcurrency);
}
self.max_concurrency = limit;
Ok(self)
}
pub fn with_probe(
mut self,
probe: Arc<dyn ResourceProbe>,
) -> Result<Self, ResourceRegistryError> {
self.push_probe(probe)?;
Ok(self)
}
pub fn push_probe(
&mut self,
probe: Arc<dyn ResourceProbe>,
) -> Result<(), ResourceRegistryError> {
if self.probes.len() >= super::model::MAX_RESOURCE_REPORT_RESOURCES {
return Err(ResourceRegistryError::TooManyResources {
max: super::model::MAX_RESOURCE_REPORT_RESOURCES,
});
}
let index = self.probes.len();
let descriptor = probe.descriptor();
if let Some(first_index) = self
.probes
.iter()
.position(|registered| registered.descriptor().id() == descriptor.id())
{
return Err(ResourceRegistryError::DuplicateResourceId {
id: descriptor.id().to_owned(),
first_index,
second_index: index,
});
}
self.probes.push(probe);
Ok(())
}
#[must_use]
pub fn service_name(&self) -> &str {
&self.service_name
}
#[must_use]
pub fn probes(&self) -> &[Arc<dyn ResourceProbe>] {
&self.probes
}
#[must_use]
pub const fn probe_timeout(&self) -> Duration {
self.probe_timeout
}
#[must_use]
pub const fn run_timeout(&self) -> Option<Duration> {
self.run_timeout
}
#[must_use]
pub const fn max_concurrency(&self) -> usize {
self.max_concurrency
}
pub async fn run(&self) -> ResourceDiagnosticsReport {
let started_at = Instant::now();
let deadline = self
.run_timeout
.map(|timeout| started_at.checked_add(timeout).unwrap_or(started_at));
let budget = async {
match deadline {
Some(deadline) => time::sleep_until(deadline).await,
None => std::future::pending().await,
}
};
tokio::pin!(budget);
let mut pending = stream::iter(
self.probes
.iter()
.enumerate()
.map(|(index, probe)| async move { (index, self.run_probe(probe.as_ref()).await) }),
)
.buffer_unordered(self.max_concurrency);
let mut diagnostics = vec![None; self.probes.len()];
loop {
tokio::select! {
biased;
() = &mut budget => break,
result = pending.next() => match result {
Some((index, diagnostic)) => diagnostics[index] = Some(diagnostic),
None => break,
}
}
}
drop(pending);
let diagnostics = diagnostics
.into_iter()
.zip(&self.probes)
.map(|(diagnostic, probe)| {
diagnostic.unwrap_or_else(|| {
ResourceDiagnostic::unknown(
probe.descriptor().clone(),
"resource diagnostics run budget exhausted",
)
})
})
.collect();
ResourceDiagnosticsReport::from_validated(self.service_name.clone(), diagnostics)
}
async fn run_probe(&self, probe: &dyn ResourceProbe) -> ResourceDiagnostic {
let descriptor = probe.descriptor();
match time::timeout(self.probe_timeout, probe.probe()).await {
Ok(Ok(mut diagnostic)) => {
if diagnostic.resource() != descriptor {
diagnostic.replace_resource(descriptor.clone());
}
diagnostic
}
Ok(Err(source)) => {
tracing::warn!(
target: "footing::diagnostics",
error_code = "resource_probe_failed",
resource_id = %descriptor.id(),
"resource diagnostics probe failed"
);
drop(source);
ResourceDiagnostic::unhealthy(descriptor.clone(), "resource probe failed")
}
Err(_) => {
tracing::warn!(
target: "footing::diagnostics",
error_code = "resource_probe_timed_out",
resource_id = %descriptor.id(),
"resource diagnostics probe timed out"
);
ResourceDiagnostic::unhealthy(descriptor.clone(), "resource probe timed out")
}
}
}
}
impl fmt::Debug for ResourceRegistry {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ResourceRegistry")
.field("service_name", &self.service_name)
.field("probes", &self.probes.len())
.field("probe_timeout", &self.probe_timeout)
.field("run_timeout", &self.run_timeout)
.field("max_concurrency", &self.max_concurrency)
.finish()
}
}
struct FnResourceProbe<F> {
descriptor: ResourceDescriptor,
probe: F,
}
#[async_trait]
impl<F, Fut> ResourceProbe for FnResourceProbe<F>
where
F: Fn(ResourceDescriptor) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<ResourceDiagnostic, BoxError>> + Send + 'static,
{
fn descriptor(&self) -> &ResourceDescriptor {
&self.descriptor
}
async fn probe(&self) -> Result<ResourceDiagnostic, BoxError> {
(self.probe)(self.descriptor.clone()).await
}
}