use std::{
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use super::*;
use crate::result::BoxError;
fn registry() -> ResourceRegistry {
ResourceRegistry::new("example_service").expect("static service name is valid")
}
fn required(id: &str, name: &str, kind: ResourceKind) -> ResourceDescriptor {
ResourceDescriptor::required(id, name, kind).expect("static descriptor is valid")
}
fn optional(id: &str, name: &str, kind: ResourceKind) -> ResourceDescriptor {
ResourceDescriptor::optional(id, name, kind).expect("static descriptor is valid")
}
#[test]
fn diagnostics_round_trip_json_and_binary_with_empty_and_populated_fields() {
fn round_trip<
T: serde::Serialize + serde::de::DeserializeOwned + PartialEq + std::fmt::Debug,
>(
value: &T,
) {
let json = serde_json::to_vec(value).unwrap();
assert_eq!(&serde_json::from_slice::<T>(&json).unwrap(), value);
let binary = postcard::to_allocvec(value).unwrap();
assert_eq!(&postcard::from_bytes::<T>(&binary).unwrap(), value);
}
round_trip(&ResourceDiagnosticsReport::new("test", vec![]).unwrap());
for populated in [false, true] {
let mut descriptor = required("database", "Database", ResourceKind::Database);
if populated {
descriptor = descriptor.with_label("role", "primary").unwrap();
}
round_trip(&descriptor);
for status in [
ResourceStatus::Healthy,
ResourceStatus::Degraded,
ResourceStatus::Unhealthy,
ResourceStatus::Unknown,
] {
let mut diagnostic =
ResourceDiagnostic::new(descriptor.clone(), status, "probe result");
if populated {
diagnostic = diagnostic
.with_detail("details")
.unwrap()
.with_remediation("retry")
.unwrap();
}
round_trip(&diagnostic);
round_trip(&ResourceDiagnosticsReport::new("test", vec![diagnostic]).unwrap());
}
}
}
#[tokio::test]
async fn registry_runs_static_probes_and_rolls_up_status() {
let contracts = ResourceDiagnostic::healthy(
required("contracts", "HTTP Contracts", ResourceKind::Configuration),
"contracts are valid",
)
.with_detail("schema_version=1")
.unwrap();
let smtp = ResourceDiagnostic::degraded(
optional("smtp", "SMTP", ResourceKind::Messaging),
"SMTP is not configured",
)
.with_remediation("configure SMTP before enabling email delivery")
.unwrap();
let registry = registry()
.with_probe(Arc::new(contracts.clone()))
.expect("first probe should register")
.with_probe(Arc::new(smtp.clone()))
.expect("second probe should register");
let report = registry.run().await;
assert!(report.ready());
assert_eq!(report.status(), ResourceStatus::Degraded);
assert_eq!(report.summary().total(), 2);
assert_eq!(report.summary().healthy(), 1);
assert_eq!(report.summary().degraded(), 1);
assert_eq!(report.resources(), &[contracts, smtp]);
assert_eq!(registry.run().await, report);
}
#[tokio::test]
async fn registry_runs_closure_probes_with_captured_state() {
let expected_schema_version = 1;
let registry = registry()
.with_probe(resource_probe(
required("contracts", "HTTP Contracts", ResourceKind::Http),
move |resource| async move {
Ok(ResourceDiagnostic::healthy(resource, "contracts are valid")
.with_detail(format!("schema_version={expected_schema_version}"))
.expect("static detail should validate"))
},
))
.expect("probe should register");
let report = registry.run().await;
assert!(report.ready());
assert_eq!(report.status(), ResourceStatus::Healthy);
assert_eq!(
report.resources()[0].details(),
["schema_version=1".to_owned()]
);
}
#[tokio::test]
async fn required_unhealthy_resource_blocks_readiness() {
let registry = registry()
.with_probe(Arc::new(ResourceDiagnostic::unhealthy(
required("oidc-key", "OIDC Key", ResourceKind::Cryptography),
"signing key is missing",
)))
.expect("probe should register");
let report = registry.run().await;
assert!(!report.ready());
assert_eq!(report.status(), ResourceStatus::Unhealthy);
assert_eq!(report.summary().readiness_blocking(), 1);
assert_eq!(report.readiness_blockers()[0].resource().id(), "oidc-key");
}
#[tokio::test]
async fn required_unknown_resource_blocks_readiness() {
let registry = registry()
.with_probe(Arc::new(ResourceDiagnostic::unknown(
required("database", "Database", ResourceKind::Database),
"database health was not checked",
)))
.expect("probe should register");
let report = registry.run().await;
assert!(!report.ready());
assert_eq!(report.status(), ResourceStatus::Unknown);
assert_eq!(report.summary().readiness_blocking(), 1);
}
#[tokio::test]
async fn registry_rejects_duplicate_resource_ids() {
let descriptor = required("config", "Configuration", ResourceKind::Configuration);
let registry = registry()
.with_probe(Arc::new(ResourceDiagnostic::healthy(
descriptor.clone(),
"ok",
)))
.expect("first probe should register")
.with_probe(Arc::new(ResourceDiagnostic::healthy(descriptor, "ok")));
assert!(matches!(
registry,
Err(ResourceRegistryError::DuplicateResourceId { .. })
));
}
#[tokio::test]
async fn registry_rejects_empty_service_name() {
assert!(matches!(
ResourceRegistry::new(" "),
Err(ResourceRegistryError::InvalidServiceName)
));
}
#[tokio::test]
async fn registry_rejects_probe_results_with_mismatched_descriptors() {
let registered = required("database", "Database", ResourceKind::Database)
.with_label("role", "primary")
.unwrap();
for returned in [
required("other", "Database", ResourceKind::Database)
.with_label("role", "primary")
.unwrap(),
optional("database", "Database", ResourceKind::Database)
.with_label("role", "primary")
.unwrap(),
registered.clone().with_label("role", "replica").unwrap(),
] {
let registry = registry()
.with_probe(resource_probe(registered.clone(), move |_| {
let returned = returned.clone();
async move {
Ok(
ResourceDiagnostic::healthy(returned, "private misleading result")
.with_detail("private wrong-resource details")
.unwrap()
.with_remediation("private wrong-resource remediation")
.unwrap(),
)
}
}))
.unwrap();
let report = registry.run().await;
assert!(!report.ready());
assert_eq!(report.summary().readiness_blocking(), 1);
let diagnostic = &report.resources()[0];
assert_eq!(diagnostic.resource(), ®istered);
assert_eq!(diagnostic.status(), ResourceStatus::Unhealthy);
assert_eq!(
diagnostic.summary(),
"resource probe returned mismatched descriptor"
);
assert!(diagnostic.details().is_empty());
assert!(diagnostic.remediations().is_empty());
assert!(!serde_json::to_string(&report).unwrap().contains("private"));
}
}
#[tokio::test]
async fn optional_probe_descriptor_mismatch_does_not_block_readiness() {
let registered = optional("smtp", "SMTP", ResourceKind::Messaging);
let registry = registry()
.with_probe(resource_probe(registered.clone(), |_| async move {
Ok(ResourceDiagnostic::healthy(
required("smtp", "SMTP", ResourceKind::Messaging),
"SMTP is reachable",
))
}))
.unwrap();
let report = registry.run().await;
assert!(report.ready());
assert_eq!(report.summary().readiness_blocking(), 0);
assert_eq!(report.status(), ResourceStatus::Degraded);
let diagnostic = &report.resources()[0];
assert_eq!(diagnostic.resource(), ®istered);
assert_eq!(diagnostic.status(), ResourceStatus::Unhealthy);
assert_eq!(
diagnostic.summary(),
"resource probe returned mismatched descriptor"
);
}
#[tokio::test]
async fn registry_preserves_matching_probe_results() {
let expected = ResourceDiagnostic::healthy(
required("database", "Database", ResourceKind::Database)
.with_label("role", "primary")
.unwrap(),
"database is reachable",
)
.with_detail("verified primary database")
.unwrap()
.with_remediation("keep monitoring")
.unwrap();
let returned = expected.clone();
let registry = registry()
.with_probe(resource_probe(expected.resource().clone(), move |_| {
let returned = returned.clone();
async move { Ok(returned) }
}))
.unwrap();
let report = registry.run().await;
assert!(report.ready());
assert_eq!(report.resources(), &[expected]);
}
#[tokio::test]
async fn registry_bounds_each_probe_and_reports_timeout() {
let registry = registry()
.with_probe_timeout(Duration::from_millis(1))
.expect("positive timeout should validate")
.with_probe(resource_probe(
required("database", "Database", ResourceKind::Database),
|_| async move { std::future::pending::<Result<ResourceDiagnostic, BoxError>>().await },
))
.expect("probe should register");
let report = registry.run().await;
assert!(!report.ready());
assert_eq!(report.resources()[0].summary(), "resource probe timed out");
assert!(report.resources()[0].details().is_empty());
}
#[tokio::test]
async fn registry_does_not_copy_probe_errors_into_serialized_reports() {
let private_detail = "private-probe-error";
let registry = registry()
.with_probe(resource_probe(
required("database", "Database", ResourceKind::Database),
move |_| async move {
Err::<ResourceDiagnostic, BoxError>(Box::new(std::io::Error::other(private_detail)))
},
))
.expect("probe should register");
let report = registry.run().await;
let wire = serde_json::to_string(&report).expect("report should serialize");
assert_eq!(report.resources()[0].summary(), "resource probe failed");
assert!(report.resources()[0].details().is_empty());
assert!(!wire.contains(private_detail));
}
#[test]
fn registry_rejects_invalid_probe_timeouts() {
for timeout in [Duration::ZERO, Duration::MAX] {
assert!(matches!(
registry().with_probe_timeout(timeout),
Err(ResourceRegistryError::InvalidProbeTimeout)
));
}
}
#[test]
fn registry_defaults_to_serial_probes_without_a_run_budget() {
let registry = registry();
assert_eq!(registry.probe_timeout(), Duration::from_secs(5));
assert_eq!(registry.run_timeout(), None);
assert_eq!(registry.max_concurrency(), 1);
for timeout in [Duration::ZERO, Duration::MAX] {
assert!(matches!(
registry.clone().with_run_timeout(timeout),
Err(ResourceRegistryError::InvalidRunTimeout)
));
}
assert!(matches!(
registry.with_max_concurrency(0),
Err(ResourceRegistryError::InvalidMaxConcurrency)
));
}
#[derive(Default)]
struct ProbeActivity {
active: AtomicUsize,
peak: AtomicUsize,
started: Mutex<Vec<&'static str>>,
}
struct ActiveProbe(Arc<ProbeActivity>);
impl Drop for ActiveProbe {
fn drop(&mut self) {
self.0.active.fetch_sub(1, Ordering::SeqCst);
}
}
fn tracked_probe(
activity: &Arc<ProbeActivity>,
id: &'static str,
delay: Duration,
) -> Arc<dyn ResourceProbe> {
let activity = Arc::clone(activity);
resource_probe(required(id, id, ResourceKind::Other), move |descriptor| {
let activity = Arc::clone(&activity);
async move {
let active = activity.active.fetch_add(1, Ordering::SeqCst) + 1;
activity.peak.fetch_max(active, Ordering::SeqCst);
activity.started.lock().unwrap().push(id);
let _active = ActiveProbe(activity);
tokio::time::sleep(delay).await;
Ok(ResourceDiagnostic::healthy(descriptor, "ready"))
}
})
}
#[tokio::test(start_paused = true)]
async fn registry_reuses_finished_slots_and_preserves_report_order() {
let activity = Arc::new(ProbeActivity::default());
let mut registry = registry().with_max_concurrency(2).unwrap();
for (id, seconds) in [("first", 3), ("second", 1), ("third", 1)] {
registry
.push_probe(tracked_probe(&activity, id, Duration::from_secs(seconds)))
.unwrap();
}
let started_at = tokio::time::Instant::now();
let report = registry.run().await;
assert_eq!(started_at.elapsed(), Duration::from_secs(3));
assert_eq!(activity.peak.load(Ordering::SeqCst), 2);
assert_eq!(activity.active.load(Ordering::SeqCst), 0);
assert_eq!(
report
.resources()
.iter()
.map(|diagnostic| diagnostic.resource().id())
.collect::<Vec<_>>(),
["first", "second", "third"]
);
assert!(report.ready());
}
#[tokio::test(start_paused = true)]
async fn registry_budget_cancels_active_probes_without_starting_queued_probes() {
let activity = Arc::new(ProbeActivity::default());
let mut registry = registry()
.with_max_concurrency(2)
.unwrap()
.with_run_timeout(Duration::from_secs(2))
.unwrap();
for (id, seconds) in [("first", 1), ("second", 60), ("third", 60), ("fourth", 60)] {
registry
.push_probe(tracked_probe(&activity, id, Duration::from_secs(seconds)))
.unwrap();
}
let started_at = tokio::time::Instant::now();
let report = registry.run().await;
assert_eq!(started_at.elapsed(), Duration::from_secs(2));
assert_eq!(registry.run_timeout(), Some(Duration::from_secs(2)));
assert_eq!(activity.active.load(Ordering::SeqCst), 0);
assert_eq!(activity.peak.load(Ordering::SeqCst), 2);
assert_eq!(
*activity.started.lock().unwrap(),
["first", "second", "third"]
);
assert_eq!(report.resources()[0].status(), ResourceStatus::Healthy);
for diagnostic in &report.resources()[1..] {
assert_eq!(diagnostic.status(), ResourceStatus::Unknown);
assert_eq!(
diagnostic.summary(),
"resource diagnostics run budget exhausted"
);
}
assert!(!report.ready());
tokio::time::advance(Duration::from_secs(60)).await;
assert_eq!(activity.active.load(Ordering::SeqCst), 0);
assert_eq!(activity.started.lock().unwrap().len(), 3);
}
#[tokio::test(start_paused = true)]
async fn registry_distinguishes_probe_timeouts_from_exhausted_run_budget() {
let activity = Arc::new(ProbeActivity::default());
let mut registry = registry()
.with_probe_timeout(Duration::from_secs(1))
.unwrap()
.with_run_timeout(Duration::from_millis(1500))
.unwrap();
for id in ["first", "second", "third"] {
registry
.push_probe(tracked_probe(&activity, id, Duration::from_secs(60)))
.unwrap();
}
let started_at = tokio::time::Instant::now();
let report = registry.run().await;
assert_eq!(started_at.elapsed(), Duration::from_millis(1500));
assert_eq!(activity.peak.load(Ordering::SeqCst), 1);
assert_eq!(activity.active.load(Ordering::SeqCst), 0);
assert_eq!(*activity.started.lock().unwrap(), ["first", "second"]);
assert_eq!(report.resources()[0].status(), ResourceStatus::Unhealthy);
assert_eq!(report.resources()[0].summary(), "resource probe timed out");
for diagnostic in &report.resources()[1..] {
assert_eq!(diagnostic.status(), ResourceStatus::Unknown);
assert_eq!(
diagnostic.summary(),
"resource diagnostics run budget exhausted"
);
}
}
#[tokio::test(start_paused = true)]
async fn cancelling_a_registry_run_drops_every_active_probe() {
let activity = Arc::new(ProbeActivity::default());
let mut registry = registry().with_max_concurrency(2).unwrap();
for id in ["first", "second", "third"] {
registry
.push_probe(tracked_probe(&activity, id, Duration::from_secs(60)))
.unwrap();
}
assert!(
tokio::time::timeout(Duration::from_secs(1), registry.run())
.await
.is_err()
);
assert_eq!(activity.active.load(Ordering::SeqCst), 0);
assert_eq!(activity.peak.load(Ordering::SeqCst), 2);
assert_eq!(*activity.started.lock().unwrap(), ["first", "second"]);
tokio::time::advance(Duration::from_secs(60)).await;
assert_eq!(activity.started.lock().unwrap().len(), 2);
}
#[test]
fn descriptor_construction_and_deserialization_share_validation() {
assert!(matches!(
ResourceDescriptor::required("", "Database", ResourceKind::Database),
Err(ResourceDescriptorError::EmptyId)
));
assert!(matches!(
ResourceDescriptor::required("bad id", "Database", ResourceKind::Database),
Err(ResourceDescriptorError::InvalidId)
));
assert!(matches!(
required("database", "Database", ResourceKind::Database).with_label("", "true"),
Err(ResourceDescriptorError::EmptyLabelKey)
));
assert!(
serde_json::from_value::<ResourceDescriptor>(serde_json::json!({
"id": "",
"name": "Database",
"kind": "database",
"criticality": "required"
}))
.is_err()
);
}
#[test]
fn diagnostic_debug_redacts_labels_and_operator_messages_without_changing_wire_data() {
let label = "private-resource-label";
let summary = "private-resource-summary";
let detail = "private-resource-detail";
let remediation = "private-resource-remediation";
let descriptor = required("database", "Database", ResourceKind::Database)
.with_label("endpoint", label)
.expect("resource label should validate");
let diagnostic = ResourceDiagnostic::unhealthy(descriptor, summary)
.with_detail(detail)
.expect("detail should validate")
.with_remediation(remediation)
.expect("remediation should validate");
let output = format!("{diagnostic:?}");
assert!(output.contains("label_count"));
for sensitive in [label, summary, detail, remediation] {
assert!(!output.contains(sensitive));
}
let wire = serde_json::to_value(&diagnostic).expect("diagnostic should serialize");
assert_eq!(wire["resource"]["labels"]["endpoint"], label);
assert_eq!(wire["summary"], summary);
assert_eq!(wire["details"][0], detail);
assert_eq!(wire["remediations"][0], remediation);
}
#[test]
fn report_deserialization_rejects_inconsistent_or_incomplete_aggregates() {
let report = ResourceDiagnosticsReport::new(
"example_service",
vec![ResourceDiagnostic::unhealthy(
required("database", "Database", ResourceKind::Database),
"database is unavailable",
)],
)
.unwrap();
let wire = serde_json::to_value(&report).unwrap();
for status in [
ResourceStatus::Healthy,
ResourceStatus::Degraded,
ResourceStatus::Unknown,
] {
let mut inconsistent = wire.clone();
inconsistent["status"] = serde_json::to_value(status).unwrap();
assert!(serde_json::from_value::<ResourceDiagnosticsReport>(inconsistent).is_err());
}
for field in [
"total",
"healthy",
"degraded",
"unhealthy",
"unknown",
"readiness_blocking",
] {
let mut inconsistent = wire.clone();
inconsistent["summary"][field] =
(inconsistent["summary"][field].as_u64().unwrap() + 1).into();
assert!(serde_json::from_value::<ResourceDiagnosticsReport>(inconsistent).is_err());
}
for field in ["status", "summary", "resources"] {
let mut incomplete = wire.clone();
incomplete.as_object_mut().unwrap().remove(field);
assert!(serde_json::from_value::<ResourceDiagnosticsReport>(incomplete).is_err());
}
let mut missing_checks = wire;
missing_checks["resources"] = serde_json::json!([]);
assert!(serde_json::from_value::<ResourceDiagnosticsReport>(missing_checks).is_err());
let mut empty =
serde_json::to_value(ResourceDiagnosticsReport::new("example_service", vec![]).unwrap())
.unwrap();
empty["status"] = serde_json::json!("unhealthy");
assert!(serde_json::from_value::<ResourceDiagnosticsReport>(empty).is_err());
}
#[test]
fn descriptor_labels_are_bounded_for_builders_and_serde() {
let mut descriptor = required("database", "Database", ResourceKind::Database);
for index in 0..MAX_RESOURCE_LABELS {
descriptor = descriptor
.with_label(format!("label_{index}"), "value")
.expect("label within the bound should validate");
}
descriptor = descriptor
.with_label("label_0", "updated")
.expect("updating an existing label should not increase cardinality");
assert_eq!(descriptor.labels().len(), MAX_RESOURCE_LABELS);
assert_eq!(
descriptor.labels().get("label_0").map(String::as_str),
Some("updated")
);
assert!(matches!(
descriptor.with_label("one_more", "value"),
Err(ResourceDescriptorError::TooManyLabels { .. })
));
let labels = (0..=MAX_RESOURCE_LABELS)
.map(|index| {
(
format!("label_{index}"),
serde_json::Value::String("value".to_owned()),
)
})
.collect::<serde_json::Map<_, _>>();
assert!(
serde_json::from_value::<ResourceDescriptor>(serde_json::json!({
"id": "database",
"name": "Database",
"kind": "database",
"criticality": "required",
"labels": labels
}))
.is_err()
);
}
#[test]
fn diagnostic_messages_and_collections_are_bounded() {
let mut diagnostic = ResourceDiagnostic::unhealthy(
required("database", "Database", ResourceKind::Database),
format!(
" unavailable\n{} ",
"界".repeat(MAX_RESOURCE_MESSAGE_LENGTH)
),
);
assert!(diagnostic.summary().len() <= MAX_RESOURCE_MESSAGE_LENGTH);
assert!(!diagnostic.summary().chars().any(char::is_control));
for _ in 0..MAX_RESOURCE_DIAGNOSTIC_DETAILS {
diagnostic = diagnostic
.with_detail("bounded detail")
.expect("detail within the bound should validate");
}
assert!(matches!(
diagnostic.clone().with_detail("one more"),
Err(ResourceDiagnosticError::TooManyDetails { .. })
));
assert!(
diagnostic
.clone()
.with_detail(" \n\t ")
.expect("an empty normalized detail is a no-op")
.details()
.len()
== MAX_RESOURCE_DIAGNOSTIC_DETAILS
);
for _ in 0..MAX_RESOURCE_DIAGNOSTIC_REMEDIATIONS {
diagnostic = diagnostic
.with_remediation("bounded remediation")
.expect("remediation within the bound should validate");
}
assert!(matches!(
diagnostic.with_remediation("one more"),
Err(ResourceDiagnosticError::TooManyRemediations { .. })
));
}
#[test]
fn report_rejects_duplicate_and_excess_resources() {
let descriptor = required("database", "Database", ResourceKind::Database);
let duplicate = ResourceDiagnosticsReport::new(
"example_service",
vec![
ResourceDiagnostic::healthy(descriptor.clone(), "ok"),
ResourceDiagnostic::healthy(descriptor, "ok"),
],
);
assert!(matches!(
duplicate,
Err(ResourceRegistryError::DuplicateResourceId { .. })
));
let resources = (0..=MAX_RESOURCE_REPORT_RESOURCES)
.map(|index| {
ResourceDiagnostic::healthy(
required(
&format!("resource_{index}"),
&format!("Resource {index}"),
ResourceKind::Other,
),
"ok",
)
})
.collect();
assert!(matches!(
ResourceDiagnosticsReport::new("example_service", resources),
Err(ResourceRegistryError::TooManyResources { .. })
));
assert!(matches!(
ResourceDiagnosticsReport::new(
"x".repeat(MAX_DIAGNOSTICS_SERVICE_NAME_LENGTH + 1),
Vec::new()
),
Err(ResourceRegistryError::ServiceNameTooLong { .. })
));
}
#[test]
fn diagnostic_deserialization_enforces_collection_bounds() {
let details = vec!["detail"; MAX_RESOURCE_DIAGNOSTIC_DETAILS + 1];
assert!(
serde_json::from_value::<ResourceDiagnostic>(serde_json::json!({
"resource": {
"id": "database",
"name": "Database",
"kind": "database",
"criticality": "required"
},
"status": "unhealthy",
"summary": "down",
"details": details
}))
.is_err()
);
}